drivers/qat: QAT symmetric crypto datapatch rework

index cde421eb77..0fda890075 100644
@@ -549,6 +549,22 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 	return data & modulo_mask;
 }
 
+uint16_t
+refactor_qat_enqueue_op_burst(__rte_unused void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		__rte_unused void **ops, __rte_unused uint16_t nb_ops)
+{
+	return 0;
+}
+
+uint16_t
+refactor_qat_dequeue_op_burst(__rte_unused void *qp, __rte_unused void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		__rte_unused uint16_t nb_ops)
+{
+	return 0;
+}
+
 uint16_t
 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 {

index deafb407b3..c5f115310c 100644
@@ -36,6 +36,19 @@ struct qat_queue {
 	/* number of responses processed since last CSR head write */
 };
 
+/**
+ * @brief Function prototype to build a QAT request.
+ * @param opaque: an opaque data may be used to store context may be useful
+ *    between 2 enqueue operations.
+ **/
+typedef int (*qat_op_build_request_t)(void *in_op, uint8_t *out_msg,
+		void *op_cookie, uint64_t *opaque, enum qat_device_gen dev_gen);
+
+typedef int (*qat_op_dequeue_t)(void **op, uint8_t *resp, void *op_cookie,
+		uint64_t *dequeue_err_count __rte_unused);
+
+#define QAT_BUILD_REQUEST_MAX_OPAQUE_SIZE	2
+
 struct qat_qp {
 	void			*mmap_bar_addr;
 	struct qat_queue	tx_q;

@@ -44,6 +57,7 @@ struct qat_qp {
 	struct rte_mempool *op_cookie_pool;
 	void **op_cookies;
 	uint32_t nb_descriptors;
+	uint64_t opaque[QAT_BUILD_REQUEST_MAX_OPAQUE_SIZE];
 	enum qat_device_gen qat_dev_gen;
 	enum qat_service_type service_type;
 	struct qat_pci_device *qat_dev;

@@ -77,6 +91,14 @@ struct qat_qp_config {
 	const char *service_str;
 };
 
+uint16_t
+refactor_qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops);
+
+uint16_t
+refactor_qat_dequeue_op_burst(void *qp, void **ops,
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops);
+
 uint16_t
 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops);
 

index 85973812a8..d5b4c66d68 100644
@@ -456,6 +456,41 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
+static __rte_always_inline int
+refactor_qat_asym_build_request(__rte_unused void *in_op,
+		__rte_unused uint8_t *out_msg, __rte_unused void *op_cookie,
+		__rte_unused uint64_t *opaque,
+		__rte_unused enum qat_device_gen dev_gen)
+{
+	return 0;
+}
+
+int
+refactor_qat_asym_process_response(__rte_unused void **op,
+		__rte_unused uint8_t *resp,
+		__rte_unused void *op_cookie,
+		__rte_unused uint64_t *dequeue_err_count)
+{
+	return 0;
+}
+
+uint16_t
+qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_enqueue_op_burst(qp,
+					refactor_qat_asym_build_request,
+					(void **)ops, nb_ops);
+}
+
+uint16_t
+qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_dequeue_op_burst(qp, (void **)ops,
+				refactor_qat_asym_process_response, nb_ops);
+}
+
 int
 qat_asym_build_request(void *in_op,
 			uint8_t *out_msg,

index 308b6b2e0b..50c2641eba 100644
@@ -92,4 +92,19 @@ void
 qat_asym_process_response(void __rte_unused **op, uint8_t *resp,
 		void *op_cookie);
 
+int
+refactor_qat_asym_process_response(__rte_unused void **op,
+		__rte_unused uint8_t *resp,
+		__rte_unused void *op_cookie,
+		__rte_unused uint64_t *dequeue_err_count);
+
+uint16_t
+qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
+
+uint16_t
+qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
 #endif /* _QAT_ASYM_H_ */

index 93b257522b..a92874cd27 100644
@@ -210,6 +210,31 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 			ICP_QAT_FW_LA_NO_PROTO);
 }
 
+static __rte_always_inline int
+refactor_qat_sym_build_request(__rte_unused void *in_op,
+		__rte_unused uint8_t *out_msg, __rte_unused	void *op_cookie,
+		__rte_unused uint64_t *opaque,
+		__rte_unused enum qat_device_gen dev_gen)
+{
+	return 0;
+}
+
+uint16_t
+refactor_qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_enqueue_op_burst(qp, refactor_qat_sym_build_request,
+			(void **)ops, nb_ops);
+}
+
+uint16_t
+refactor_qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_dequeue_op_burst(qp, (void **)ops,
+				refactor_qat_sym_process_response, nb_ops);
+}
+
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen)

index e3ec7f0de4..17b2c871bd 100644
@@ -54,6 +54,22 @@ struct qat_sym_op_cookie {
 	} opt;
 };
 
+static __rte_always_inline int
+refactor_qat_sym_process_response(__rte_unused void **op,
+		__rte_unused uint8_t *resp, __rte_unused void *op_cookie,
+		__rte_unused uint64_t *dequeue_err_count)
+{
+	return 0;
+}
+
+uint16_t
+refactor_qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
+uint16_t
+refactor_qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen);

index 6ebc176729..73493ec864 100644
@@ -55,6 +55,11 @@
 #define QAT_SESSION_IS_SLICE_SET(flags, flag)	\
 	(!!((flags) & (flag)))
 
+struct qat_sym_session;
+
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_NONE = 0,
 	QAT_CRYPTO_PROTO_FLAG_CCM = 1,

@@ -107,6 +112,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

index 67a4d2cb2c..07020741bd 100644
@@ -8,14 +8,1082 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_BASE_GEN1_SYM_CAPABILITIES \
+	QAT_SYM_PLAIN_AUTH_CAP(SHA1, CAP_SET(block_size, 64), \
+		CAP_RNG(digest_size, 1, 20, 1)), \
+	QAT_SYM_AEAD_CAP(AES_GCM, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AEAD_CAP(AES_CCM,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), \
+		CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), \
+	QAT_SYM_AUTH_CAP(AES_GMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AUTH_CAP(AES_CMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), \
+			CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA1_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(MD5_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_XCBC_MAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SNOW3G_UIA2, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(KASUMI_F9, CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(NULL, CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_CBC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_CTR,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_XTS,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(SNOW3G_UEA2,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(KASUMI_F8,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(NULL,  CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(3DES_CBC,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(3DES_CTR,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(DES_CBC,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(DES_DOCSISBPI,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0))
+
+#define QAT_BASE_GEN1_ASYM_CAPABILITIES				\
+	QAT_ASYM_CAP(MODEX, 0, 1, 512, 1),			\
+	QAT_ASYM_CAP(MODINV, 0, 1, 512, 1),			\
+	QAT_ASYM_CAP(RSA,					\
+			((1 << RTE_CRYPTO_ASYM_OP_SIGN) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_VERIFY) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_ENCRYPT) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_DECRYPT)),	\
+			64, 512, 64)
+
+#define QAT_EXTRA_GEN2_SYM_CAPABILITIES \
+	QAT_SYM_CIPHER_CAP(ZUC_EEA3, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(ZUC_EIA3, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)) \
+
+#define QAT_EXTRA_GEN3_SYM_CAPABILITIES \
+	QAT_SYM_AEAD_CAP(CHACHA20_POLY1305, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 32, 32, 0), \
+		CAP_RNG(digest_size, 16, 16, 0), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 12, 12, 0))
+
+#define QAT_BASE_GEN4_SYM_CAPABILITIES \
+	QAT_SYM_CIPHER_CAP(AES_CBC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(SHA1_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_XCBC_MAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_CMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(NULL, CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(NULL,  CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_PLAIN_AUTH_CAP(SHA1, CAP_SET(block_size, 64), \
+		CAP_RNG(digest_size, 1, 20, 1)), \
+	QAT_SYM_AUTH_CAP(SHA224, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_CTR,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AEAD_CAP(AES_GCM, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AEAD_CAP(AES_CCM,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), \
+		CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), \
+	QAT_SYM_AUTH_CAP(AES_GMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)) \
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_chk_len_in_bits_auth(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_chk_len_in_bits_cipher(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_chk_len_in_bits_cipher(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_chk_len_in_bits_auth(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_chk_len_in_bits_cipher(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_chk_len_in_bits_auth(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+/* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
+int
+qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest __rte_unused,
+	struct rte_crypto_va_iova_ptr *aad __rte_unused,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv __rte_unused,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+void *
+qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
+	int *dequeue_status, enum rte_crypto_op_status *op_status);
+
+uint32_t
+qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
+	rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
+	uint32_t max_nb_to_dequeue,
+	rte_cryptodev_raw_post_dequeue_t post_dequeue,
+	void **out_user_data, uint8_t is_user_data_array,
+	uint32_t *n_success_jobs, int *return_status);
+
+int
+qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
+int
+qat_sym_dp_denqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
+int
+qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

@@ -23,6 +1091,9 @@ qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 struct qat_capabilities_info
 qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 

901 lines not on this page · raw message

index 17b2c871bd..4801bd50a7 100644
@@ -54,6 +54,14 @@ struct qat_sym_op_cookie {
 	} opt;
 };
 
+struct qat_sym_dp_ctx {
+	struct qat_sym_session *session;
+	uint32_t tail;
+	uint32_t head;
+	uint16_t cached_enqueue;
+	uint16_t cached_dequeue;
+};
+
 static __rte_always_inline int
 refactor_qat_sym_process_response(__rte_unused void **op,
 		__rte_unused uint8_t *resp, __rte_unused void *op_cookie,

index 12825e448b..94589458d0 100644
@@ -13,14 +13,6 @@
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

409 lines not on this page · raw message

402 lines not on this page · raw message

index 9ed1f21d9d..99d90fa56c 100644
@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

994 lines not on this page · raw message

125 lines not on this page · raw message

index b4ec440e05..72609703f9 100644
@@ -166,6 +166,90 @@ qat_sym_crypto_qp_setup_gen2(struct rte_cryptodev *dev, uint16_t qp_id,
 	return 0;
 }
 
+void
+qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
+		uint8_t hash_flag)
+{
+	struct icp_qat_fw_comn_req_hdr *header = &session->fw_req.comn_hdr;
+	struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *cd_ctrl =
+			(struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *)
+			session->fw_req.cd_ctrl.content_desc_ctrl_lw;
+
+	/* Set the Use Extended Protocol Flags bit in LW 1 */
+	QAT_FIELD_SET(header->comn_req_flags,
+			QAT_COMN_EXT_FLAGS_USED,
+			QAT_COMN_EXT_FLAGS_BITPOS,
+			QAT_COMN_EXT_FLAGS_MASK);
+
+	/* Set Hash Flags in LW 28 */
+	cd_ctrl->hash_flags |= hash_flag;
+
+	/* Set proto flags in LW 1 */
+	switch (session->qat_cipher_alg) {
+	case ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2:
+		ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
+				ICP_QAT_FW_LA_SNOW_3G_PROTO);
+		ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
+				header->serv_specif_flags, 0);
+		break;
+	case ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3:
+		ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
+				ICP_QAT_FW_LA_NO_PROTO);
+		ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
+				header->serv_specif_flags,
+				ICP_QAT_FW_LA_ZUC_3G_PROTO);
+		break;
+	default:
+		ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
+				ICP_QAT_FW_LA_NO_PROTO);
+		ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
+				header->serv_specif_flags, 0);
+		break;
+	}
+}
+
+static int
+qat_sym_crypto_set_session_gen2(void *cdev, void *session)
+{
+	struct rte_cryptodev *dev = cdev;
+	struct qat_sym_session *ctx = session;
+	const struct qat_cryptodev_private *qat_private =
+			dev->data->dev_private;
+	int ret;
+
+	ret = qat_sym_crypto_set_session_gen1(cdev, session);
+	if (ret == 0 || ret != -ENOTSUP)
+		return ret;
+
+	/* GEN1 returning -ENOTSUP as it cannot handle some mixed algo,
+	 * but some are not supported by GEN2, so checking here
+	 */
+	if ((qat_private->internal_capabilities &
+			QAT_SYM_CAP_MIXED_CRYPTO) == 0)
+		return -ENOTSUP;
+
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
+			ctx->qat_cipher_alg !=
+			ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		qat_sym_session_set_ext_hash_flags_gen2(ctx,
+			1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
+	} else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
+			ctx->qat_cipher_alg !=
+			ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
+		qat_sym_session_set_ext_hash_flags_gen2(ctx,
+			1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
+	} else if ((ctx->aes_cmac ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) &&
+			(ctx->qat_cipher_alg ==
+			ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+			ctx->qat_cipher_alg ==
+			ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
+		qat_sym_session_set_ext_hash_flags_gen2(ctx, 0);
+	}
+
+	return 0;
+}
+
 struct rte_cryptodev_ops qat_sym_crypto_ops_gen2 = {
 
 	/* Device related operations */

@@ -204,6 +288,10 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 	qat_sym_gen_dev_ops[QAT_GEN2].cryptodev_ops = &qat_sym_crypto_ops_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_capabilities =
 			qat_sym_crypto_cap_get_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
+			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;
 

@@ -221,4 +309,6 @@ RTE_INIT(qat_asym_crypto_gen2_init)
 			qat_asym_crypto_cap_get_gen1;
 	qat_asym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_asym_crypto_feature_flags_get_gen1;
+	qat_asym_gen_dev_ops[QAT_GEN2].set_session =
+			qat_asym_crypto_set_session_gen1;
 }

483 lines not on this page · raw message

258 lines not on this page · raw message

index e1fd14956b..e32bdf1c4b 100644
@@ -6,6 +6,7 @@
 #ifdef RTE_LIB_SECURITY
 #include <rte_security_driver.h>
 #endif
+#include <cryptodev_pmd.h>
 
 #include "adf_transport_access_macros.h"
 #include "icp_qat_fw.h"

@@ -154,7 +155,7 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 };
 
 static struct qat_capabilities_info
-qat_sym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev __rte_unused)
+qat_sym_crypto_cap_get_gen1(struct qat_pci_device * qat_dev __rte_unused)
 {
 	struct qat_capabilities_info capa_info;
 	capa_info.data = qat_sym_crypto_caps_gen1;

@@ -1169,6 +1170,10 @@ RTE_INIT(qat_sym_crypto_gen1_init)
 	qat_sym_gen_dev_ops[QAT_GEN1].cryptodev_ops = &qat_sym_crypto_ops_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN1].get_capabilities =
 			qat_sym_crypto_cap_get_gen1;
+	qat_sym_gen_dev_ops[QAT_GEN1].set_session =
+			qat_sym_crypto_set_session_gen1;
+	qat_sym_gen_dev_ops[QAT_GEN1].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN1].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;
 #ifdef RTE_LIB_SECURITY

index 01d2439b93..6f17abd404 100644
@@ -2,6 +2,7 @@
  * Copyright(c) 2021 Intel Corporation
  */
 
+#include <cryptodev_pmd.h>
 #include "qat_device.h"
 #include "qat_qp.h"
 #include "qat_crypto.h"

index 6eaa15b975..8fe3f0b061 100644
@@ -48,15 +48,24 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

index a92874cd27..de687de857 100644
@@ -237,7 +237,7 @@ refactor_qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -302,12 +302,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

186 lines not on this page · raw message

index 73493ec864..e1b5edb2f9 100644
@@ -100,7 +100,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

99 lines not on this page · raw message

index c5f115310c..b4518df7d0 100644
@@ -14,6 +14,24 @@
 
 struct qat_pci_device;
 
+/* Default qp configuration for GEN4 devices */
+#define QAT_GEN4_QP_DEFCON	(QAT_SERVICE_SYMMETRIC |	\
+				QAT_SERVICE_SYMMETRIC << 8 |	\
+				QAT_SERVICE_SYMMETRIC << 16 |	\
+				QAT_SERVICE_SYMMETRIC << 24)
+
+/* QAT GEN 4 specific macros */
+#define QAT_GEN4_BUNDLE_NUM             4
+#define QAT_GEN4_QPS_PER_BUNDLE_NUM     1
+
+/* Queue pair setup error codes */
+#define QAT_NOMEM		1
+#define QAT_QP_INVALID_DESC_NO	2
+#define QAT_QP_BUSY		3
+#define QAT_PCI_NO_RESOURCE	4
+
+#define ADF_ARB_RINGSRVARBEN_OFFSET		0x19C
+
 /**
  * Structure associated with each queue.
  */

@@ -92,21 +110,15 @@ struct qat_qp_config {
 };
 
 uint16_t
-refactor_qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops);
 
-uint16_t
-refactor_qat_dequeue_op_burst(void *qp, void **ops,
-		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops);
-
-uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops);
-
 uint16_t
 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops);
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops);
+qat_dequeue_op_burst(void *qp, void **ops,
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops);
 
 int
 qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr);

@@ -118,7 +130,7 @@ qat_qp_setup(struct qat_pci_device *qat_dev,
 
 int
 qat_qps_per_service(struct qat_pci_device *qat_dev,
-		enum qat_service_type service);
+			enum qat_service_type service);
 
 const struct qat_qp_hw_data *
 qat_qp_get_hw_data(struct qat_pci_device *qat_dev,

index 9b24d46e97..2f17f8825e 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

index 72609703f9..86c124f6c8 100644
@@ -5,8 +5,8 @@
 #include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
 #include "qat_sym_session.h"
-#include "qat_sym.h"
-#include "qat_asym.h"
+#include "qat_sym_refactor.h"
+#include "qat_asym_refactor.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
 

index 167c95abcf..108e07ee7f 100644
@@ -5,8 +5,8 @@
 #include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
 #include "qat_sym_session.h"
-#include "qat_sym.h"
-#include "qat_asym.h"
+#include "qat_sym_refactor.h"
+#include "qat_asym_refactor.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
 

index e32bdf1c4b..aea56fba4c 100644
@@ -13,7 +13,7 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym_session.h"
-#include "qat_sym.h"
+#include "qat_sym_refactor.h"
 #include "qat_sym_session.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"

index 8e789920cb..3a9b1d4054 100644
@@ -5,7 +5,7 @@
 
 #include <stdarg.h>
 
-#include <rte_cryptodev_pmd.h>
+#include <cryptodev_pmd.h>
 
 #include "icp_qat_fw_pke.h"
 #include "icp_qat_fw.h"

@@ -14,7 +14,7 @@
 #include "qat_device.h"
 
 #include "qat_logs.h"
-#include "qat_asym.h"
+#include "qat_asym_refactor.h"
 
 uint8_t qat_asym_driver_id;
 

index 9ecabdbe8f..6d3d991bc7 100644
@@ -5,7 +5,7 @@
 #ifndef _QAT_ASYM_H_
 #define _QAT_ASYM_H_
 
-#include <rte_cryptodev_pmd.h>
+#include <cryptodev_pmd.h>
 #include <rte_crypto_asym.h>
 #include "icp_qat_fw_pke.h"
 #include "qat_device.h"

index 94589458d0..af75ac2011 100644
@@ -8,7 +8,7 @@
 #include "icp_qat_fw.h"
 #include "icp_qat_fw_la.h"
 
-#include "qat_sym.h"
+#include "qat_sym_refactor.h"
 #include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"

63 lines not on this page · raw message

index d4bfe8f364..44feca8251 100644
@@ -5,7 +5,7 @@
 #ifndef _QAT_SYM_H_
 #define _QAT_SYM_H_
 
-#include <rte_cryptodev_pmd.h>
+#include <cryptodev_pmd.h>
 #ifdef RTE_LIB_SECURITY
 #include <rte_net_crc.h>
 #endif

index 814a0b401d..dd791a181a 100644
@@ -179,6 +179,10 @@ post_process_raw_dp_op(void *user_data,	uint32_t index __rte_unused,
 			RTE_CRYPTO_OP_STATUS_ERROR;
 }
 
+static struct crypto_testsuite_params testsuite_params = { NULL };
+struct crypto_testsuite_params *p_testsuite_params = &testsuite_params;
+static struct crypto_unittest_params unittest_params;
+
 void
 process_sym_raw_dp_op(uint8_t dev_id, uint16_t qp_id,
 		struct rte_crypto_op *op, uint8_t is_cipher, uint8_t is_auth,

@@ -193,6 +197,7 @@ process_sym_raw_dp_op(uint8_t dev_id, uint16_t qp_id,
 	struct rte_crypto_sgl sgl, dest_sgl;
 	uint32_t max_len;
 	union rte_cryptodev_session_ctx sess;
+	uint64_t auth_end_iova;
 	uint32_t count = 0;
 	struct rte_crypto_raw_dp_ctx *ctx;
 	uint32_t cipher_offset = 0, cipher_len = 0, auth_offset = 0,

@@ -202,6 +207,9 @@ process_sym_raw_dp_op(uint8_t dev_id, uint16_t qp_id,
 	int ctx_service_size;
 	int32_t status = 0;
 	int enqueue_status, dequeue_status;
+	struct crypto_unittest_params *ut_params = &unittest_params;
+	/* oop is not supported in raw hw dp api*/
+	int is_sgl = sop->m_src->nb_segs > 1;
 
 	ctx_service_size = rte_cryptodev_get_raw_dp_ctx_size(dev_id);
 	if (ctx_service_size < 0) {

@@ -267,6 +275,30 @@ process_sym_raw_dp_op(uint8_t dev_id, uint16_t qp_id,
 		digest.va = (void *)sop->auth.digest.data;
 		digest.iova = sop->auth.digest.phys_addr;
 
+		if (is_sgl) {
+			uint32_t remaining_off = auth_offset + auth_len;
+			struct rte_mbuf *sgl_buf = sop->m_src;
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else {
+			/* oop is not supported in raw hw dp api */
+			auth_end_iova = rte_pktmbuf_iova(op->sym->m_src) +
+							 auth_offset + auth_len;
+		}
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_offset + auth_len < cipher_offset + cipher_len) &&
+				(digest.iova == auth_end_iova) && is_sgl)
+			max_len = RTE_MAX(max_len,
+				auth_offset + auth_len +
+				ut_params->auth_xform.auth.digest_length);
+
 	} else if (is_cipher) {
 		cipher_offset = sop->cipher.data.offset;
 		cipher_len = sop->cipher.data.length;

@@ -503,10 +535,6 @@ process_crypto_request(uint8_t dev_id, struct rte_crypto_op *op)
 	return op;
 }
 
-static struct crypto_testsuite_params testsuite_params = { NULL };
-struct crypto_testsuite_params *p_testsuite_params = &testsuite_params;
-static struct crypto_unittest_params unittest_params;
-
 static int
 testsuite_setup(void)
 {

@@ -4077,9 +4105,9 @@ test_kasumi_decryption(const struct kasumi_test_data *tdata)
 
 	/* Create KASUMI operation */
 	retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data,
-					tdata->cipher_iv.len,
-					tdata->ciphertext.len,
-					tdata->validCipherOffsetInBits.len);
+			tdata->cipher_iv.len,
+			RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8),
+			tdata->validCipherOffsetInBits.len);
 	if (retval < 0)
 		return retval;
 

@@ -7310,6 +7338,7 @@ test_mixed_auth_cipher(const struct mixed_cipher_auth_test_data *tdata,
 	unsigned int plaintext_len;
 	unsigned int ciphertext_pad_len;
 	unsigned int ciphertext_len;
+	unsigned int data_len;
 
 	struct rte_cryptodev_info dev_info;
 	struct rte_crypto_op *op;

@@ -7370,21 +7399,22 @@ test_mixed_auth_cipher(const struct mixed_cipher_auth_test_data *tdata,
 	plaintext_len = ceil_byte_length(tdata->plaintext.len_bits);
 	ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 16);
 	plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16);
+	data_len = RTE_MAX(ciphertext_pad_len, plaintext_pad_len);
 
 	if (verify) {
 		ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
-				ciphertext_pad_len);
+				data_len);
 		memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len);
 		if (op_mode == OUT_OF_PLACE)
-			rte_pktmbuf_append(ut_params->obuf, ciphertext_pad_len);
+			rte_pktmbuf_append(ut_params->obuf, data_len);
 		debug_hexdump(stdout, "ciphertext:", ciphertext,
 				ciphertext_len);
 	} else {
 		plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
-				plaintext_pad_len);
+				data_len);
 		memcpy(plaintext, tdata->plaintext.data, plaintext_len);
 		if (op_mode == OUT_OF_PLACE)
-			rte_pktmbuf_append(ut_params->obuf, plaintext_pad_len);
+			rte_pktmbuf_append(ut_params->obuf, data_len);
 		debug_hexdump(stdout, "plaintext:", plaintext, plaintext_len);
 	}
 

index 6494019050..c59c25fe8f 100644
@@ -467,8 +467,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 

@@ -564,8 +574,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

index 108e07ee7f..1b6cf10589 100644
@@ -295,8 +295,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 

80 lines not on this page · raw message

index 437996f2e8..e1fc1c37ec 100644
@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

index 82adda0698..56facf0fbe 100644
@@ -15,8 +15,8 @@
 #include "qat_logs.h"
 #include "qat_device.h"
 #include "qat_qp.h"
-#include "qat_sym_refactor.h"
-#include "qat_asym_refactor.h"
+#include "qat_sym.h"
+#include "qat_asym.h"
 #include "qat_comp.h"
 
 #define QAT_CQ_MAX_DEQ_RETRIES 10

index b4518df7d0..fc77173fc1 100644
@@ -130,7 +130,7 @@ qat_qp_setup(struct qat_pci_device *qat_dev,
 
 int
 qat_qps_per_service(struct qat_pci_device *qat_dev,
-			enum qat_service_type service);
+		enum qat_service_type service);
 
 const struct qat_qp_hw_data *
 qat_qp_get_hw_data(struct qat_pci_device *qat_dev,

index 86c124f6c8..72609703f9 100644
@@ -5,8 +5,8 @@
 #include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
 #include "qat_sym_session.h"
-#include "qat_sym_refactor.h"
-#include "qat_asym_refactor.h"
+#include "qat_sym.h"
+#include "qat_asym.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
 

index 1b6cf10589..529878e3e7 100644
@@ -5,8 +5,8 @@
 #include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
 #include "qat_sym_session.h"
-#include "qat_sym_refactor.h"
-#include "qat_asym_refactor.h"
+#include "qat_sym.h"
+#include "qat_asym.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
 

182 lines not on this page · raw message

index 85d5e45a42..8b88443a40 100644
@@ -13,7 +13,7 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym_session.h"
-#include "qat_sym_refactor.h"
+#include "qat_sym.h"
 #include "qat_sym_session.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"

879 lines not on this page · raw message

123 lines not on this page · raw message

231 lines not on this page · raw message

54 lines not on this page · raw message

994 lines not on this page · raw message

125 lines not on this page · raw message

index 8fe3f0b061..203f74f46f 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

337 lines not on this page · raw message

141 lines not on this page · raw message

975 lines not on this page · raw message

251 lines not on this page · raw message

95 lines not on this page · raw message

360 lines not on this page · raw message

402 lines not on this page · raw message

index 52837d7c9c..27cdeb1de2 100644

Lines of this file are not shown. View the raw message.

index cde421eb77..0fda890075 100644
@@ -549,6 +549,22 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 	return data & modulo_mask;
 }
 
+uint16_t
+refactor_qat_enqueue_op_burst(__rte_unused void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		__rte_unused void **ops, __rte_unused uint16_t nb_ops)
+{
+	return 0;
+}
+
+uint16_t
+refactor_qat_dequeue_op_burst(__rte_unused void *qp, __rte_unused void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		__rte_unused uint16_t nb_ops)
+{
+	return 0;
+}
+
 uint16_t
 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 {

72 lines not on this page · raw message

49 lines not on this page · raw message

index 308b6b2e0b..50c2641eba 100644
@@ -92,4 +92,19 @@ void
 qat_asym_process_response(void __rte_unused **op, uint8_t *resp,
 		void *op_cookie);
 
+int
+refactor_qat_asym_process_response(__rte_unused void **op,
+		__rte_unused uint8_t *resp,
+		__rte_unused void *op_cookie,
+		__rte_unused uint64_t *dequeue_err_count);
+
+uint16_t
+qat_asym_crypto_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
+
+uint16_t
+qat_asym_crypto_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
 #endif /* _QAT_ASYM_H_ */

index 93b257522b..576ef532a7 100644
@@ -210,6 +210,43 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 			ICP_QAT_FW_LA_NO_PROTO);
 }
 
+/*
+ * Sym build request refactor function template,
+ * will be removed in the later patchset.
+ */
+static __rte_always_inline int
+refactor_qat_sym_build_request(__rte_unused void *in_op,
+		__rte_unused uint8_t *out_msg, __rte_unused	void *op_cookie,
+		__rte_unused uint64_t *opaque,
+		__rte_unused enum qat_device_gen dev_gen)
+{
+	return 0;
+}
+
+/*
+ * Sym enqueue burst refactor function template,
+ * will be removed in the later patchset.
+ */
+uint16_t
+refactor_qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_enqueue_op_burst(qp, refactor_qat_sym_build_request,
+			(void **)ops, nb_ops);
+}
+
+/*
+ * Sym dequeue burst refactor function template,
+ * will be removed in the later patchset.
+ */
+uint16_t
+refactor_qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return refactor_qat_dequeue_op_burst(qp, (void **)ops,
+				refactor_qat_sym_process_response, nb_ops);
+}
+
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen)

index e3ec7f0de4..17b2c871bd 100644
@@ -54,6 +54,22 @@ struct qat_sym_op_cookie {
 	} opt;
 };
 
+static __rte_always_inline int
+refactor_qat_sym_process_response(__rte_unused void **op,
+		__rte_unused uint8_t *resp, __rte_unused void *op_cookie,
+		__rte_unused uint64_t *dequeue_err_count)
+{
+	return 0;
+}
+
+uint16_t
+refactor_qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
+uint16_t
+refactor_qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops);
+
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen);

index 6ebc176729..73493ec864 100644
@@ -55,6 +55,11 @@
 #define QAT_SESSION_IS_SLICE_SET(flags, flag)	\
 	(!!((flags) & (flag)))
 
+struct qat_sym_session;
+
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_NONE = 0,
 	QAT_CRYPTO_PROTO_FLAG_CCM = 1,

@@ -107,6 +112,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

index 67a4d2cb2c..07020741bd 100644
@@ -8,14 +8,1082 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_BASE_GEN1_SYM_CAPABILITIES \
+	QAT_SYM_PLAIN_AUTH_CAP(SHA1, CAP_SET(block_size, 64), \
+		CAP_RNG(digest_size, 1, 20, 1)), \
+	QAT_SYM_AEAD_CAP(AES_GCM, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AEAD_CAP(AES_CCM,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), \
+		CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), \
+	QAT_SYM_AUTH_CAP(AES_GMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AUTH_CAP(AES_CMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), \
+			CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA1_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(MD5_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_XCBC_MAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SNOW3G_UIA2, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(KASUMI_F9, CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(NULL, CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_CBC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_CTR,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_XTS,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(SNOW3G_UEA2,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_CIPHER_CAP(KASUMI_F8,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(NULL,  CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(3DES_CBC,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(3DES_CTR,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(DES_CBC,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)), \
+	QAT_SYM_CIPHER_CAP(DES_DOCSISBPI,  CAP_SET(block_size, 8), \
+		CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0))
+
+#define QAT_BASE_GEN1_ASYM_CAPABILITIES				\
+	QAT_ASYM_CAP(MODEX, 0, 1, 512, 1),			\
+	QAT_ASYM_CAP(MODINV, 0, 1, 512, 1),			\
+	QAT_ASYM_CAP(RSA,					\
+			((1 << RTE_CRYPTO_ASYM_OP_SIGN) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_VERIFY) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_ENCRYPT) |	\
+			(1 << RTE_CRYPTO_ASYM_OP_DECRYPT)),	\
+			64, 512, 64)
+
+#define QAT_EXTRA_GEN2_SYM_CAPABILITIES \
+	QAT_SYM_CIPHER_CAP(ZUC_EEA3, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(ZUC_EIA3, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)) \
+
+#define QAT_EXTRA_GEN3_SYM_CAPABILITIES \
+	QAT_SYM_AEAD_CAP(CHACHA20_POLY1305, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 32, 32, 0), \
+		CAP_RNG(digest_size, 16, 16, 0), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 12, 12, 0))
+
+#define QAT_BASE_GEN4_SYM_CAPABILITIES \
+	QAT_SYM_CIPHER_CAP(AES_CBC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(SHA1_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA224_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256_HMAC, CAP_SET(block_size, 64), \
+		CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512_HMAC, CAP_SET(block_size, 128), \
+		CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_XCBC_MAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(AES_CMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AUTH_CAP(NULL, CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(NULL,  CAP_SET(block_size, 1), \
+		CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_PLAIN_AUTH_CAP(SHA1, CAP_SET(block_size, 64), \
+		CAP_RNG(digest_size, 1, 20, 1)), \
+	QAT_SYM_AUTH_CAP(SHA224, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA256, CAP_SET(block_size, 64), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA384, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_AUTH_CAP(SHA512, CAP_SET(block_size, 128), \
+		CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)), \
+	QAT_SYM_CIPHER_CAP(AES_CTR,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)), \
+	QAT_SYM_AEAD_CAP(AES_GCM, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)), \
+	QAT_SYM_AEAD_CAP(AES_CCM,  CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2), \
+		CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)), \
+	QAT_SYM_AUTH_CAP(AES_GMAC, CAP_SET(block_size, 16), \
+		CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4), \
+		CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)) \
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_chk_len_in_bits_auth(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_chk_len_in_bits_cipher(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_chk_len_in_bits_cipher(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_chk_len_in_bits_auth(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_chk_len_in_bits_cipher(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_chk_len_in_bits_auth(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+/* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
+int
+qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest __rte_unused,
+	struct rte_crypto_va_iova_ptr *aad __rte_unused,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv __rte_unused,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+int
+qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_vec *data, uint16_t n_data_vecs,
+	union rte_crypto_sym_ofs ofs,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	void *user_data);
+
+uint32_t
+qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
+	struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
+	void *user_data[], int *status);
+
+void *
+qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
+	int *dequeue_status, enum rte_crypto_op_status *op_status);
+
+uint32_t
+qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
+	rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
+	uint32_t max_nb_to_dequeue,
+	rte_cryptodev_raw_post_dequeue_t post_dequeue,
+	void **out_user_data, uint8_t is_user_data_array,
+	uint32_t *n_success_jobs, int *return_status);
+
+int
+qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
+int
+qat_sym_dp_denqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
+int
+qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);