drivers/qat: QAT symmetric crypto datapatch rework

PATCH Superseded Kai Ji 3 participants 156 messages Oct 26, 2021 – Feb 23, 2022 Page 2 of 3

Subsystems (2)

Arm/Amlogic Meson SoC support Crypto API

lore

Share & tools

lore

Kai Ji

author

2022-02-04 18:51 UTC #63 lore raw

PATCH v6 4/10 crypto/qat: rework session APIs

In reply to Kai Ji

This patch introduces the set_session methods for different generations of QAT. In addition, the patch replaces 'min_qat_dev_gen_id' with 'qat_dev_gen'. Thus, the session no longer allow to be created by one generation of QAT used by another generation.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 +++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 139 +++++++++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c |  91 ++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 +++------------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 425 insertions(+), 107 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +139 -0

155 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +91 -0

106 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:51 UTC #64 lore raw

PATCH v6 5/10 crypto/qat: rework asymmetric crypto build operation

In reply to Kai Ji

This patch reworks the asymmetric crypto data path implementation to QAT driver. The change includes separation of different QAT generations' asymmetric crypto data path implementations and shrink the device capabilities declaration code size.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 624 +++++++++++++++++-----------------
 drivers/crypto/qat/qat_asym.h |  63 +++-
 3 files changed, 380 insertions(+), 312 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index ed632b5ebe..c3265241a3 100644
@@ -622,7 +622,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -850,7 +850,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +320 ~~-304~~

669 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:51 UTC #65 lore raw

PATCH v6 6/10 crypto/qat: unify qat sym pmd apis

In reply to Kai Ji

This patch removes qat_sym_pmd.c and integrates all the apis into qat_sym.c. The unified/integrated qat sym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 437996f2e8..e5459fdfd1 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index c3265241a3..dd9056650d 100644
@@ -841,7 +841,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 12825e448b..2576cb1be7 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:51 UTC #66 lore raw

PATCH v6 7/10 crypto/qat: unify qat asym pmd apis

In reply to Kai Ji

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:52 UTC #67 lore raw

PATCH v6 8/10 crypto/qat: op burst data path rework

In reply to Kai Ji

This patch enable op_build_request function in qat_enqueue_op_burst, and qat_dequeue_process_response function in qat_dequeue_op_burst. The op_build_request invoked in crypto build request op is based on crypto operations setup during session init.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 829 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 270 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index dd9056650d..9bbadc8f8e 100644
@@ -550,8 +550,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -602,29 +601,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -820,8 +808,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -839,21 +826,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index 501132a448..c58a628915 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index da8d7e965c..07e3baa172 100644
@@ -773,7 +773,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index aba49d57cb..72e62120c5 100644
@@ -104,28 +104,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +260 ~~-569~~

895 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:52 UTC #68 lore raw

PATCH v6 9/10 crypto/qat: raw dp api integration

In reply to Kai Ji

This patch "unifies" QAT's raw dp api implementations to share the same enqueue/dequeue methods as the crypto operation enqueue/dequeue methods. In addition, different QAT generation specific implementations are done respectively. The qat_sym_hw_dp.c is removed as no longer required.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 +++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 974 -------------------
 10 files changed, 1137 insertions(+), 982 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index 8e497e7a09..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-974~~

974 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-04 18:52 UTC #69 lore raw

PATCH v6 10/10 crypto/qat: support out of place SG list

In reply to Kai Ji

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-08 18:15 UTC #70 lore raw

v7 cover drivers/qat: QAT symmetric crypto datapatch rework

This patch reworks QAT symmetric crypto datapatch implementation where each generation request building separated and the crypto operation under the raw datapath api implementation are unified.

In addtion this patchset also enables QAT OOP support in raw datapath api implementation.

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

Kai Ji (10):
common/qat: define build op request and dequeue op crypto/qat: sym build op request specific implementation crypto/qat: qat generation specific enqueue crypto/qat: rework session APIs
crypto/qat: rework asymmetric crypto build operation crypto/qat: unify qat sym pmd apis
crypto/qat: unify qat asym pmd apis
crypto/qat: op burst data path rework crypto/qat: raw dp api integration
crypto/qat: support out of place SG list

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 762 ++++++++------
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 983 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3813 insertions(+), 2739 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

--
2.17.1

Kai Ji

author

2022-02-08 18:15 UTC #71 lore raw

PATCH v7 1/10 common/qat: define build op request and dequeue op

This patch introduce build request op and dequeue op function pointers to qat queue pair implementation. Those two functions are used to be assigned during qat session generation based on crypto operation

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  4 +--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 76 insertions(+), 13 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index cde421eb77..ed632b5ebe 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -550,7 +550,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -817,7 +819,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +2 -2

index da6404c017..8e497e7a09 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 1/10

2022-02-09 10:22 UTC #72 lore raw

RE: [dpdk-dev v7 01/10] common/qat: define build op request and dequeue op

Quoted 8 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 01/10] common/qat: define build op request and dequeue op

Quoted 7 lines

This patch introduce build request op and dequeue op function pointers to qat queue pair implementation. Those two functions are used to be assigned during qat session generation based on crypto operation
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #73 lore raw

PATCH v7 2/10 crypto/qat: sym build op request specific implementation

In reply to Kai Ji

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op and the build op request function pointer implementation for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 2/10

2022-02-09 10:22 UTC #74 lore raw

RE: [dpdk-dev v7 02/10] crypto/qat: sym build op request specific implementation

Quoted 8 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 02/10] crypto/qat: sym build op request specific implementation

Quoted 6 lines

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op and the build op request function pointer implementation for QAT generation 1.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #75 lore raw

PATCH v7 3/10 crypto/qat: qat generation specific enqueue

In reply to Kai Ji

This patch add in specific aead & auth build op enqueue functions for QAT Gen3 & Gen4

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 117 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c |  34 +++++-
 2 files changed, 149 insertions(+), 2 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +116 -1

index d3336cf4a1..fca7af2b7e 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -143,6 +143,121 @@ qat_sym_crypto_cap_get_gen3(struct qat_pci_device *qat_dev __rte_unused)
 	return capa_info;
 }
 
+static __rte_always_inline void
+enqueue_one_aead_job_gen3(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	if (ctx->is_single_pass) {
+		struct icp_qat_fw_la_cipher_req_params *cipher_param =
+			(void *)&req->serv_specif_rqpars;
+
+		/* QAT GEN3 uses single pass to treat AEAD as
+		 * cipher operation
+		 */
+		cipher_param = (void *)&req->serv_specif_rqpars;
+
+		qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+		cipher_param->cipher_offset = ofs.ofs.cipher.head;
+		cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+				ofs.ofs.cipher.tail;
+
+		cipher_param->spc_aad_addr = aad->iova;
+		cipher_param->spc_auth_res_addr = digest->iova;
+
+		return;
+	}
+
+	enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen3(struct qat_sym_session *ctx,
+	struct qat_sym_op_cookie *cookie,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl;
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	uint32_t ver_key_offset;
+	uint32_t auth_data_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+
+	if (!ctx->is_single_pass_gmac ||
+			(auth_data_len > QAT_AES_GMAC_SPC_MAX_SIZE)) {
+		enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,
+				data_len);
+		return;
+	}
+
+	cipher_cd_ctrl = (void *) &req->cd_ctrl;
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	ver_key_offset = sizeof(struct icp_qat_hw_auth_setup) +
+			ICP_QAT_HW_GALOIS_128_STATE1_SZ +
+			ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ +
+			ICP_QAT_HW_GALOIS_E_CTR0_SZ +
+			sizeof(struct icp_qat_hw_cipher_config);
+
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+		/* AES-GMAC */
+		qat_set_cipher_iv(cipher_param, auth_iv, ctx->auth_iv.length,
+				req);
+	}
+
+	/* Fill separate Content Descriptor for this op */
+	rte_memcpy(cookie->opt.spc_gmac.cd_cipher.key,
+			ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
+				ctx->cd.cipher.key :
+				RTE_PTR_ADD(&ctx->cd, ver_key_offset),
+			ctx->auth_key_length);
+	cookie->opt.spc_gmac.cd_cipher.cipher_config.val =
+			ICP_QAT_HW_CIPHER_CONFIG_BUILD(
+				ICP_QAT_HW_CIPHER_AEAD_MODE,
+				ctx->qat_cipher_alg,
+				ICP_QAT_HW_CIPHER_NO_CONVERT,
+				(ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
+					ICP_QAT_HW_CIPHER_ENCRYPT :
+					ICP_QAT_HW_CIPHER_DECRYPT));
+	QAT_FIELD_SET(cookie->opt.spc_gmac.cd_cipher.cipher_config.val,
+			ctx->digest_length,
+			QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
+			QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
+	cookie->opt.spc_gmac.cd_cipher.cipher_config.reserved =
+			ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(auth_data_len);
+
+	/* Update the request */
+	req->cd_pars.u.s.content_desc_addr =
+			cookie->opt.spc_gmac.cd_phys_addr;
+	req->cd_pars.u.s.content_desc_params_sz = RTE_ALIGN_CEIL(
+			sizeof(struct icp_qat_hw_cipher_config) +
+			ctx->auth_key_length, 8) >> 3;
+	req->comn_mid.src_length = data_len;
+	req->comn_mid.dst_length = 0;
+
+	cipher_param->spc_aad_addr = 0;
+	cipher_param->spc_auth_res_addr = digest->iova;
+	cipher_param->spc_aad_sz = auth_data_len;
+	cipher_param->reserved = 0;
+	cipher_param->spc_auth_res_sz = ctx->digest_length;
+
+	req->comn_hdr.service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
+	cipher_cd_ctrl->cipher_cfg_offset = 0;
+	ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
+	ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
+	ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+			ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
+	ICP_QAT_FW_LA_PROTO_SET(
+			req->comn_hdr.serv_specif_flags,
+			ICP_QAT_FW_LA_NO_PROTO);
+}
+
 RTE_INIT(qat_sym_crypto_gen3_init)
 {
 	qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +33 -1

index 37a58c026f..8462c0b9b1 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -103,6 +103,38 @@ qat_sym_crypto_cap_get_gen4(struct qat_pci_device *qat_dev __rte_unused)
 	return capa_info;
 }
 
+static __rte_always_inline void
+enqueue_one_aead_job_gen4(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	if (ctx->is_single_pass && ctx->is_ucs) {
+		struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
+			(void *)&req->serv_specif_rqpars;
+		struct icp_qat_fw_la_cipher_req_params *cipher_param =
+			(void *)&req->serv_specif_rqpars;
+
+		/* QAT GEN4 uses single pass to treat AEAD as cipher
+		 * operation
+		 */
+		qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length,
+				req);
+		cipher_param->cipher_offset = ofs.ofs.cipher.head;
+		cipher_param->cipher_length = data_len -
+				ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
+
+		cipher_param_20->spc_aad_addr = aad->iova;
+		cipher_param_20->spc_auth_res_addr = digest->iova;
+
+		return;
+	}
+
+	enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
+}
+
 RTE_INIT(qat_sym_crypto_gen4_init)
 {
 	qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 3/10

2022-02-09 10:23 UTC #76 lore raw

RE: [dpdk-dev v7 03/10] crypto/qat: qat generation specific enqueue

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 03/10] crypto/qat: qat generation specific enqueue

Quoted 5 lines

This patch add in specific aead & auth build op enqueue functions for QAT Gen3 & Gen4
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #77 lore raw

PATCH v7 4/10 crypto/qat: rework session APIs

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 +++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 139 +++++++++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c |  91 ++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 +++------------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 425 insertions(+), 107 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +139 -0

155 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +91 -0

106 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 4/10

2022-02-09 10:23 UTC #78 lore raw

RE: [dpdk-dev v7 04/10] crypto/qat: rework session APIs

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 04/10] crypto/qat: rework session APIs

Quoted 8 lines

This patch introduces the set_session methods for different generations of QAT. In addition, the patch replaces 'min_qat_dev_gen_id' with 'qat_dev_gen'. Thus, the session no longer allow to be created by one generation of QAT used by another generation.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #79 lore raw

PATCH v7 5/10 crypto/qat: rework asymmetric crypto build operation

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 624 +++++++++++++++++-----------------
 drivers/crypto/qat/qat_asym.h |  63 +++-
 3 files changed, 380 insertions(+), 312 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index ed632b5ebe..c3265241a3 100644
@@ -622,7 +622,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -850,7 +850,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +320 ~~-304~~

669 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 5/10

2022-02-09 10:24 UTC #80 lore raw

RE: [dpdk-dev v7 05/10] crypto/qat: rework asymmetric crypto build operation

Quoted 8 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 05/10] crypto/qat: rework asymmetric crypto build operation

Quoted 8 lines

This patch reworks the asymmetric crypto data path implementation to QAT driver. The change includes separation of different QAT generations' asymmetric crypto data path implementations and shrink the device capabilities declaration code size.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #81 lore raw

PATCH v7 6/10 crypto/qat: unify qat sym pmd apis

In reply to Kai Ji

This patch removes qat_sym_pmd.c and integrates all the apis into qat_sym.c. The unified/integrated qat sym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 437996f2e8..e5459fdfd1 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index c3265241a3..dd9056650d 100644
@@ -841,7 +841,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 12825e448b..2576cb1be7 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 6/10

2022-02-09 10:24 UTC #82 lore raw

RE: [dpdk-dev v7 06/10] crypto/qat: unify qat sym pmd apis

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 06/10] crypto/qat: unify qat sym pmd apis

Quoted 6 lines

This patch removes qat_sym_pmd.c and integrates all the apis into qat_sym.c. The unified/integrated qat sym crypto pmd apis should make them easier to maintain.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #83 lore raw

PATCH v7 7/10 crypto/qat: unify qat asym pmd apis

In reply to Kai Ji

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 7/10

2022-02-09 10:24 UTC #84 lore raw

RE: [dpdk-dev v7 07/10] crypto/qat: unify qat asym pmd apis

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 07/10] crypto/qat: unify qat asym pmd apis

Quoted 6 lines

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd apis should make them easier to maintain.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #85 lore raw

PATCH v7 8/10 crypto/qat: op burst data path rework

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 830 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 271 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index dd9056650d..9bbadc8f8e 100644
@@ -550,8 +550,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -602,29 +601,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -820,8 +808,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -839,21 +826,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index 501132a448..c58a628915 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index da8d7e965c..07e3baa172 100644
@@ -773,7 +773,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index aba49d57cb..72e62120c5 100644
@@ -104,28 +104,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +261 ~~-569~~

896 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 8/10

2022-02-09 10:24 UTC #86 lore raw

RE: [dpdk-dev v7 08/10] crypto/qat: op burst data path rework

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 08/10] crypto/qat: op burst data path rework

Quoted 7 lines

This patch enable op_build_request function in qat_enqueue_op_burst, and qat_dequeue_process_response function in qat_dequeue_op_burst. The op_build_request invoked in crypto build request op is based on crypto operations setup during session init.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:15 UTC #87 lore raw

PATCH v7 9/10 crypto/qat: raw dp api integration

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 +++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 974 -------------------
 10 files changed, 1137 insertions(+), 982 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index 8e497e7a09..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-974~~

974 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 9/10

2022-02-09 10:25 UTC #88 lore raw

RE: [dpdk-dev v7 09/10] crypto/qat: raw dp api integration

Quoted 7 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 09/10] crypto/qat: raw dp api integration

Quoted 8 lines

This patch "unifies" QAT's raw dp api implementations to share the same enqueue/dequeue methods as the crypto operation enqueue/dequeue methods. In addition, different QAT generation specific implementations are done respectively. The qat_sym_hw_dp.c is removed as no longer required.
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-08 18:16 UTC #89 lore raw

PATCH v7 10/10 crypto/qat: support out of place SG list

In reply to Kai Ji

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v7 10/10

2022-02-09 10:25 UTC #90 lore raw

RE: [dpdk-dev v7 10/10] crypto/qat: support out of place SG list

Quoted 11 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 10/10] crypto/qat: support out of place SG list
This patch adds the SGL out of place support to QAT PMD
Signed-off-by Kai Ji <[email protected]>
---

Acked-by Fan Zhang <[email protected]>

Zhang, Roy Fan

2022-02-09 10:20 UTC #91 lore raw

In reply to Kai Ji

Quoted 80 lines

-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 00/10] drivers/qat: QAT symmetric crypto datapatch rework
This patch reworks QAT symmetric crypto datapatch implementation where each
generation request building separated and the crypto operation under the raw datapath api implementation are unified.
In addtion this patchset also enables QAT OOP support in raw datapath api implementation.
v7:
- fix of pointer cast compile error in x86
v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main
v5:
- rebase to the latest for-main
- patchset reconstruct
v4:
- patchset break down and reconstruct
v3:
- sperate a single patch 6 to two different patches
v2:
- review comments addressed
Kai Ji (10):
common/qat: define build op request and dequeue op crypto/qat: sym build op request specific implementation crypto/qat: qat generation specific enqueue crypto/qat: rework session APIs
crypto/qat: rework asymmetric crypto build operation crypto/qat: unify qat sym pmd apis
crypto/qat: unify qat asym pmd apis
crypto/qat: op burst data path rework crypto/qat: raw dp api integration
crypto/qat: support out of place SG list
 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913
++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942
+++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 762 ++++++++------
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 983 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3813 insertions(+), 2739 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

Quoted 2 lines

--
2.17.1

Series-acked-by Fan Zhang <[email protected]>

Akhil Goyal

2022-02-12 11:33 UTC #92 lore raw

Quoted 80 lines

Quoted 80 lines
-----Original Message-----
From: Kai Ji <[email protected]>
Sent: Tuesday, February 8, 2022 6:15 PM
To: [email protected]
Cc: [email protected]; Ji, Kai <[email protected]>
Subject: [dpdk-dev v7 00/10] drivers/qat: QAT symmetric crypto datapatch rework
This patch reworks QAT symmetric crypto datapatch implementation where each
generation request building separated and the crypto operation under the raw datapath api implementation are unified.
In addtion this patchset also enables QAT OOP support in raw datapath api implementation.
v7:
- fix of pointer cast compile error in x86
v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main
v5:
- rebase to the latest for-main
- patchset reconstruct
v4:
- patchset break down and reconstruct
v3:
- sperate a single patch 6 to two different patches
v2:
- review comments addressed
Kai Ji (10):
common/qat: define build op request and dequeue op crypto/qat: sym build op request specific implementation crypto/qat: qat generation specific enqueue crypto/qat: rework session APIs
crypto/qat: rework asymmetric crypto build operation crypto/qat: unify qat sym pmd apis
crypto/qat: unify qat asym pmd apis
crypto/qat: op burst data path rework crypto/qat: raw dp api integration
crypto/qat: support out of place SG list
 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913
++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942
+++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 762 ++++++++------
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 983 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3813 insertions(+), 2739 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

Quoted 3 lines

Quoted 2 lines
--
2.17.1
Series-acked-by Fan Zhang <[email protected]>

The patchset does not apply on next-crypto TOT.
Please rebase.

Kai Ji

author

2022-02-17 16:29 UTC #93 lore raw

v8 cover drivers/qat: QAT symmetric crypto datapatch rework

In reply to Kai Ji

This patch reworks QAT symmetric crypto datapatch implementation where each generation request building separated and the crypto operation under the raw datapath api implementation are unified.

In addtion this patchset also enables QAT OOP support in raw datapath api implementation.

v8:
- rebase to 22.03-rc1

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

--
2.17.1

Kai Ji

author

2022-02-17 16:29 UTC #94 lore raw

PATCH v8 1/10 common/qat: define build op request and dequeue op

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  4 +--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 76 insertions(+), 13 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index 57ac8fefca..56234ca1a4 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -547,7 +547,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -814,7 +816,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +2 -2

index da6404c017..8e497e7a09 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:29 UTC #95 lore raw

PATCH v8 2/10 crypto/qat: sym build op request specific implementation

In reply to Kai Ji

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op and the build op request function pointer implementation for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:29 UTC #96 lore raw

PATCH v8 3/10 crypto/qat: qat generation specific enqueue

In reply to Kai Ji

This patch add in specific aead & auth build op enqueue functions for QAT Gen3 & Gen4

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 117 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c |  34 +++++-
 2 files changed, 149 insertions(+), 2 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +116 -1

index d3336cf4a1..fca7af2b7e 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -143,6 +143,121 @@ qat_sym_crypto_cap_get_gen3(struct qat_pci_device *qat_dev __rte_unused)
 	return capa_info;
 }
 
+static __rte_always_inline void
+enqueue_one_aead_job_gen3(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	if (ctx->is_single_pass) {
+		struct icp_qat_fw_la_cipher_req_params *cipher_param =
+			(void *)&req->serv_specif_rqpars;
+
+		/* QAT GEN3 uses single pass to treat AEAD as
+		 * cipher operation
+		 */
+		cipher_param = (void *)&req->serv_specif_rqpars;
+
+		qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+		cipher_param->cipher_offset = ofs.ofs.cipher.head;
+		cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+				ofs.ofs.cipher.tail;
+
+		cipher_param->spc_aad_addr = aad->iova;
+		cipher_param->spc_auth_res_addr = digest->iova;
+
+		return;
+	}
+
+	enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen3(struct qat_sym_session *ctx,
+	struct qat_sym_op_cookie *cookie,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl;
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	uint32_t ver_key_offset;
+	uint32_t auth_data_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+
+	if (!ctx->is_single_pass_gmac ||
+			(auth_data_len > QAT_AES_GMAC_SPC_MAX_SIZE)) {
+		enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,
+				data_len);
+		return;
+	}
+
+	cipher_cd_ctrl = (void *) &req->cd_ctrl;
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	ver_key_offset = sizeof(struct icp_qat_hw_auth_setup) +
+			ICP_QAT_HW_GALOIS_128_STATE1_SZ +
+			ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ +
+			ICP_QAT_HW_GALOIS_E_CTR0_SZ +
+			sizeof(struct icp_qat_hw_cipher_config);
+
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+		/* AES-GMAC */
+		qat_set_cipher_iv(cipher_param, auth_iv, ctx->auth_iv.length,
+				req);
+	}
+
+	/* Fill separate Content Descriptor for this op */
+	rte_memcpy(cookie->opt.spc_gmac.cd_cipher.key,
+			ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
+				ctx->cd.cipher.key :
+				RTE_PTR_ADD(&ctx->cd, ver_key_offset),
+			ctx->auth_key_length);
+	cookie->opt.spc_gmac.cd_cipher.cipher_config.val =
+			ICP_QAT_HW_CIPHER_CONFIG_BUILD(
+				ICP_QAT_HW_CIPHER_AEAD_MODE,
+				ctx->qat_cipher_alg,
+				ICP_QAT_HW_CIPHER_NO_CONVERT,
+				(ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
+					ICP_QAT_HW_CIPHER_ENCRYPT :
+					ICP_QAT_HW_CIPHER_DECRYPT));
+	QAT_FIELD_SET(cookie->opt.spc_gmac.cd_cipher.cipher_config.val,
+			ctx->digest_length,
+			QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
+			QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
+	cookie->opt.spc_gmac.cd_cipher.cipher_config.reserved =
+			ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(auth_data_len);
+
+	/* Update the request */
+	req->cd_pars.u.s.content_desc_addr =
+			cookie->opt.spc_gmac.cd_phys_addr;
+	req->cd_pars.u.s.content_desc_params_sz = RTE_ALIGN_CEIL(
+			sizeof(struct icp_qat_hw_cipher_config) +
+			ctx->auth_key_length, 8) >> 3;
+	req->comn_mid.src_length = data_len;
+	req->comn_mid.dst_length = 0;
+
+	cipher_param->spc_aad_addr = 0;
+	cipher_param->spc_auth_res_addr = digest->iova;
+	cipher_param->spc_aad_sz = auth_data_len;
+	cipher_param->reserved = 0;
+	cipher_param->spc_auth_res_sz = ctx->digest_length;
+
+	req->comn_hdr.service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
+	cipher_cd_ctrl->cipher_cfg_offset = 0;
+	ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
+	ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
+	ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+			ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
+	ICP_QAT_FW_LA_PROTO_SET(
+			req->comn_hdr.serv_specif_flags,
+			ICP_QAT_FW_LA_NO_PROTO);
+}
+
 RTE_INIT(qat_sym_crypto_gen3_init)
 {
 	qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +33 -1

index 37a58c026f..8462c0b9b1 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -103,6 +103,38 @@ qat_sym_crypto_cap_get_gen4(struct qat_pci_device *qat_dev __rte_unused)
 	return capa_info;
 }
 
+static __rte_always_inline void
+enqueue_one_aead_job_gen4(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	if (ctx->is_single_pass && ctx->is_ucs) {
+		struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
+			(void *)&req->serv_specif_rqpars;
+		struct icp_qat_fw_la_cipher_req_params *cipher_param =
+			(void *)&req->serv_specif_rqpars;
+
+		/* QAT GEN4 uses single pass to treat AEAD as cipher
+		 * operation
+		 */
+		qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length,
+				req);
+		cipher_param->cipher_offset = ofs.ofs.cipher.head;
+		cipher_param->cipher_length = data_len -
+				ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
+
+		cipher_param_20->spc_aad_addr = aad->iova;
+		cipher_param_20->spc_auth_res_addr = digest->iova;
+
+		return;
+	}
+
+	enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
+}
+
 RTE_INIT(qat_sym_crypto_gen4_init)
 {
 	qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:29 UTC #97 lore raw

PATCH v8 4/10 crypto/qat: rework session APIs

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 +++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 139 +++++++++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c |  91 ++++++++++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 +++------------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 425 insertions(+), 107 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +139 -0

155 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +91 -0

106 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:29 UTC #98 lore raw

PATCH v8 5/10 crypto/qat: rework asymmetric crypto build operation

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 129 +++++++++++++++++++---------------
 drivers/crypto/qat/qat_asym.h |  63 +++++++++++++++--
 3 files changed, 131 insertions(+), 66 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index 56234ca1a4..7f2fdc53ce 100644
@@ -619,7 +619,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -847,7 +847,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +71 ~~-58~~

174 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:30 UTC #99 lore raw

PATCH v8 6/10 crypto/qat: unify qat sym pmd apis

In reply to Kai Ji

This patch removes qat_sym_pmd.c and integrates all the apis into qat_sym.c. The unified/integrated qat sym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 1f870d689a..6824d97050 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index 7f2fdc53ce..b36ffa6f6d 100644
@@ -838,7 +838,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 792ad2b213..5322faff44 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:30 UTC #100 lore raw

PATCH v8 7/10 crypto/qat: unify qat asym pmd apis

In reply to Kai Ji

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd apis should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:30 UTC #101 lore raw

PATCH v8 8/10 crypto/qat: op burst data path rework

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 830 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 271 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index b36ffa6f6d..08ac91eac4 100644
@@ -547,8 +547,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -599,29 +598,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -817,8 +805,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -836,21 +823,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index 501132a448..c58a628915 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index 76d50dd6f9..3c7cba6140 100644
@@ -507,7 +507,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index 3ae95f2e7b..78caa5649c 100644
@@ -103,28 +103,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +261 ~~-569~~

896 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:30 UTC #102 lore raw

PATCH v8 9/10 crypto/qat: raw dp api integration

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 ++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 986 -------------------
 10 files changed, 1137 insertions(+), 994 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index 8e497e7a09..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-986~~

986 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-17 16:30 UTC #103 lore raw

PATCH v8 10/10 crypto/qat: support out of place SG list

In reply to Kai Ji

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.17.1

Akhil Goyal

2022-02-17 17:59 UTC #104 lore raw

In reply to Kai Ji

Quoted 32 lines

----------------------------------------------------------------------
This patch reworks QAT symmetric crypto datapatch implementation where each
generation request building separated and the crypto operation under the raw datapath api implementation are unified.
In addtion this patchset also enables QAT OOP support in raw datapath api implementation.
v8:
- rebase to 22.03-rc1
v7:
- fix of pointer cast compile error in x86
v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main
v5:
- rebase to the latest for-main
- patchset reconstruct
v4:
- patchset break down and reconstruct
v3:
- sperate a single patch 6 to two different patches
v2:
- review comments addressed

Quoted 11 lines

Kai Ji (10):
common/qat: define build op request and dequeue op crypto/qat: sym build op request specific implementation crypto/qat: qat generation specific enqueue crypto/qat: rework session APIs
crypto/qat: rework asymmetric crypto build operation crypto/qat: unify qat sym pmd apis
crypto/qat: unify qat asym pmd apis
crypto/qat: op burst data path rework crypto/qat: raw dp api integration
crypto/qat: support out of place SG list

Reword the patch titles as follows. crypto/qat: support out of place SG list crypto/qat: unify raw data path functions crypto/qat: rework burst data path crypto/qat: unify asymmetric functions crypto/qat: unify symmetric functions crypto/qat: rework asymmetric op build operation crypto/qat: rework session functions crypto/qat: add generation specific enqueue crypto/qat: support symmetric build op request common/qat: define build request and dequeue ops

Infact I changed some of the descriptions also. Please reword the descriptions also.

Quoted 22 lines

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --

Quoted 9 lines

 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

Individual patches do not compile

ninja: Entering directory `./build-gcc-shared'
[9/9] Linking target drivers/librte_common_qat.so.22.1.
ninja: Entering directory `./build-clang-static'
[1/27] Compiling C object 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen4.c.o'.
FAILED: drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen4.c.o
clang -Idrivers/a715181@@tmp_rte_common_qat@sta -Idrivers -I../drivers -Idrivers/common/qat -I../drivers/common/qat -I../drivers/common/qat/qat_adf -I../drivers/common/qat/../../crypto/qat -I../drivers/common/qat/../../compress/qat -Idrivers/bus/pci -I../drivers/bus/pci -I../drivers/bus/pci/linux -I. -I../ -Iconfig -I../config -Ilib/eal/include -I../lib/eal/include -Ilib/eal/linux/include -I../lib/eal/linux/include -Ilib/eal/x86/include -I../lib/eal/x86/include -Ilib/eal/common -I../lib/eal/common -Ilib/eal -I../lib/eal -Ilib/kvargs -I../lib/kvargs -Ilib/telemetry/../metrics -I../lib/telemetry/../metrics -Ilib/telemetry -I../lib/telemetry -Ilib/pci -I../lib/pci -Ilib/cryptodev -I../lib/cryptodev -Ilib/mbuf -I../lib/mbuf -Ilib/mempool -I../lib/mempool -Ilib/ring -I../lib/ring -Ilib/rcu -I../lib/rcu -Ilib/net -I../lib/net -Ilib/compressdev -I../lib/compressdev -Ilib/security -I../lib/security -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-address-of-packed-member -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -DBUILD_QAT_SYM -DBUILD_QAT_ASYM -DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.qat -MD -MQ 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen4.c.o' -MF 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen4.c.o.d' -o 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen4.c.o' -c ../drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c ../drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c:107:1: error: unused function 'enqueue_one_aead_job_gen4' [-Werror,-Wunused-function]
enqueue_one_aead_job_gen4(struct qat_sym_session *ctx,
^
1 error generated.
[2/27] Compiling C object 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen3.c.o'.
FAILED: drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen3.c.o
clang -Idrivers/a715181@@tmp_rte_common_qat@sta -Idrivers -I../drivers -Idrivers/common/qat -I../drivers/common/qat -I../drivers/common/qat/qat_adf -I../drivers/common/qat/../../crypto/qat -I../drivers/common/qat/../../compress/qat -Idrivers/bus/pci -I../drivers/bus/pci -I../drivers/bus/pci/linux -I. -I../ -Iconfig -I../config -Ilib/eal/include -I../lib/eal/include -Ilib/eal/linux/include -I../lib/eal/linux/include -Ilib/eal/x86/include -I../lib/eal/x86/include -Ilib/eal/common -I../lib/eal/common -Ilib/eal -I../lib/eal -Ilib/kvargs -I../lib/kvargs -Ilib/telemetry/../metrics -I../lib/telemetry/../metrics -Ilib/telemetry -I../lib/telemetry -Ilib/pci -I../lib/pci -Ilib/cryptodev -I../lib/cryptodev -Ilib/mbuf -I../lib/mbuf -Ilib/mempool -I../lib/mempool -Ilib/ring -I../lib/ring -Ilib/rcu -I../lib/rcu -Ilib/net -I../lib/net -Ilib/compressdev -I../lib/compressdev -Ilib/security -I../lib/security -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-address-of-packed-member -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -DBUILD_QAT_SYM -DBUILD_QAT_ASYM -DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.qat -MD -MQ 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen3.c.o' -MF 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen3.c.o.d' -o 'drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_dev_qat_crypto_pmd_gen3.c.o' -c ../drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c ../drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c:147:1: error: unused function 'enqueue_one_aead_job_gen3' [-Werror,-Wunused-function]
enqueue_one_aead_job_gen3(struct qat_sym_session *ctx,
^
../drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c:178:1: error: unused function 'enqueue_one_auth_job_gen3' [-Werror,-Wunused-function]
enqueue_one_auth_job_gen3(struct qat_sym_session *ctx,
^
2 errors generated.
ninja: build stopped: subcommand failed.

Kai Ji

author

2022-02-18 17:15 UTC #105 lore raw

v9 cover drivers/qat: QAT symmetric crypto datapatch rework

In reply to Kai Ji

This patch reworks QAT symmetric crypto datapatch implementation where each generation request building separated and the crypto operation under the raw datapath api implementation are unified.

In addtion this patchset also enables QAT OOP support in raw datapath api implementation.

v9:
- commit messages reword
- fix of unused function error

v8:
- rebase to 22.03-rc1

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

Kai Ji (9):
common/qat: define build request and dequeue ops crypto/qat: support symmetric build op request crypto/qat: rework session functions crypto/qat: rework asymmetric op build operation crypto/qat: unify symmetric functions crypto/qat: unify asymmetric functions crypto/qat: rework burst data path crypto/qat: unify raw data path functions crypto/qat: support out of place SG list

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

--
2.17.1

Kai Ji

author

2022-02-18 17:15 UTC #106 lore raw

PATCH v9 1/9 common/qat: define build request and dequeue ops

This patch introduce build request and dequeue op function pointers to the qat queue pair implementation. The function poniters are assigned during qat session generation based on input crypto operation request.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  4 +--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 76 insertions(+), 13 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index 57ac8fefca..56234ca1a4 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -547,7 +547,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -814,7 +816,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +2 -2

index da6404c017..8e497e7a09 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #107 lore raw

PATCH v9 2/9 crypto/qat: support symmetric build op request

In reply to Kai Ji

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op, and the implementation of build op request function for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:15 UTC #108 lore raw

PATCH v9 3/9 crypto/qat: rework session functions

In reply to Kai Ji

This patch introduces a set of set_session methods to QAT generations. In addition, the reuse of QAT session between generations is prohibit as the support of min_qat_dev_gen_id' is removed.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 ++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 256 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 125 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 ++------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 574 insertions(+), 109 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +255 -1

276 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +124 -1

144 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #109 lore raw

PATCH v9 4/9 crypto/qat: rework asymmetric op build operation

In reply to Kai Ji

This patch reworks the asymmetric crypto data path implementation in QAT driver. The changes include asymmetric crypto data path separation for QAT hardware generations, and code optimisation of the device capabilities declaration.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 129 +++++++++++++++++++---------------
 drivers/crypto/qat/qat_asym.h |  63 +++++++++++++++--
 3 files changed, 131 insertions(+), 66 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index 56234ca1a4..7f2fdc53ce 100644
@@ -619,7 +619,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -847,7 +847,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +71 ~~-58~~

174 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #110 lore raw

PATCH v9 5/9 crypto/qat: unify symmetric functions

In reply to Kai Ji

This patch removes qat_sym_pmd.c and integrates all the functions into qat_sym.c. The unified/integrated qat sym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 1f870d689a..6824d97050 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index 7f2fdc53ce..b36ffa6f6d 100644
@@ -838,7 +838,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 792ad2b213..5322faff44 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #111 lore raw

PATCH v9 6/9 crypto/qat: unify asymmetric functions

In reply to Kai Ji

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #112 lore raw

PATCH v9 7/9 crypto/qat: rework burst data path

In reply to Kai Ji

This patch enable the op_build_request function in qat_enqueue_op_burst, and the qat_dequeue_process_response function in qat_dequeue_op_burst.
The op_build_request invoked in crypto build request op is based on crypto operations setup'd during session init.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 830 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 271 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index b36ffa6f6d..08ac91eac4 100644
@@ -547,8 +547,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -599,29 +598,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -817,8 +805,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -836,21 +823,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index 501132a448..c58a628915 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index 76d50dd6f9..3c7cba6140 100644
@@ -507,7 +507,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index 3ae95f2e7b..78caa5649c 100644
@@ -103,28 +103,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +261 ~~-569~~

896 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #113 lore raw

PATCH v9 8/9 crypto/qat: unify raw data path functions

In reply to Kai Ji

This patch unifies QAT's raw dp api implementations
to the same enqueue/dequeue methods used in crypto operations. The specific functions for different QAT generation are updated respectively. The qat_sym_hw_dp.c is removed as no longer required.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 ++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 986 -------------------
 10 files changed, 1137 insertions(+), 994 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index 8e497e7a09..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-986~~

986 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-18 17:16 UTC #114 lore raw

PATCH v9 9/9 crypto/qat: support out of place SG list

In reply to Kai Ji

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.17.1

Kai Ji

author

2022-02-22 17:03 UTC #115 lore raw

v10 cover drivers/qat: QAT symmetric crypto datapatch rework

In reply to Kai Ji

This patch reworks QAT symmetric crypto datapatch implementation where each generation request building separated and the crypto operation under the raw datapath api implementation are unified.

In addtion this patchset also enables QAT OOP support in raw datapath api implementation.

v10:
- rebase to the lastest for-main
- fix of build rerror when RTE_LOG_DEBUG enabled

v9:
- commit messages reword
- fix of unused function error

v8:
- rebase to 22.03-rc1

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

--
2.17.1

Kai Ji

author

2022-02-22 17:03 UTC #116 lore raw

PATCH v10 1/9 common/qat: define build request and dequeue ops

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  4 +--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 76 insertions(+), 13 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index 57ac8fefca..56234ca1a4 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -547,7 +547,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -814,7 +816,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +2 -2

index da6404c017..8e497e7a09 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 1/9

2022-02-22 18:52 UTC #117 lore raw

RE: [dpdk-dev v10 1/9] common/qat: define build request and dequeue ops

Quoted 9 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 1/9] common/qat: define build request and dequeue ops

Quoted 7 lines

This patch introduce build request and dequeue op function pointers to the qat queue pair implementation. The function poniters are assigned during qat session generation based on input crypto operation request.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:03 UTC #118 lore raw

PATCH v10 2/9 crypto/qat: support symmetric build op request

In reply to Kai Ji

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op, and the implementation of build op request function for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 2/9

2022-02-22 18:53 UTC #119 lore raw

RE: [dpdk-dev v10 2/9] crypto/qat: support symmetric build op request

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 2/9] crypto/qat: support symmetric build op request

Quoted 6 lines

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op, and the implementation of build op request function for QAT generation 1.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:03 UTC #120 lore raw

PATCH v10 3/9 crypto/qat: rework session functions

In reply to Kai Ji

This patch introduces a set of set_session methods to QAT generations. In addition, the reuse of QAT session between generations is prohibit as the support of min_qat_dev_gen_id' is removed.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 ++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 256 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 125 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 ++------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 574 insertions(+), 109 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +255 -1

276 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +124 -1

144 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 3/9

2022-02-22 18:53 UTC #121 lore raw

RE: [dpdk-dev v10 3/9] crypto/qat: rework session functions

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 3/9] crypto/qat: rework session functions

Quoted 7 lines

This patch introduces a set of set_session methods to QAT generations. In addition, the reuse of QAT session between generations is prohibit as the support of min_qat_dev_gen_id' is removed.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:03 UTC #122 lore raw

PATCH v10 4/9 crypto/qat: rework asymmetric op build operation

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 129 +++++++++++++++++++---------------
 drivers/crypto/qat/qat_asym.h |  63 +++++++++++++++--
 3 files changed, 131 insertions(+), 66 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index 56234ca1a4..7f2fdc53ce 100644
@@ -619,7 +619,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -847,7 +847,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +71 ~~-58~~

174 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 4/9

2022-02-22 18:53 UTC #123 lore raw

RE: [dpdk-dev v10 4/9] crypto/qat: rework asymmetric op build operation

Quoted 9 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 4/9] crypto/qat: rework asymmetric op build operation

Quoted 7 lines

This patch reworks the asymmetric crypto data path implementation in QAT driver. The changes include asymmetric crypto data path separation for QAT hardware generations, and code optimisation of the device capabilities declaration.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:03 UTC #124 lore raw

PATCH v10 5/9 crypto/qat: unify symmetric functions

In reply to Kai Ji

This patch removes qat_sym_pmd.c and integrates all the functions into qat_sym.c. The unified/integrated qat sym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 1f870d689a..6824d97050 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index 7f2fdc53ce..b36ffa6f6d 100644
@@ -838,7 +838,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 792ad2b213..5322faff44 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 5/9

2022-02-22 18:54 UTC #125 lore raw

RE: [dpdk-dev v10 5/9] crypto/qat: unify symmetric functions

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 5/9] crypto/qat: unify symmetric functions

Quoted 6 lines

This patch removes qat_sym_pmd.c and integrates all the functions into qat_sym.c. The unified/integrated qat sym crypto pmd functions should make them easier to maintain.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:04 UTC #126 lore raw

PATCH v10 6/9 crypto/qat: unify asymmetric functions

In reply to Kai Ji

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 6/9

2022-02-22 18:52 UTC #127 lore raw

RE: [dpdk-dev v10 6/9] crypto/qat: unify asymmetric functions

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 6/9] crypto/qat: unify asymmetric functions

Quoted 6 lines

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd functions should make them easier to maintain.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:04 UTC #128 lore raw

PATCH v10 7/9 crypto/qat: rework burst data path

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 830 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 271 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index b36ffa6f6d..08ac91eac4 100644
@@ -547,8 +547,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -599,29 +598,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -817,8 +805,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -836,21 +823,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index ff176dddf1..c088fd50d2 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index 76d50dd6f9..3c7cba6140 100644
@@ -507,7 +507,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index 3ae95f2e7b..78caa5649c 100644
@@ -103,28 +103,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +261 ~~-569~~

896 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 7/9

2022-02-22 18:54 UTC #129 lore raw

RE: [dpdk-dev v10 7/9] crypto/qat: rework burst data path

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 7/9] crypto/qat: rework burst data path

Quoted 8 lines

This patch enable the op_build_request function in qat_enqueue_op_burst, and the qat_dequeue_process_response function in qat_dequeue_op_burst.
The op_build_request invoked in crypto build request op is based on crypto operations setup'd during session init.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:04 UTC #130 lore raw

PATCH v10 8/9 crypto/qat: unify raw data path functions

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 ++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 986 -------------------
 10 files changed, 1137 insertions(+), 994 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index 8e497e7a09..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-986~~

986 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 8/9

2022-02-22 18:54 UTC #131 lore raw

RE: [dpdk-dev v10 8/9] crypto/qat: unify raw data path functions

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 8/9] crypto/qat: unify raw data path functions

Quoted 7 lines

This patch unifies QAT's raw dp api implementations
to the same enqueue/dequeue methods used in crypto operations. The specific functions for different QAT generation are updated respectively. The qat_sym_hw_dp.c is removed as no longer required.
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Kai Ji

author

2022-02-22 17:04 UTC #132 lore raw

PATCH v10 9/9 crypto/qat: support out of place SG list

In reply to Kai Ji

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.17.1

Zhang, Roy Fan

Acked-by on v10 9/9

2022-02-22 18:55 UTC #133 lore raw

RE: [dpdk-dev v10 9/9] crypto/qat: support out of place SG list

Quoted 8 lines

-----Original Message-----
From: Ji, Kai <[email protected]>
Sent: Tuesday, February 22, 2022 5:02 PM
To: [email protected]
Cc: [email protected]; Zhang, Roy Fan <[email protected]>; Ji, Kai <[email protected]>
Subject: [dpdk-dev v10 9/9] crypto/qat: support out of place SG list

Quoted 4 lines

This patch adds the SGL out of place support to QAT PMD
Signed-off-by Kai Ji <[email protected]>
---

May need to address the compile issue
Other than that

Acked-by Fan Zhang <[email protected]>

Akhil Goyal

2022-02-22 18:24 UTC #134 lore raw

In reply to Kai Ji

Hi Kai,

This is again not compiling. It has been v10 and most of the version are fixing one or the other compilation error. You should fix your build system before sending out the next version.

/usr/bin/ld: drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_qat_sym.c.o:/home/gakhil/up/dpdk-next-crypto/build-clang-static/../drivers/crypto/qat/qat_sym.c:16: multiple definition of `qat_sym_driver_id'; drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_qat_sym_pmd.c.o:/home/gakhil/up/dpdk-next-crypto/build-clang-static/../drivers/crypto/qat/qat_sym_pmd.c:23: first defined here /usr/bin/ld: drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_qat_sym.c.o:/home/gakhil/up/dpdk-next-crypto/build-clang-static/../drivers/crypto/qat/qat_sym.c:18: multiple definition of `qat_sym_gen_dev_ops'; drivers/a715181@@tmp_rte_common_qat@sta/crypto_qat_qat_sym_pmd.c.o:/home/gakhil/up/dpdk-next-crypto/build-clang-static/../drivers/crypto/qat/qat_sym_pmd.c:25: first defined here
clang: error: linker command failed with exit code 1 (use -v to see invocation)

Quoted 82 lines

This patch reworks QAT symmetric crypto datapatch implementation where each
generation request building separated and the crypto operation under the raw datapath api implementation are unified.
In addtion this patchset also enables QAT OOP support in raw datapath api implementation.
v10:
- rebase to the lastest for-main
- fix of build rerror when RTE_LOG_DEBUG enabled
v9:
- commit messages reword
- fix of unused function error
v8:
- rebase to 22.03-rc1
v7:
- fix of pointer cast compile error in x86
v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main
v5:
- rebase to the latest for-main
- patchset reconstruct
v4:
- patchset break down and reconstruct
v3:
- sperate a single patch 6 to two different patches
v2:
- review comments addressed
Kai Ji (9):
common/qat: define build request and dequeue ops crypto/qat: support symmetric build op request crypto/qat: rework session functions crypto/qat: rework asymmetric op build operation crypto/qat: unify symmetric functions crypto/qat: unify asymmetric functions crypto/qat: rework burst data path crypto/qat: unify raw data path functions crypto/qat: support out of place SG list
 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h
--
2.17.1

Fan Zhang

2022-02-22 20:30 UTC #135 lore raw

v11 cover drivers/qat: QAT symmetric crypto datapatch rework

In reply to Kai Ji

This patch reworks QAT symmetric crypto datapatch implementation where each generation request building separated and the crypto operation under the raw datapath api implementation are unified.

In addtion this patchset also enables QAT OOP support in raw datapath api implementation.

v11:
- fixed a compile issue

v10:
- rebase to the lastest for-main
- fix of build rerror when RTE_LOG_DEBUG enabled

v9:
- commit messages reword
- fix of unused function error

v8:
- rebase to 22.03-rc1

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:30 UTC #136 lore raw

PATCH v11 1/9 common/qat: define build request and dequeue ops

From: Kai Ji <[email protected]>

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  6 ++--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 77 insertions(+), 14 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index 57ac8fefca..56234ca1a4 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -547,7 +547,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -814,7 +816,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +3 -3

index da6404c017..efe5d08a99 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -639,7 +639,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
 					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_pmd_enq_deq_dummy_op_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #137 lore raw

PATCH v11 2/9 crypto/qat: support symmetric build op request

In reply to Fan Zhang

From: Kai Ji <[email protected]>

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op, and the implementation of build op request function for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #138 lore raw

PATCH v11 3/9 crypto/qat: rework session functions

In reply to Fan Zhang

From: Kai Ji <[email protected]>

This patch introduces a set of set_session methods to QAT generations. In addition, the reuse of QAT session between generations is prohibit as the support of min_qat_dev_gen_id' is removed.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 ++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 256 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 125 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 ++------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 10 files changed, 574 insertions(+), 109 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +255 -1

276 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +124 -1

144 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #139 lore raw

PATCH v11 4/9 crypto/qat: rework asymmetric op build operation

In reply to Fan Zhang

From: Kai Ji <[email protected]>

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/qat_qp.c   |   5 +-
 drivers/crypto/qat/qat_asym.c | 129 +++++++++++++++++++---------------
 drivers/crypto/qat/qat_asym.h |  63 +++++++++++++++--
 3 files changed, 131 insertions(+), 66 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index 56234ca1a4..7f2fdc53ce 100644
@@ -619,7 +619,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -847,7 +847,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +71 ~~-58~~

174 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #140 lore raw

PATCH v11 5/9 crypto/qat: unify symmetric functions

In reply to Fan Zhang

From: Kai Ji <[email protected]>

This patch removes qat_sym_pmd.c and integrates all the functions into qat_sym.c. The unified/integrated qat sym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/meson.build       |   4 +-
 drivers/common/qat/qat_device.c      |   4 +-
 drivers/common/qat/qat_qp.c          |   3 +-
 drivers/crypto/qat/qat_crypto.h      |   5 +-
 drivers/crypto/qat/qat_sym.c         |  21 +++
 drivers/crypto/qat/qat_sym.h         | 147 ++++++++++++++--
 drivers/crypto/qat/qat_sym_hw_dp.c   |  11 +-
 drivers/crypto/qat/qat_sym_pmd.c     | 251 ---------------------------
 drivers/crypto/qat/qat_sym_pmd.h     |  95 ----------
 drivers/crypto/qat/qat_sym_session.c |   2 +-
 10 files changed, 168 insertions(+), 375 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

drivers/common/qat/meson.build +2 -2

index af92271a75..1bf6896a7e 100644
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017-2018 Intel Corporation
+# Copyright(c) 2017-2022 Intel Corporation
 
 if is_windows
     build = false

@@ -73,7 +73,7 @@ if qat_compress
 endif
 
 if qat_crypto
-    foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
+    foreach f: ['qat_sym.c', 'qat_sym_session.c',
             'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',

drivers/common/qat/qat_device.c +2 -2

index 1f870d689a..6824d97050 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2018-2020 Intel Corporation
+ * Copyright(c) 2018-2022 Intel Corporation
  */
 
 #include <rte_string_fns.h>

@@ -8,7 +8,7 @@
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 #include "qat_comp_pmd.h"
 #include "adf_pf2vf_msg.h"
 #include "qat_pf2vf.h"

drivers/common/qat/qat_qp.c +2 -1

index 7f2fdc53ce..b36ffa6f6d 100644
@@ -838,7 +838,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 			nb_fw_responses = qat_comp_process_response(
 				ops, resp_msg,

drivers/crypto/qat/qat_crypto.h +4 -1

index 5ca76fcaa6..c01266f81c 100644
@@ -12,7 +12,10 @@
 extern uint8_t qat_sym_driver_id;
 extern uint8_t qat_asym_driver_id;
 
-/** helper macro to set cryptodev capability range **/
+/**
+ * helper macro to set cryptodev capability range
+ * <n: name> <l: min > <r: max> <i: increment> <v: value>
+ **/
 #define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
 
 #define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}

drivers/crypto/qat/qat_sym.c +21 -0

index 83bf55c933..aad4b243b7 100644
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;
 
 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+	struct qat_sym_op_cookie *cookie = op_cookie;
+
+	cookie->qat_sgl_src_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_src);
+
+	cookie->qat_sgl_dst_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			qat_sgl_dst);
+
+	cookie->opt.spc_gmac.cd_phys_addr =
+			rte_mempool_virt2iova(cookie) +
+			offsetof(struct qat_sym_op_cookie,
+			opt.spc_gmac.cd_cipher);
+}
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

drivers/crypto/qat/qat_sym.h +135 ~~-12~~

193 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c +1 ~~-10~~

index 792ad2b213..5322faff44 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2020 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
 #include <cryptodev_pmd.h>

@@ -9,18 +9,9 @@
 #include "icp_qat_fw_la.h"
 
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
 #include "qat_sym_session.h"
 #include "qat_qp.h"
 
-struct qat_sym_dp_ctx {
-	struct qat_sym_session *session;
-	uint32_t tail;
-	uint32_t head;
-	uint16_t cached_enqueue;
-	uint16_t cached_dequeue;
-};
-
 static __rte_always_inline int32_t
 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
 		struct rte_crypto_vec *data, uint16_t n_data_vecs)

drivers/crypto/qat/qat_sym_pmd.c deleted +0 ~~-251~~

251 lines not on this page · raw message

drivers/crypto/qat/qat_sym_pmd.h deleted +0 ~~-95~~

95 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.c +1 -1

index 3a880096c4..9d6a19c0be 100644
@@ -20,7 +20,7 @@
 
 #include "qat_logs.h"
 #include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
 
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #141 lore raw

PATCH v11 6/9 crypto/qat: unify asymmetric functions

In reply to Fan Zhang

From: Kai Ji <[email protected]>

This patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd functions should make them easier to maintain.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/meson.build    |   2 +-
 drivers/crypto/qat/qat_asym.c     | 180 +++++++++++++++++++++++
 drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------
 drivers/crypto/qat/qat_asym_pmd.h |  54 -------
 4 files changed, 181 insertions(+), 286 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h

drivers/common/qat/meson.build +1 -1

index 1bf6896a7e..f687f5c9d8 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/crypto/qat/qat_asym.c +180 -0

192 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c deleted +0 ~~-231~~

231 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.h deleted +0 ~~-54~~

54 lines not on this page · raw message

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #142 lore raw

PATCH v11 7/9 crypto/qat: rework burst data path

In reply to Fan Zhang

From: Kai Ji <[email protected]>

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/qat_qp.c               |  42 +-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c |   4 -
 drivers/crypto/qat/qat_asym.c             |   2 +-
 drivers/crypto/qat/qat_asym.h             |  22 -
 drivers/crypto/qat/qat_sym.c              | 830 +++++++---------------
 drivers/crypto/qat/qat_sym.h              |   5 -
 6 files changed, 271 insertions(+), 634 deletions(-)

drivers/common/qat/qat_qp.c +9 ~~-33~~

index b36ffa6f6d..08ac91eac4 100644
@@ -547,8 +547,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp,
-		__rte_unused qat_op_build_request_t op_build_request,
+qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request,
 		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;

@@ -599,29 +598,18 @@ qat_enqueue_op_burst(void *qp,
 		}
 	}
 
-#ifdef BUILD_QAT_SYM
+#ifdef RTE_LIB_SECURITY
 	if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 		qat_sym_preprocess_requests(ops, nb_ops_possible);
 #endif
 
+	memset(tmp_qp->opaque, 0xff, sizeof(tmp_qp->opaque));
+
 	while (nb_ops_sent != nb_ops_possible) {
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
-#ifdef BUILD_QAT_SYM
-			ret = qat_sym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-#endif
-		} else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
-			ret = qat_comp_build_request(*ops, base_addr + tail,
+		ret = op_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
-		} else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
-#ifdef BUILD_QAT_ASYM
-			ret = qat_asym_build_request(*ops, base_addr + tail,
-				tmp_qp->op_cookies[tail >> queue->trailz],
-				NULL, tmp_qp->qat_dev_gen);
-#endif
-		}
+				tmp_qp->opaque, tmp_qp->qat_dev_gen);
+
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
 			/* This message cannot be enqueued */

@@ -817,8 +805,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops,
-		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
-		uint16_t nb_ops)
+		qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -836,21 +823,10 @@ qat_dequeue_op_burst(void *qp, void **ops,
 
 		nb_fw_responses = 1;
 
-		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
-			qat_sym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
-			nb_fw_responses = qat_comp_process_response(
+		nb_fw_responses = qat_dequeue_process_response(
 				ops, resp_msg,
 				tmp_qp->op_cookies[head >> rx_queue->trailz],
 				&tmp_qp->stats.dequeue_err_count);
-#ifdef BUILD_QAT_ASYM
-		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
-			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz],
-				NULL);
-#endif
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4

index ff176dddf1..c088fd50d2 100644
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
 	.sym_session_get_size	= qat_sym_session_get_private_size,
 	.sym_session_configure	= qat_sym_session_configure,
 	.sym_session_clear	= qat_sym_session_clear,
-
-	/* Raw data-path API related operations */
-	.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
-	.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
 };
 
 static struct qat_capabilities_info

drivers/crypto/qat/qat_asym.c +1 -1

index 353c36534a..6e65950baf 100644
@@ -507,7 +507,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,
 	return 0;
 }
 
-int
+static __rte_always_inline int
 qat_asym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
 		__rte_unused uint64_t *opaque,
 		__rte_unused enum qat_device_gen dev_gen)

drivers/crypto/qat/qat_asym.h +0 ~~-22~~

index 3ae95f2e7b..78caa5649c 100644
@@ -103,28 +103,6 @@ void
 qat_asym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_asym_session *sess);
 
-/*
- * Build PKE request to be sent to the fw, partially uses template
- * request generated during session creation.
- *
- * @param	in_op		Pointer to the crypto operation, for every
- *				service it points to service specific struct.
- * @param	out_msg		Message to be returned to enqueue function
- * @param	op_cookie	Cookie pointer that holds private metadata
- * @param	qat_dev_gen	Generation of QAT hardware
- *
- * @return
- *	This function always returns zero,
- *	it is because of backward compatibility.
- *	- 0: Always returned
- *
- */
-int
-qat_asym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie,
-		__rte_unused uint64_t *opaque,
-		enum qat_device_gen qat_dev_gen);
-
 /*
  * Process PKE response received from outgoing queue of QAT
  *

drivers/crypto/qat/qat_sym.c +261 ~~-569~~

896 lines not on this page · raw message

drivers/crypto/qat/qat_sym.h +0 -5

index f4ff2ce4cd..074612c11b 100644
@@ -131,11 +131,6 @@ uint16_t
 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops);
 
-int
-qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen);
-
-
 /** Encrypt a single partial block
  *  Depends on openssl libcrypto
  *  Uses ECB+XOR to do CFB encryption, same result, more performant

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:31 UTC #143 lore raw

PATCH v11 8/9 crypto/qat: unify raw data path functions

In reply to Fan Zhang

From: Kai Ji <[email protected]>

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/meson.build               |   2 +-
 drivers/compress/qat/qat_comp_pmd.c          |  12 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |   2 +
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |  78 ++
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 656 ++++++++++++
 drivers/crypto/qat/qat_crypto.h              |   3 +
 drivers/crypto/qat/qat_sym.c                 |  56 +-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 986 -------------------
 10 files changed, 1137 insertions(+), 994 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c

drivers/common/qat/meson.build +1 -1

index f687f5c9d8..b7027f3164 100644
@@ -74,7 +74,7 @@ endif
 
 if qat_crypto
     foreach f: ['qat_sym.c', 'qat_sym_session.c',
-            'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
+            'qat_asym.c', 'qat_crypto.c',
             'dev/qat_sym_pmd_gen1.c',
             'dev/qat_asym_pmd_gen1.c',
             'dev/qat_crypto_pmd_gen2.c',

drivers/compress/qat/qat_comp_pmd.c +9 -3

index efe5d08a99..dc8db84a68 100644
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+static uint16_t
+qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops,  uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
+				nb_ops);
+}
+
 static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
+	uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
-					(compressdev_dequeue_pkt_burst_t)
-					qat_comp_pmd_enq_deq_dummy_op_burst;
+					qat_comp_dequeue_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0

index 64e6ae66ec..0c64c1e43f 100644
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
 			qat_sym_crypto_cap_get_gen2;
 	qat_sym_gen_dev_ops[QAT_GEN2].set_session =
 			qat_sym_crypto_set_session_gen2;
+	qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
+			qat_sym_configure_raw_dp_ctx_gen1;
 	qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
 			qat_sym_crypto_feature_flags_get_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0

226 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0

134 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0

84 lines not on this page · raw message

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0

674 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +3 -0

index c01266f81c..cf386d0ed0 100644
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
 
 typedef int (*set_session_t)(void *cryptodev, void *session);
 
+typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
 	set_session_t set_session;
+	set_raw_dp_ctx_t set_raw_dp_ctx;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif

drivers/crypto/qat/qat_sym.c +52 -4

80 lines not on this page · raw message

drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 ~~-986~~

986 lines not on this page · raw message

Signature · 1 line

2.25.1

Fan Zhang

2022-02-22 20:32 UTC #144 lore raw

PATCH v11 9/9 crypto/qat: support out of place SG list

In reply to Fan Zhang

From: Kai Ji <[email protected]>

This patch adds the SGL out of place support to QAT PMD

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++--
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 55 +++++++++++++++++---
 3 files changed, 83 insertions(+), 14 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4

index ffa093a7a3..5084a5fcd1 100644
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;
 		enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2

index f803bc1459..bd7f3785df 100644
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
 			(uint8_t *)tx_queue->base_addr + tail);
 		rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 
-		data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
-			vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
+		if (vec->dest_sgl) {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec, vec->src_sgl[i].num,
+				vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
+		} else {
+			data_len = qat_sym_build_req_set_data(req,
+				user_data[i], cookie,
+				vec->src_sgl[i].vec,
+				vec->src_sgl[i].num, NULL, 0);
+		}
+
 		if (unlikely(data_len < 0))
 			break;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8

80 lines not on this page · raw message

Signature · 1 line

2.25.1

Kai Ji

author

2022-02-23 00:50 UTC #145 lore raw

v12 cover drivers/qat: QAT symmetric crypto datapatch rework

In reply to Fan Zhang

v11 - v12:
- fixed a compile issue

v10:
- rebase to the lastest for-main
- fix of build rerror when RTE_LOG_DEBUG enabled

v9:
- commit messages reword
- fix of unused function error

v8:
- rebase to 22.03-rc1

v7:
- fix of pointer cast compile error in x86

v6:
- fix of pointer cast error in x86
- rebase to the lastest for-main

v5:
- rebase to the latest for-main
- patchset reconstruct

v4:
- patchset break down and reconstruct

v3:
- sperate a single patch 6 to two different patches

v2:
- review comments addressed

 drivers/common/qat/meson.build               |   6 +-
 drivers/common/qat/qat_device.c              |   4 +-
 drivers/common/qat/qat_qp.c                  |  42 +-
 drivers/common/qat/qat_qp.h                  |  54 +-
 drivers/compress/qat/qat_comp_pmd.c          |  14 +-
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  93 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 490 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 257 ++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 913 ++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 942 +++++++++++++++++-
 drivers/crypto/qat/qat_asym.c                | 303 +++++-
 drivers/crypto/qat/qat_asym.h                |  79 +-
 drivers/crypto/qat/qat_asym_pmd.c            | 231 -----
 drivers/crypto/qat/qat_asym_pmd.h            |  54 -
 drivers/crypto/qat/qat_crypto.h              |  16 +-
 drivers/crypto/qat/qat_sym.c                 | 979 ++++++------------
 drivers/crypto/qat/qat_sym.h                 | 148 ++-
 drivers/crypto/qat/qat_sym_hw_dp.c           | 995 -------------------
 drivers/crypto/qat/qat_sym_pmd.c             | 251 -----
 drivers/crypto/qat/qat_sym_pmd.h             |  95 --
 drivers/crypto/qat/qat_sym_session.c         | 115 +--
 drivers/crypto/qat/qat_sym_session.h         |  15 +-
 23 files changed, 3582 insertions(+), 2523 deletions(-)
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
 delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c
 delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h

Signature · 1 line

2.30.2

Kai Ji

author

2022-02-23 00:50 UTC #146 lore raw

PATCH v12 1/9 common/qat: define build request and dequeue ops

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/qat_qp.c          | 10 ++++--
 drivers/common/qat/qat_qp.h          | 54 ++++++++++++++++++++++++++--
 drivers/compress/qat/qat_comp_pmd.c  |  6 ++--
 drivers/crypto/qat/qat_asym_pmd.c    |  4 +--
 drivers/crypto/qat/qat_sym_pmd.c     |  4 +--
 drivers/crypto/qat/qat_sym_session.h | 13 ++++++-
 6 files changed, 77 insertions(+), 14 deletions(-)

drivers/common/qat/qat_qp.c +7 -3

index 57ac8fefca..56234ca1a4 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_common.h>

@@ -547,7 +547,9 @@ adf_modulo(uint32_t data, uint32_t modulo_mask)
 }
 
 uint16_t
-qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_enqueue_op_burst(void *qp,
+		__rte_unused qat_op_build_request_t op_build_request,
+		void **ops, uint16_t nb_ops)
 {
 	register struct qat_queue *queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

@@ -814,7 +816,9 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 }
 
 uint16_t
-qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+qat_dequeue_op_burst(void *qp, void **ops,
+		__rte_unused qat_op_dequeue_t qat_dequeue_process_response,
+		uint16_t nb_ops)
 {
 	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;

drivers/common/qat/qat_qp.h +51 -3

81 lines not on this page · raw message

drivers/compress/qat/qat_comp_pmd.c +3 -3

index da6404c017..efe5d08a99 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 
 #include <rte_malloc.h>

@@ -620,7 +620,7 @@ static uint16_t
 qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 				   uint16_t nb_ops)
 {
-	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 
 	if (ret) {

@@ -639,7 +639,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
 		} else {
 			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
 					(compressdev_dequeue_pkt_burst_t)
-					qat_dequeue_op_burst;
+					qat_comp_pmd_enq_deq_dummy_op_burst;
 		}
 	}
 	return ret;

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index addee384e3..9a7596b227 100644
@@ -62,13 +62,13 @@ static struct rte_cryptodev_ops crypto_qat_ops = {
 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 				       uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index b835245f17..28a26260fb 100644
@@ -49,14 +49,14 @@ static uint16_t
 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_enqueue_op_burst(qp, NULL, (void **)ops, nb_ops);
 }
 
 static uint16_t
 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	return qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
 }
 
 /* An rte_driver is needed in the registration of both the device and the driver

drivers/crypto/qat/qat_sym_session.h +12 -1

index 6ebc176729..fe875a7fd0 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
  */
 #ifndef _QAT_SYM_SESSION_H_
 #define _QAT_SYM_SESSION_H_

@@ -63,6 +63,16 @@ enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
 };
 
+struct qat_sym_session;
+
+/*
+ * typedef qat_op_build_request_t function pointer, passed in as argument
+ * in enqueue op burst, where a build request assigned base on the type of
+ * crypto op.
+ */
+typedef int (*qat_sym_build_request_t)(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* Common content descriptor */
 struct qat_sym_cd {
 	struct icp_qat_hw_cipher_algo_blk cipher;

@@ -107,6 +117,7 @@ struct qat_sym_session {
 	/* Some generations need different setup of counter */
 	uint32_t slice_types;
 	enum qat_sym_proto_flag qat_proto_flag;
+	qat_sym_build_request_t build_request[2];
 };
 
 int

Signature · 1 line

2.30.2

Kai Ji

author

2022-02-23 00:50 UTC #147 lore raw

PATCH v12 2/9 crypto/qat: support symmetric build op request

In reply to Kai Ji

This patch adds common inline functions for QAT symmetric crypto driver to process crypto op, and the implementation of build op request function for QAT generation 1.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 832 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    | 187 ++++-
 drivers/crypto/qat/qat_sym.c                 |  90 +-
 3 files changed, 1019 insertions(+), 90 deletions(-)

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +831 -1

index 67a4d2cb2c..1130e0e76f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #ifndef _QAT_CRYPTO_PMD_GENS_H_

@@ -8,14 +8,844 @@
 #include <rte_cryptodev.h>
 #include "qat_crypto.h"
 #include "qat_sym_session.h"
+#include "qat_sym.h"
+
+#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
+	RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
+
+#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
+	(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
+	ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
+
+static __rte_always_inline int
+op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+static __rte_always_inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely((sym_op->m_dst != NULL)
+				&& (sym_op->m_dst != sym_op->m_src)))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
+			dst, last_block_len);
+#endif
+		op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static __rte_always_inline int
+qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+		ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+		if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+				(op->sym->auth.data.length % BYTE_LENGTH != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int
+qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
+		struct rte_crypto_op *op)
+{
+	if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+		ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+		if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			((op->sym->cipher.data.offset %
+			BYTE_LENGTH) != 0)))
+			return -EINVAL;
+		return 1;
+	}
+	return 0;
+}
+
+static __rte_always_inline int32_t
+qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
+		void *opaque, struct qat_sym_op_cookie *cookie,
+		struct rte_crypto_vec *src_vec, uint16_t n_src,
+		struct rte_crypto_vec *dst_vec, uint16_t n_dst)
+{
+	struct qat_sgl *list;
+	uint32_t i;
+	uint32_t tl_src = 0, total_len_src, total_len_dst;
+	uint64_t src_data_start = 0, dst_data_start = 0;
+	int is_sgl = n_src > 1 || n_dst > 1;
+
+	if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
+			n_dst > QAT_SYM_SGL_MAX_NUMBER))
+		return -1;
+
+	if (likely(!is_sgl)) {
+		src_data_start = src_vec[0].iova;
+		tl_src = total_len_src =
+				src_vec[0].len;
+		if (unlikely(n_dst)) { /* oop */
+			total_len_dst = dst_vec[0].len;
+
+			dst_data_start = dst_vec[0].iova;
+			if (unlikely(total_len_src != total_len_dst))
+				return -EINVAL;
+		} else {
+			dst_data_start = src_data_start;
+			total_len_dst = tl_src;
+		}
+	} else { /* sgl */
+		total_len_dst = total_len_src = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
+			QAT_COMN_PTR_TYPE_SGL);
+
+		list = (struct qat_sgl *)&cookie->qat_sgl_src;
+		for (i = 0; i < n_src; i++) {
+			list->buffers[i].len = src_vec[i].len;
+			list->buffers[i].resrvd = 0;
+			list->buffers[i].addr = src_vec[i].iova;
+			if (tl_src + src_vec[i].len > UINT32_MAX) {
+				QAT_DP_LOG(ERR, "Message too long");
+				return -1;
+			}
+			tl_src += src_vec[i].len;
+		}
+
+		list->num_bufs = i;
+		src_data_start = cookie->qat_sgl_src_phys_addr;
+
+		if (unlikely(n_dst > 0)) { /* oop sgl */
+			uint32_t tl_dst = 0;
+
+			list = (struct qat_sgl *)&cookie->qat_sgl_dst;
+
+			for (i = 0; i < n_dst; i++) {
+				list->buffers[i].len = dst_vec[i].len;
+				list->buffers[i].resrvd = 0;
+				list->buffers[i].addr = dst_vec[i].iova;
+				if (tl_dst + dst_vec[i].len > UINT32_MAX) {
+					QAT_DP_LOG(ERR, "Message too long");
+					return -ENOTSUP;
+				}
+
+				tl_dst += dst_vec[i].len;
+			}
+
+			if (tl_src != tl_dst)
+				return -EINVAL;
+			list->num_bufs = i;
+			dst_data_start = cookie->qat_sgl_dst_phys_addr;
+		} else
+			dst_data_start = src_data_start;
+	}
+
+	req->comn_mid.src_data_addr = src_data_start;
+	req->comn_mid.dest_data_addr = dst_data_start;
+	req->comn_mid.src_length = total_len_src;
+	req->comn_mid.dst_length = total_len_dst;
+	req->comn_mid.opaque_data = (uintptr_t)opaque;
+
+	return tl_src;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
+		struct rte_crypto_va_iova_ptr *digest __rte_unused)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int n_src = 0;
+	int ret;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->cipher.data.length >> 3;
+		cipher_ofs = op->sym->cipher.data.offset >> 3;
+		break;
+	case 0:
+		if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device.
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
+			cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
+		struct rte_crypto_va_iova_ptr *auth_iv,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t auth_ofs = 0, auth_len = 0;
+	int n_src, ret;
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+				ctx->auth_iv.offset);
+		auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+				ctx->auth_iv.offset);
+		break;
+	case 0:
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
+					ctx->auth_iv.offset);
+			auth_iv->iova = rte_crypto_op_ctophys_offset(op,
+					ctx->auth_iv.offset);
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+			auth_iv->va = NULL;
+			auth_iv->iova = 0;
+		}
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return UINT64_MAX;
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
+			auth_ofs + auth_len, in_sgl->vec,
+			QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
+				auth_ofs + auth_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+
+		if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	return 0;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	union rte_crypto_sym_ofs ofs;
+	uint32_t min_ofs = 0, max_len = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	int is_oop = (op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src);
+	int is_sgl = op->sym->m_src->nb_segs > 1;
+	int n_src;
+	int ret;
+
+	if (unlikely(is_oop))
+		is_sgl |= op->sym->m_dst->nb_segs > 1;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->auth_iv.offset);
+	auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->auth_iv.offset);
+	digest->va = (void *)op->sym->auth.digest.data;
+	digest->iova = op->sym->auth.digest.phys_addr;
+
+	ret = qat_cipher_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		cipher_len = op->sym->aead.data.length >> 3;
+		cipher_ofs = op->sym->aead.data.offset >> 3;
+		break;
+	case 0:
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	ret = qat_auth_is_len_in_bits(ctx, op);
+	switch (ret) {
+	case 1:
+		auth_len = op->sym->auth.data.length >> 3;
+		auth_ofs = op->sym->auth.data.offset >> 3;
+		break;
+	case 0:
+		auth_len = op->sym->auth.data.length;
+		auth_ofs = op->sym->auth.data.offset;
+		break;
+	default:
+		QAT_DP_LOG(ERR,
+	"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		return -EINVAL;
+	}
+
+	min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
+
+	/* digest in buffer check. Needed only for wireless algos */
+	if (ret == 1) {
+		/* Handle digest-encrypted cases, i.e.
+		 * auth-gen-then-cipher-encrypt and
+		 * cipher-decrypt-then-auth-verify
+		 */
+		uint64_t auth_end_iova;
+
+		if (unlikely(is_sgl)) {
+			uint32_t remaining_off = auth_ofs + auth_len;
+			struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
+				op->sym->m_src);
+
+			while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+					&& sgl_buf->next != NULL) {
+				remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+				sgl_buf = sgl_buf->next;
+			}
+
+			auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
+				sgl_buf, remaining_off);
+		} else
+			auth_end_iova = (is_oop ?
+				rte_pktmbuf_iova(op->sym->m_dst) :
+				rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
+					auth_len;
+
+		/* Then check if digest-encrypted conditions are met */
+		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova))
+			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
+					ctx->digest_length);
+	}
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+	in_sgl->num = n_src;
+
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
+				max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return -1;
+		}
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	ofs.ofs.cipher.head = cipher_ofs;
+	ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
+	ofs.ofs.auth.head = auth_ofs;
+	ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
+
+	return ofs.raw;
+}
+
+static __rte_always_inline uint64_t
+qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
+		struct qat_sym_session *ctx,
+		struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
+		struct rte_crypto_va_iova_ptr *cipher_iv,
+		struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
+		struct rte_crypto_va_iova_ptr *digest)
+{
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	int32_t n_src = 0;
+
+	cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
+			ctx->cipher_iv.offset);
+	cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
+			ctx->cipher_iv.offset);
+	auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
+	auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
+	digest->va = (void *)op->sym->aead.digest.data;
+	digest->iova = op->sym->aead.digest.phys_addr;
+
+	cipher_len = op->sym->aead.data.length;
+	cipher_ofs = op->sym->aead.data.offset;
+
+	n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
+			in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
+	if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return UINT64_MAX;
+	}
+	in_sgl->num = n_src;
+
+	/* Out-Of-Place operation */
+	if (unlikely((op->sym->m_dst != NULL) &&
+			(op->sym->m_dst != op->sym->m_src))) {
+		int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
+				cipher_len, out_sgl->vec,
+				QAT_SYM_SGL_MAX_NUMBER);
+		if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return UINT64_MAX;
+		}
+
+		out_sgl->num = n_dst;
+	} else
+		out_sgl->num = 0;
+
+	return 0;
+}
+
+static __rte_always_inline void
+qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
+				iv_len);
+	else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
+	}
+}
+
+static __rte_always_inline void
+qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++)
+		sta[i] = status;
+}
+
+static __rte_always_inline void
+enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+
+	/* cipher IV */
+	qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+}
+
+static __rte_always_inline void
+enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = data_len - ofs.ofs.auth.head -
+			ofs.ofs.auth.tail;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
+				ctx->auth_iv.length);
+		break;
+	default:
+		break;
+	}
+}
+
+static __rte_always_inline int
+enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_vec *src_vec,
+	uint16_t n_src_vecs,
+	struct rte_crypto_vec *dst_vec,
+	uint16_t n_dst_vecs,
+	struct rte_crypto_va_iova_ptr *cipher_iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *auth_iv,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
+			dst_vec : src_vec;
+	rte_iova_t auth_iova_end;
+	int cipher_len, auth_len;
+	int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
+
+	cipher_param = (void *)&req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	cipher_len = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
+
+	if (unlikely(cipher_len < 0 || auth_len < 0))
+		return -1;
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = cipher_len;
+	qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
+
+	auth_param->auth_off = ofs.ofs.auth.head;
+	auth_param->auth_len = auth_len;
+	auth_param->auth_res_addr = digest->iova;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
+	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
+	case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
+		auth_param->u1.aad_adr = auth_iv->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		break;
+	default:
+		break;
+	}
+
+	if (unlikely(is_sgl)) {
+		/* sgl */
+		int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
+		uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
+
+		while (remaining_off >= cvec->len && i >= 1) {
+			i--;
+			remaining_off -= cvec->len;
+			cvec++;
+		}
+
+		auth_iova_end = cvec->iova + remaining_off;
+	} else
+		auth_iova_end = cvec[0].iova + auth_param->auth_off +
+			auth_param->auth_len;
+
+	/* Then check if digest-encrypted conditions are met */
+	if ((auth_param->auth_off + auth_param->auth_len <
+		cipher_param->cipher_offset + cipher_param->cipher_length) &&
+			(digest->iova == auth_iova_end)) {
+		/* Handle partial digest encryption */
+		if (cipher_param->cipher_offset + cipher_param->cipher_length <
+			auth_param->auth_off + auth_param->auth_len +
+				ctx->digest_length && !is_sgl)
+			req->comn_mid.dst_length = req->comn_mid.src_length =
+				auth_param->auth_off + auth_param->auth_len +
+					ctx->digest_length;
+		struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+		ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+	}
+
+	return 0;
+}
+
+static __rte_always_inline void
+enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
+	struct icp_qat_fw_la_bulk_req *req,
+	struct rte_crypto_va_iova_ptr *iv,
+	struct rte_crypto_va_iova_ptr *digest,
+	struct rte_crypto_va_iova_ptr *aad,
+	union rte_crypto_sym_ofs ofs, uint32_t data_len)
+{
+	struct icp_qat_fw_la_cipher_req_params *cipher_param =
+		(void *)&req->serv_specif_rqpars;
+	struct icp_qat_fw_la_auth_req_params *auth_param =
+		(void *)((uint8_t *)&req->serv_specif_rqpars +
+		ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+	uint8_t *aad_data;
+	uint8_t aad_ccm_real_len;
+	uint8_t aad_len_field_sz;
+	uint32_t msg_len_be;
+	rte_iova_t aad_iova = 0;
+	uint8_t q;
+
+	switch (ctx->qat_hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
+	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
+		ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+			req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+		rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
+				ctx->cipher_iv.length);
+		aad_iova = aad->iova;
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
+		aad_data = aad->va;
+		aad_iova = aad->iova;
+		aad_ccm_real_len = 0;
+		aad_len_field_sz = 0;
+		msg_len_be = rte_bswap32((uint32_t)data_len -
+				ofs.ofs.cipher.head);
+
+		if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+			aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			aad_ccm_real_len = ctx->aad_len -
+				ICP_QAT_HW_CCM_AAD_B0_LEN -
+				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+		} else {
+			aad_data = iv->va;
+			aad_iova = iv->iova;
+		}
+
+		q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
+		aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+			aad_len_field_sz, ctx->digest_length, q);
+		if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				(uint8_t *)&msg_len_be,
+				ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+		} else {
+			memcpy(aad_data	+ ctx->cipher_iv.length +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+				(uint8_t *)&msg_len_be +
+				(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				- q), q);
+		}
+
+		if (aad_len_field_sz > 0) {
+			*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
+				rte_bswap16(aad_ccm_real_len);
+
+			if ((aad_ccm_real_len + aad_len_field_sz)
+				% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+				uint8_t pad_len = 0;
+				uint8_t pad_idx = 0;
+
+				pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len +
+					aad_len_field_sz) %
+					ICP_QAT_HW_CCM_AAD_B0_LEN);
+				pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					aad_ccm_real_len +
+					aad_len_field_sz;
+				memset(&aad_data[pad_idx], 0, pad_len);
+			}
+		}
+
+		rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+			+ ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va +
+			ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
+		*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+		rte_memcpy((uint8_t *)aad->va +
+				ICP_QAT_HW_CCM_NONCE_OFFSET,
+			(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			ctx->cipher_iv.length);
+		break;
+	default:
+		break;
+	}
+
+	cipher_param->cipher_offset = ofs.ofs.cipher.head;
+	cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
+			ofs.ofs.cipher.tail;
+	auth_param->auth_off = ofs.ofs.cipher.head;
+	auth_param->auth_len = cipher_param->cipher_length;
+	auth_param->auth_res_addr = digest->iova;
+	auth_param->u1.aad_adr = aad_iova;
+}
 
 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 
+/* -----------------GEN 1 sym crypto op data path APIs ---------------- */
+int
+qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
+	uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
+int
+qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
+		uint8_t *out_msg, void *op_cookie);
+
 /* -----------------GENx control path APIs ---------------- */
 uint64_t
 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 void
 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 		uint8_t hash_flag);

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +186 -1

197 lines not on this page · raw message

drivers/crypto/qat/qat_sym.c +2 ~~-88~~

100 lines not on this page · raw message

Signature · 1 line

2.30.2

Kai Ji

author

2022-02-23 00:50 UTC #148 lore raw

PATCH v12 3/9 crypto/qat: rework session functions

In reply to Kai Ji

This patch introduces a set of set_session methods to QAT generations. In addition, the reuse of QAT session between generations is prohibit as the support of min_qat_dev_gen_id' is removed.

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/crypto/qat/dev/qat_asym_pmd_gen1.c   |   9 +-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c |  91 ++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 256 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 125 ++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   3 +
 drivers/crypto/qat/dev/qat_sym_pmd_gen1.c    |  64 +++++
 drivers/crypto/qat/qat_crypto.h              |   8 +-
 drivers/crypto/qat/qat_sym.c                 |  12 +-
 drivers/crypto/qat/qat_sym_pmd.c             |   4 +-
 drivers/crypto/qat/qat_sym_session.c         | 113 ++------
 drivers/crypto/qat/qat_sym_session.h         |   2 +-
 11 files changed, 576 insertions(+), 111 deletions(-)

drivers/crypto/qat/dev/qat_asym_pmd_gen1.c +8 -1

index 9ed1f21d9d..01a897a21f 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2021 Intel Corporation
+ * Copyright(c) 2017-2022 Intel Corporation
  */
 
 #include <rte_cryptodev.h>

@@ -65,6 +65,13 @@ qat_asym_crypto_feature_flags_get_gen1(
 	return feature_flags;
 }
 
+int
+qat_asym_crypto_set_session_gen1(void *cdev __rte_unused,
+		void *session __rte_unused)
+{
+	return 0;
+}
+
 RTE_INIT(qat_asym_crypto_gen1_init)
 {
 	qat_asym_gen_dev_ops[QAT_GEN1].cryptodev_ops =

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +90 -1

111 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +255 -1

276 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +124 -1

144 lines not on this page · raw message

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +3 -0

index 1130e0e76f..96cdb97a26 100644
@@ -856,6 +856,9 @@ qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 uint64_t
 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 
+int
+qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
+
 #ifdef RTE_LIB_SECURITY
 extern struct rte_security_ops security_qat_ops_gen1;

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +64 -0

76 lines not on this page · raw message

drivers/crypto/qat/qat_crypto.h +7 -1

index 6eaa15b975..5ca76fcaa6 100644
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Intel Corporation
+ * Copyright(c) 2022 Intel Corporation
  */
 
  #ifndef _QAT_CRYPTO_H_

@@ -48,15 +48,21 @@ typedef uint64_t (*get_feature_flags_t)(struct qat_pci_device *qat_dev);
 
 typedef void * (*create_security_ctx_t)(void *cryptodev);
 
+typedef int (*set_session_t)(void *cryptodev, void *session);
+
 struct qat_crypto_gen_dev_ops {
 	get_feature_flags_t get_feature_flags;
 	get_capabilities_info_t get_capabilities;
 	struct rte_cryptodev_ops *cryptodev_ops;
+	set_session_t set_session;
 #ifdef RTE_LIB_SECURITY
 	create_security_ctx_t create_security_ctx;
 #endif
 };
 
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[];
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[];
+
 int
 qat_cryptodev_config(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config);

drivers/crypto/qat/qat_sym.c +5 -7

index f814bf8f75..83bf55c933 100644
@@ -13,6 +13,10 @@
 #include "qat_sym.h"
 #include "dev/qat_crypto_pmd_gens.h"
 
+uint8_t qat_sym_driver_id;
+
+struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,

@@ -126,7 +130,7 @@ handle_spc_gmac(struct qat_sym_session *ctx, struct rte_crypto_op *op,
 
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
-		void *op_cookie, enum qat_device_gen qat_dev_gen)
+		void *op_cookie, __rte_unused enum qat_device_gen qat_dev_gen)
 {
 	int ret = 0;
 	struct qat_sym_session *ctx = NULL;

@@ -191,12 +195,6 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		return -EINVAL;
 	}
 
-	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
-		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
 	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;

drivers/crypto/qat/qat_sym_pmd.c +2 -2

index 28a26260fb..63ab9a5a8b 100644
@@ -20,9 +20,9 @@
 
 #define MIXED_CRYPTO_MIN_FW_VER 0x04090000
 
-uint8_t qat_sym_driver_id;
+extern uint8_t qat_sym_driver_id;
 
-struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
+extern struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
 
 void
 qat_sym_init_op_cookie(void *op_cookie)

drivers/crypto/qat/qat_sym_session.c +17 ~~-96~~

201 lines not on this page · raw message

drivers/crypto/qat/qat_sym_session.h +1 -1

index fe875a7fd0..01908abd9e 100644
@@ -105,7 +105,7 @@ struct qat_sym_session {
 	uint16_t auth_key_length;
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
+	uint16_t dev_id;
 	uint8_t aes_cmac;
 	uint8_t is_single_pass;
 	uint8_t is_single_pass_gmac;

Signature · 1 line

2.30.2

Kai Ji

author

2022-02-23 00:50 UTC #149 lore raw

PATCH v12 4/9 crypto/qat: rework asymmetric op build operation

In reply to Kai Ji

Signed-off-by Kai Ji <[email protected]>

Acked-by Fan Zhang <[email protected]>

---
 drivers/common/qat/qat_qp.c       |   5 +-
 drivers/crypto/qat/qat_asym.c     | 129 ++++++++++++++++--------------
 drivers/crypto/qat/qat_asym.h     |  63 +++++++++++++--
 drivers/crypto/qat/qat_asym_pmd.c |   4 +-
 4 files changed, 133 insertions(+), 68 deletions(-)

drivers/common/qat/qat_qp.c +3 -2

index 56234ca1a4..7f2fdc53ce 100644
@@ -619,7 +619,7 @@ qat_enqueue_op_burst(void *qp,
 #ifdef BUILD_QAT_ASYM
 			ret = qat_asym_build_request(*ops, base_addr + tail,
 				tmp_qp->op_cookies[tail >> queue->trailz],
-				tmp_qp->qat_dev_gen);
+				NULL, tmp_qp->qat_dev_gen);
 #endif
 		}
 		if (ret != 0) {

@@ -847,7 +847,8 @@ qat_dequeue_op_burst(void *qp, void **ops,
 #ifdef BUILD_QAT_ASYM
 		else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 			qat_asym_process_response(ops, resp_msg,
-				tmp_qp->op_cookies[head >> rx_queue->trailz]);
+				tmp_qp->op_cookies[head >> rx_queue->trailz],
+				NULL);
 #endif
 
 		head = adf_modulo(head + rx_queue->msg_size,

drivers/crypto/qat/qat_asym.c +71 ~~-58~~

174 lines not on this page · raw message

drivers/crypto/qat/qat_asym.h +57 -6

99 lines not on this page · raw message

drivers/crypto/qat/qat_asym_pmd.c +2 -2

index 9a7596b227..d56294d3d9 100644
@@ -10,8 +10,8 @@
 #include "qat_asym.h"
 #include "qat_asym_pmd.h"
 
-uint8_t qat_asym_driver_id;
-struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
+extern uint8_t qat_asym_driver_id;
+extern struct qat_crypto_gen_dev_ops qat_asym_gen_dev_ops[QAT_N_GENS];
 
 void
 qat_asym_init_op_cookie(void *op_cookie)

Signature · 1 line

2.30.2