Skip to contentSkip to conversation

crypto/qat: add fix for Gen4 WRITE

PATCH Applied Brian Dooley 3 participants 3 messages Jul 12, 2024 – Jul 18, 2024 Crypto API

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 crypto/qat: add fix for Gen4 WRITE

All generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.

Fixes 85fec6fd9674 ("crypto/qat: unify raw data path functions")
---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 121 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   6 +
 2 files changed, 123 insertions(+), 4 deletions(-)
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +117 -4
index 5e808a60bf..6a5d6e78b9 100644
@@ -9,6 +9,7 @@
 #include "qat_asym.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
+#include "adf_transport_access_macros_gen4vf.h"
 
 
 static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen4[] = {
@@ -233,6 +234,78 @@ qat_sym_build_op_aead_gen4(void *in_op, struct qat_sym_session *ctx,
 	return 0;
 }
 
+int
+qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n)
+{
+	struct qat_qp *qp = qp_data;
+	struct qat_queue *tx_queue = &qp->tx_q;
+	struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
+
+	if (unlikely(dp_ctx->cached_enqueue != n))
+		return -1;
+
+	qp->enqueued += n;
+	qp->stats.enqueued_count += n;
+
+	tx_queue->tail = dp_ctx->tail;
+
+	WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
+		tx_queue->hw_bundle_number,
+		tx_queue->hw_queue_number, tx_queue->tail);
+
+	tx_queue->csr_tail = tx_queue->tail;
+	dp_ctx->cached_enqueue = 0;
+
+	return 0;
+}
+
+int
+qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n)
+{
+	struct qat_qp *qp = qp_data;
+	struct qat_queue *rx_queue = &qp->rx_q;
+	struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
+
+	if (unlikely(dp_ctx->cached_dequeue != n))
+		return -1;
+
+	rx_queue->head = dp_ctx->head;
+	rx_queue->nb_processed_responses += n;
+	qp->dequeued += n;
+	qp->stats.dequeued_count += n;
+	if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
+		uint32_t old_head, new_head;
+		uint32_t max_head;
+
+		old_head = rx_queue->csr_head;
+		new_head = rx_queue->head;
+		max_head = qp->nb_descriptors * rx_queue->msg_size;
+
+		/* write out free descriptors */
+		void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
+
+		if (new_head < old_head) {
+			memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
+					max_head - old_head);
+			memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
+					new_head);
+		} else {
+			memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
+					old_head);
+		}
+		rx_queue->nb_processed_responses = 0;
+		rx_queue->csr_head = new_head;
+
+		/* write current head to CSR */
+		WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
+			rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
+			new_head);
+	}
+
+	dp_ctx->cached_dequeue = 0;
+	return 0;
+}
+
 int
 qat_sym_crypto_set_session_gen4(void *cdev, void *session)
 {
@@ -390,11 +463,51 @@ qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx)
 {
 	struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
 	struct qat_sym_session *ctx = _ctx;
-	int ret;
 
-	ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
-	if (ret < 0)
-		return ret;
+	raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen4;
+	raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1;
+	raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1;
+	raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen4;
+
+	if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
+			ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
+			!ctx->is_gmac) {
+		/* AES-GCM or AES-CCM */
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
+			(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
+			&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
+			&& ctx->qat_hash_alg ==
+					ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
+			raw_dp_ctx->enqueue_burst =
+					qat_sym_dp_enqueue_aead_jobs_gen1;
+			raw_dp_ctx->enqueue =
+					qat_sym_dp_enqueue_single_aead_gen1;
+		} else {
+			raw_dp_ctx->enqueue_burst =
+					qat_sym_dp_enqueue_chain_jobs_gen1;
+			raw_dp_ctx->enqueue =
+					qat_sym_dp_enqueue_single_chain_gen1;
+		}
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
+		raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1;
+		raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1;
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
+		if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
+			ctx->qat_cipher_alg ==
+				ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
+			raw_dp_ctx->enqueue_burst =
+					qat_sym_dp_enqueue_aead_jobs_gen1;
+			raw_dp_ctx->enqueue =
+					qat_sym_dp_enqueue_single_aead_gen1;
+		} else {
+			raw_dp_ctx->enqueue_burst =
+					qat_sym_dp_enqueue_cipher_jobs_gen1;
+			raw_dp_ctx->enqueue =
+					qat_sym_dp_enqueue_single_cipher_gen1;
+		}
+	} else
+		return -1;
 
 	if (ctx->is_single_pass && ctx->is_ucs) {
 		raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;
drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +6 -0
index 1f5d2583c4..2c5816e696 100644
@@ -1040,6 +1040,12 @@ qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
 int
 qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
 
+int
+qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
+int
+qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t n);
+
 int
 qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
 
Signature · 1 line
2.25.1
Acked-by on v1 1/1
#1 lore raw
Quoted 215 lines

-----Original Message-----
From: Dooley, Brian <[email protected]>
Sent: Friday, July 12, 2024 4:49 PM
To: Ji, Kai <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; Kusztal, ArkadiuszX <[email protected]>; Dooley, Brian <[email protected]>
Subject: [PATCH v1] crypto/qat: add fix for Gen4 WRITE

All generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.

Fixes 85fec6fd9674 ("crypto/qat: unify raw data path functions")
---
 drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 121 ++++++++++++++++++-
 drivers/crypto/qat/dev/qat_crypto_pmd_gens.h |   6 +
 2 files changed, 123 insertions(+), 4 deletions(-)
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +58 -0
index 5e808a60bf..6a5d6e78b9 100644
@@ -9,6 +9,7 @@
 #include "qat_asym.h"
 #include "qat_crypto.h"
 #include "qat_crypto_pmd_gens.h"
+#include "adf_transport_access_macros_gen4vf.h"
 
 
 static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen4[] = {
@@ -233,6 +234,78 @@ qat_sym_build_op_aead_gen4(void *in_op, struct
 qat_sym_session *ctx,
 	return 0;
 }
 
+int
+qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t
+n) {
+	struct qat_qp *qp = qp_data;
+	struct qat_queue *tx_queue = &qp->tx_q;
+	struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
+
+	if (unlikely(dp_ctx->cached_enqueue != n))
+		return -1;
+
+	qp->enqueued += n;
+	qp->stats.enqueued_count += n;
+
+	tx_queue->tail = dp_ctx->tail;
+
+	WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
+		tx_queue->hw_bundle_number,
+		tx_queue->hw_queue_number, tx_queue->tail);
+
+	tx_queue->csr_tail = tx_queue->tail;
+	dp_ctx->cached_enqueue = 0;
+
+	return 0;
+}
+
+int
+qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t
+n) {
+	struct qat_qp *qp = qp_data;
+	struct qat_queue *rx_queue = &qp->rx_q;
+	struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
+
+	if (unlikely(dp_ctx->cached_dequeue != n))
+		return -1;
+
+	rx_queue->head = dp_ctx->head;
+	rx_queue->nb_processed_responses += n;
+	qp->dequeued += n;
+	qp->stats.dequeued_count += n;
+	if (rx_queue->nb_processed_responses >
 QAT_CSR_HEAD_WRITE_THRESH) {
+		uint32_t old_head, new_head;
+		uint32_t max_head;
+
+		old_head = rx_queue->csr_head;
+		new_head = rx_queue->head;
+		max_head = qp->nb_descriptors * rx_queue->msg_size;
+
+		/* write out free descriptors */
+		void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
+
+		if (new_head < old_head) {
+			memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
+					max_head - old_head);
+			memset(rx_queue->base_addr,
 ADF_RING_EMPTY_SIG_BYTE,
+					new_head);
+		} else {
+			memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,

new_head -

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +8 -0
+					old_head);
+		}
+		rx_queue->nb_processed_responses = 0;
+		rx_queue->csr_head = new_head;
+
+		/* write current head to CSR */
+		WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
+			rx_queue->hw_bundle_number, rx_queue-
Quoted 1 line

hw_queue_number,

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +8 -0
+			new_head);
+	}
+
+	dp_ctx->cached_dequeue = 0;
+	return 0;
+}
+
 int
 qat_sym_crypto_set_session_gen4(void *cdev, void *session)  { @@ -390,11
+463,51 @@ qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void

*_ctx) {

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +7 -4
 	struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
 	struct qat_sym_session *ctx = _ctx;
-	int ret;
 
-	ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
-	if (ret < 0)
-		return ret;
+	raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen4;
+	raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1;
+	raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1;
+	raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen4;
+
+	if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
+			ctx->qat_cmd ==

ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3 -0
+			!ctx->is_gmac) {
+		/* AES-GCM or AES-CCM */
+		if (ctx->qat_hash_alg ==

ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1 -0
+			ctx->qat_hash_alg ==

ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1 -0
+			(ctx->qat_cipher_alg ==

ICP_QAT_HW_CIPHER_ALGO_AES128

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1 -0
+			&& ctx->qat_mode ==

ICP_QAT_HW_CIPHER_CTR_MODE

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			&& ctx->qat_hash_alg ==
+
	ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue_burst =
+
	qat_sym_dp_enqueue_aead_jobs_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue =
+
	qat_sym_dp_enqueue_single_aead_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3 -0
+		} else {
+			raw_dp_ctx->enqueue_burst =
+
	qat_sym_dp_enqueue_chain_jobs_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue =
+
	qat_sym_dp_enqueue_single_chain_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+		}
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx-
Quoted 1 line

is_gmac) {

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1 -0
+		raw_dp_ctx->enqueue_burst =

qat_sym_dp_enqueue_auth_jobs_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1 -0
+		raw_dp_ctx->enqueue =

qat_sym_dp_enqueue_single_auth_gen1;

drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +4 -0
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
+		if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
+			ctx->qat_cipher_alg ==
+
	ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue_burst =
+
	qat_sym_dp_enqueue_aead_jobs_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue =
+
	qat_sym_dp_enqueue_single_aead_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3 -0
+		} else {
+			raw_dp_ctx->enqueue_burst =
+
	qat_sym_dp_enqueue_cipher_jobs_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2 -0
+			raw_dp_ctx->enqueue =
+
	qat_sym_dp_enqueue_single_cipher_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3 -0
+		}
+	} else
+		return -1;
 
 	if (ctx->is_single_pass && ctx->is_ucs) {
 		raw_dp_ctx->enqueue_burst =

qat_sym_dp_enqueue_aead_jobs_gen4;

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +0 -0
index 1f5d2583c4..2c5816e696 100644
@@ -1040,6 +1040,12 @@ qat_sym_dp_enqueue_done_gen1(void *qp_data,

uint8_t *drv_ctx, uint32_t n); int qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +8 -1
+int
+qat_sym_dp_enqueue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t
+n);
+
+int
+qat_sym_dp_dequeue_done_gen4(void *qp_data, uint8_t *drv_ctx, uint32_t
+n);
+
 int
 qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
 
--

2.25.1

Applied
#2 lore raw
Quoted 10 lines
Quoted 9 lines

All generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.

Fixes 85fec6fd9674 ("crypto/qat: unify raw data path functions")

Applied to dpdk-next-crypto