crypto/qat: add fix for Gen4 WRITE
PATCH 1/1 crypto/qat: add fix for Gen4 WRITE
All generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.
--- drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 121 ++++++++++++++++++- drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 6 + 2 files changed, 123 insertions(+), 4 deletions(-)
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +117 -4
@@ -9,6 +9,7 @@#include "qat_asym.h" #include "qat_crypto.h" #include "qat_crypto_pmd_gens.h" +#include "adf_transport_access_macros_gen4vf.h" static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen4[] = {
@@ -233,6 +234,78 @@ qat_sym_build_op_aead_gen4(void *in_op, struct qat_sym_session *ctx,return 0 ; } +int +qat_sym_dp_enqueue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t n) +{ +struct qat_qp *qp = qp_data; +struct qat_queue *tx_queue = &qp->tx_q; +struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; + +if (unlikely (dp_ctx->cached_enqueue != n)) +return -1 ; + + qp->enqueued += n; + qp->stats.enqueued_count += n; + + tx_queue->tail = dp_ctx->tail; + +WRITE_CSR_RING_TAIL_GEN4VF (qp->mmap_bar_addr, + tx_queue->hw_bundle_number, + tx_queue->hw_queue_number, tx_queue->tail); + + tx_queue->csr_tail = tx_queue->tail; + dp_ctx->cached_enqueue =0 ; + +return 0 ; +} + +int +qat_sym_dp_dequeue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t n) +{ +struct qat_qp *qp = qp_data; +struct qat_queue *rx_queue = &qp->rx_q; +struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; + +if (unlikely (dp_ctx->cached_dequeue != n)) +return -1 ; + + rx_queue->head = dp_ctx->head; + rx_queue->nb_processed_responses += n; + qp->dequeued += n; + qp->stats.dequeued_count += n; +if (rx_queue->nb_processed_responses >QAT_CSR_HEAD_WRITE_THRESH ) { +uint32_t old_head, new_head; +uint32_t max_head; + + old_head = rx_queue->csr_head; + new_head = rx_queue->head; + max_head = qp->nb_descriptors * rx_queue->msg_size; + +/* write out free descriptors */ +void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head; + +if (new_head < old_head) { +memset (cur_desc,ADF_RING_EMPTY_SIG_BYTE , + max_head - old_head); +memset (rx_queue->base_addr,ADF_RING_EMPTY_SIG_BYTE , + new_head); + }else { +memset (cur_desc,ADF_RING_EMPTY_SIG_BYTE , new_head - + old_head); + } + rx_queue->nb_processed_responses =0 ; + rx_queue->csr_head = new_head; + +/* write current head to CSR */ +WRITE_CSR_RING_HEAD_GEN4VF (qp->mmap_bar_addr, + rx_queue->hw_bundle_number, rx_queue->hw_queue_number, + new_head); + } + + dp_ctx->cached_dequeue =0 ; +return 0 ; +} +int qat_sym_crypto_set_session_gen4 (void *cdev,void *session) {
@@ -390,11 +463,51 @@ qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx) {struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;struct qat_sym_session *ctx = _ctx; -int ret; - ret =qat_sym_configure_raw_dp_ctx_gen1 (_raw_dp_ctx, _ctx); -if (ret <0 ) -return ret; + raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen4; + raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1; + raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1; + raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen4; + +if ((ctx->qat_cmd ==ICP_QAT_FW_LA_CMD_HASH_CIPHER || + ctx->qat_cmd ==ICP_QAT_FW_LA_CMD_CIPHER_HASH ) && + !ctx->is_gmac) { +/* AES-GCM or AES-CCM */ +if (ctx->qat_hash_alg ==ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg ==ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || + (ctx->qat_cipher_alg ==ICP_QAT_HW_CIPHER_ALGO_AES128 + && ctx->qat_mode ==ICP_QAT_HW_CIPHER_CTR_MODE + && ctx->qat_hash_alg == +ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC )) { + raw_dp_ctx->enqueue_burst = + qat_sym_dp_enqueue_aead_jobs_gen1; + raw_dp_ctx->enqueue = + qat_sym_dp_enqueue_single_aead_gen1; + }else { + raw_dp_ctx->enqueue_burst = + qat_sym_dp_enqueue_chain_jobs_gen1; + raw_dp_ctx->enqueue = + qat_sym_dp_enqueue_single_chain_gen1; + } + }else if (ctx->qat_cmd ==ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) { + raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1; + raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1; + }else if (ctx->qat_cmd ==ICP_QAT_FW_LA_CMD_CIPHER ) { +if (ctx->qat_mode ==ICP_QAT_HW_CIPHER_AEAD_MODE || + ctx->qat_cipher_alg == +ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305 ) { + raw_dp_ctx->enqueue_burst = + qat_sym_dp_enqueue_aead_jobs_gen1; + raw_dp_ctx->enqueue = + qat_sym_dp_enqueue_single_aead_gen1; + }else { + raw_dp_ctx->enqueue_burst = + qat_sym_dp_enqueue_cipher_jobs_gen1; + raw_dp_ctx->enqueue = + qat_sym_dp_enqueue_single_cipher_gen1; + } + }else +return -1 ;if (ctx->is_single_pass && ctx->is_ucs) { raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;
drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +6 -0
@@ -1040,6 +1040,12 @@ qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);int qat_sym_dp_dequeue_done_gen1 (void *qp_data,uint8_t *drv_ctx,uint32_t n); +int +qat_sym_dp_enqueue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t n); + +int +qat_sym_dp_dequeue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t n); +int qat_sym_configure_raw_dp_ctx_gen1 (void *_raw_dp_ctx,void *_ctx);
Signature · 1 line
2.25.1
Quoted 215 lines
-----Original Message-----
From: Dooley, Brian <[email protected]>
Sent: Friday, July 12, 2024 4:49 PM
To: Ji, Kai <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; Kusztal, ArkadiuszX <[email protected]>; Dooley, Brian <[email protected]>
Subject: [PATCH v1]crypto/qat: add fix for Gen4 WRITEAll generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.
Fixes 85fec6fd9674 ("crypto/qat: unify raw data path functions")Signed-off-by Brian Dooley <[email protected]>--- drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 121 ++++++++++++++++++- drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 6 + 2 files changed, 123 insertions(+), 4 deletions(-)drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +58
-0@@ -9,6 +9,7 @@#include "qat_asym.h" #include "qat_crypto.h" #include "qat_crypto_pmd_gens.h" +#include "adf_transport_access_macros_gen4vf.h" static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen4[] = {@@ -233,6 +234,78 @@ qat_sym_build_op_aead_gen4(void *in_op, structqat_sym_session *ctx, return 0; } +int +qat_sym_dp_enqueue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t +n) { +struct qat_qp *qp = qp_data; +struct qat_queue *tx_queue = &qp->tx_q; +struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; + +if (unlikely (dp_ctx->cached_enqueue != n)) +return -1 ; + + qp->enqueued += n; + qp->stats.enqueued_count += n; + + tx_queue->tail = dp_ctx->tail; + +WRITE_CSR_RING_TAIL_GEN4VF (qp->mmap_bar_addr, + tx_queue->hw_bundle_number, + tx_queue->hw_queue_number, tx_queue->tail); + + tx_queue->csr_tail = tx_queue->tail; + dp_ctx->cached_enqueue =0 ; + +return 0 ; +} + +int +qat_sym_dp_dequeue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t +n) { +struct qat_qp *qp = qp_data; +struct qat_queue *rx_queue = &qp->rx_q; +struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx; + +if (unlikely (dp_ctx->cached_dequeue != n)) +return -1 ; + + rx_queue->head = dp_ctx->head; + rx_queue->nb_processed_responses += n; + qp->dequeued += n; + qp->stats.dequeued_count += n; +if (rx_queue->nb_processed_responses >QAT_CSR_HEAD_WRITE_THRESH ) { +uint32_t old_head, new_head; +uint32_t max_head; + + old_head = rx_queue->csr_head; + new_head = rx_queue->head; + max_head = qp->nb_descriptors * rx_queue->msg_size; + +/* write out free descriptors */ +void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head; + +if (new_head < old_head) { +memset (cur_desc,ADF_RING_EMPTY_SIG_BYTE , + max_head - old_head); +memset (rx_queue->base_addr,ADF_RING_EMPTY_SIG_BYTE , + new_head); + }else { + memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
new_head-drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +8
-0+ old_head); + } + rx_queue->nb_processed_responses =0 ; + rx_queue->csr_head = new_head; + +/* write current head to CSR */ + WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr, + rx_queue->hw_bundle_number, rx_queue-Quoted 1 line
hw_queue_number,drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +8
-0+ new_head); + } + + dp_ctx->cached_dequeue =0 ; +return 0 ; +} +int qat_sym_crypto_set_session_gen4(void *cdev, void *session) { @@ -390,11 +463,51 @@ qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void
*_ctx) {drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +7
-4struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;struct qat_sym_session *ctx = _ctx; -int ret; - ret =qat_sym_configure_raw_dp_ctx_gen1 (_raw_dp_ctx, _ctx); -if (ret <0 ) -return ret; + raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen4; + raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1; + raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1; + raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen4; + + if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || + ctx->qat_cmd ==
ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3
-0+ !ctx->is_gmac) { +/* AES-GCM or AES-CCM */ + if (ctx->qat_hash_alg ==
ICP_QAT_HW_AUTH_ALGO_GALOIS_128||drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1
-0+ ctx->qat_hash_alg ==
ICP_QAT_HW_AUTH_ALGO_GALOIS_64||drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1
-0+ (ctx->qat_cipher_alg ==
ICP_QAT_HW_CIPHER_ALGO_AES128drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1
-0+ && ctx->qat_mode ==
ICP_QAT_HW_CIPHER_CTR_MODEdrivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ && ctx->qat_hash_alg == +ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue_burst = +qat_sym_dp_enqueue_aead_jobs_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue = +qat_sym_dp_enqueue_single_aead_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3
-0+ }else { + raw_dp_ctx->enqueue_burst = +qat_sym_dp_enqueue_chain_jobs_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue = +qat_sym_dp_enqueue_single_chain_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ } + } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx-Quoted 1 line
is_gmac) {drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1
-0+ raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_auth_jobs_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +1
-0+ raw_dp_ctx->enqueue =
qat_sym_dp_enqueue_single_auth_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +4
-0+ } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { + if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE || + ctx->qat_cipher_alg == +ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue_burst = +qat_sym_dp_enqueue_aead_jobs_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue = +qat_sym_dp_enqueue_single_aead_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3
-0+ }else { + raw_dp_ctx->enqueue_burst = +qat_sym_dp_enqueue_cipher_jobs_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +2
-0+ raw_dp_ctx->enqueue = +qat_sym_dp_enqueue_single_cipher_gen1;drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +3
-0+ } + }else +return -1 ;if (ctx->is_single_pass && ctx->is_ucs) { raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_aead_jobs_gen4;drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +0
-0@@ -1040,6 +1040,12 @@ qat_sym_dp_enqueue_done_gen1(void *qp_data,
uint8_t*drv_ctx,uint32_tn); intqat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +8
-1+int +qat_sym_dp_enqueue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t +n); + +int +qat_sym_dp_dequeue_done_gen4 (void *qp_data,uint8_t *drv_ctx,uint32_t +n); +int qat_sym_configure_raw_dp_ctx_gen1 (void *_raw_dp_ctx,void *_ctx); --2.25.1
Akhil Goyal
AppliedQuoted 10 lines
Quoted 9 lines
All generations of QAT use the same Gen1 raw datapath. Gen4 needs a different WRITE function than other generations. Added separation for configuration of the raw ctx for Gen4 from the Gen1 codepath.
Fixes 85fec6fd9674 ("crypto/qat: unify raw data path functions")Signed-off-by Brian Dooley <[email protected]>
Applied to dpdk-next-crypto