drivers/qat: QAT symmetric crypto datapatch rework
Subsystems (2)
PATCH v12 5/9 crypto/qat: unify symmetric functions
In reply to Kai JiThis patch removes qat_sym_pmd.c and integrates all the functions into qat_sym.c. The unified/integrated qat sym crypto pmd functions should make them easier to maintain.
--- drivers/common/qat/meson.build | 4 +- drivers/common/qat/qat_device.c | 4 +- drivers/common/qat/qat_qp.c | 3 +- drivers/crypto/qat/qat_crypto.h | 5 +- drivers/crypto/qat/qat_sym.c | 21 +++ drivers/crypto/qat/qat_sym.h | 147 ++++++++++++++-- drivers/crypto/qat/qat_sym_hw_dp.c | 11 +- drivers/crypto/qat/qat_sym_pmd.c | 251 --------------------------- drivers/crypto/qat/qat_sym_pmd.h | 95 ---------- drivers/crypto/qat/qat_sym_session.c | 2 +- 10 files changed, 168 insertions(+), 375 deletions(-) delete mode 100644 drivers/crypto/qat/qat_sym_pmd.c delete mode 100644 drivers/crypto/qat/qat_sym_pmd.h
drivers/common/qat/meson.build +2 -2
@@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2017-2018 Intel Corporation +# Copyright(c) 2017-2022 Intel Corporation if is_windows build = false
@@ -73,7 +73,7 @@ if qat_compress endif if qat_crypto - foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c', + foreach f: ['qat_sym.c', 'qat_sym_session.c', 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c', 'dev/qat_sym_pmd_gen1.c', 'dev/qat_asym_pmd_gen1.c',
drivers/common/qat/qat_device.c +2 -2
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright(c) 2018-2020 Intel Corporation +* Copyright(c) 2018-2022 Intel Corporation */ #include <rte_string_fns.h>
@@ -8,7 +8,7 @@#include "qat_device.h" #include "adf_transport_access_macros.h" -#include "qat_sym_pmd.h" +#include "qat_sym.h" #include "qat_comp_pmd.h" #include "adf_pf2vf_msg.h" #include "qat_pf2vf.h"
drivers/common/qat/qat_qp.c +2 -1
@@ -838,7 +838,8 @@ qat_dequeue_op_burst(void *qp, void **ops,if (tmp_qp->service_type ==QAT_SERVICE_SYMMETRIC )qat_sym_process_response (ops, resp_msg, - tmp_qp->op_cookies[head >> rx_queue->trailz]); + tmp_qp->op_cookies[head >> rx_queue->trailz], +NULL );else if (tmp_qp->service_type ==QAT_SERVICE_COMPRESSION ) nb_fw_responses = qat_comp_process_response( ops, resp_msg,
drivers/crypto/qat/qat_crypto.h +4 -1
@@ -12,7 +12,10 @@extern uint8_t qat_sym_driver_id;extern uint8_t qat_asym_driver_id; -/** helper macro to set cryptodev capability range **/ +/** +* helper macro to set cryptodev capability range +* <n: name> <l: min > <r: max> <i: increment> <v: value> +**/ #define CAP_RNG (n, l, r, i) .n = {.min = l, .max = r, .increment = i}#define CAP_RNG_ZERO (n) .n = {.min = 0, .max = 0, .increment = 0}
drivers/crypto/qat/qat_sym.c +21 -0
@@ -17,6 +17,27 @@ uint8_t qat_sym_driver_id;struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS ]; +void +qat_sym_init_op_cookie (void *op_cookie) +{ +struct qat_sym_op_cookie *cookie = op_cookie; + + cookie->qat_sgl_src_phys_addr = +rte_mempool_virt2iova (cookie) + + offsetof(struct qat_sym_op_cookie , + qat_sgl_src); + + cookie->qat_sgl_dst_phys_addr = +rte_mempool_virt2iova (cookie) + + offsetof(struct qat_sym_op_cookie , + qat_sgl_dst); + + cookie->opt.spc_gmac.cd_phys_addr = +rte_mempool_virt2iova (cookie) + + offsetof(struct qat_sym_op_cookie , + opt.spc_gmac.cd_cipher); +} + static inline void set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, struct icp_qat_fw_la_cipher_req_params *cipher_param,
drivers/crypto/qat/qat_sym.h +135 -12
193 lines not on this page · raw message
drivers/crypto/qat/qat_sym_hw_dp.c +1 -10
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright(c) 2020 Intel Corporation +* Copyright(c) 2022 Intel Corporation */ #include <cryptodev_pmd.h>
@@ -9,18 +9,9 @@#include "icp_qat_fw_la.h" #include "qat_sym.h" -#include "qat_sym_pmd.h" #include "qat_sym_session.h" #include "qat_qp.h" -struct qat_sym_dp_ctx { -struct qat_sym_session *session; -uint32_t tail; -uint32_t head; -uint16_t cached_enqueue; -uint16_t cached_dequeue; -}; - static __rte_always_inline int32_tqat_sym_dp_parse_data_vec (struct qat_qp *qp,struct icp_qat_fw_la_bulk_req *req,struct rte_crypto_vec *data,uint16_t n_data_vecs)
drivers/crypto/qat/qat_sym_pmd.c deleted +0 -251
251 lines not on this page · raw message
drivers/crypto/qat/qat_sym_pmd.h deleted +0 -95
95 lines not on this page · raw message
drivers/crypto/qat/qat_sym_session.c +1 -1
@@ -20,7 +20,7 @@#include "qat_logs.h" #include "qat_sym_session.h" -#include "qat_sym_pmd.h" +#include "qat_sym.h" /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha1InitialState[] = {
Signature · 1 line
2.30.2
PATCH v12 6/9 crypto/qat: unify asymmetric functions
In reply to Kai JiThis patch removes qat_asym_pmd.c and integrates all the functions into qat_asym.c. The unified/integrated asym crypto pmd functions should make them easier to maintain.
--- drivers/common/qat/meson.build | 2 +- drivers/crypto/qat/qat_asym.c | 180 +++++++++++++++++++++++ drivers/crypto/qat/qat_asym_pmd.c | 231 ------------------------------ drivers/crypto/qat/qat_asym_pmd.h | 54 ------- 4 files changed, 181 insertions(+), 286 deletions(-) delete mode 100644 drivers/crypto/qat/qat_asym_pmd.c delete mode 100644 drivers/crypto/qat/qat_asym_pmd.h
drivers/common/qat/meson.build +1 -1
@@ -74,7 +74,7 @@ endif if qat_crypto foreach f: ['qat_sym.c', 'qat_sym_session.c', - 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c', + 'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c', 'dev/qat_sym_pmd_gen1.c', 'dev/qat_asym_pmd_gen1.c', 'dev/qat_crypto_pmd_gen2.c',
drivers/crypto/qat/qat_asym.c +180 -0
192 lines not on this page · raw message
drivers/crypto/qat/qat_asym_pmd.c deleted +0 -231
231 lines not on this page · raw message
drivers/crypto/qat/qat_asym_pmd.h deleted +0 -54
54 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v12 7/9 crypto/qat: rework burst data path
In reply to Kai JiThis patch enable the op_build_request function in qat_enqueue_op_burst, and the qat_dequeue_process_response function in qat_dequeue_op_burst.
The op_build_request invoked in crypto build request op is based on crypto operations setup'd during session init.
--- drivers/common/qat/qat_qp.c | 42 +- drivers/crypto/qat/dev/qat_sym_pmd_gen1.c | 4 - drivers/crypto/qat/qat_asym.c | 2 +- drivers/crypto/qat/qat_asym.h | 22 - drivers/crypto/qat/qat_sym.c | 830 +++++++--------------- drivers/crypto/qat/qat_sym.h | 5 - 6 files changed, 271 insertions(+), 634 deletions(-)
drivers/common/qat/qat_qp.c +9 -33
@@ -547,8 +547,7 @@ adf_modulo(uint32_t data, uint32_t modulo_mask) }uint16_t -qat_enqueue_op_burst (void *qp, - __rte_unused qat_op_build_request_t op_build_request, +qat_enqueue_op_burst (void *qp,qat_op_build_request_t op_build_request,void **ops,uint16_t nb_ops) { registerstruct qat_queue *queue;
@@ -599,29 +598,18 @@ qat_enqueue_op_burst(void *qp, } } -#ifdef BUILD_QAT_SYM +#ifdef RTE_LIB_SECURITY if (tmp_qp->service_type ==QAT_SERVICE_SYMMETRIC )qat_sym_preprocess_requests (ops, nb_ops_possible);#endif +memset (tmp_qp->opaque,0xff ,sizeof (tmp_qp->opaque)); +while (nb_ops_sent != nb_ops_possible) { -if (tmp_qp->service_type ==QAT_SERVICE_SYMMETRIC ) { -#ifdef BUILD_QAT_SYM - ret =qat_sym_build_request (*ops, base_addr + tail, - tmp_qp->op_cookies[tail >> queue->trailz], - tmp_qp->qat_dev_gen); -#endif - }else if (tmp_qp->service_type ==QAT_SERVICE_COMPRESSION ) { - ret =qat_comp_build_request (*ops, base_addr + tail, + ret =op_build_request (*ops, base_addr + tail, tmp_qp->op_cookies[tail >> queue->trailz], - tmp_qp->qat_dev_gen); - }else if (tmp_qp->service_type ==QAT_SERVICE_ASYMMETRIC ) { -#ifdef BUILD_QAT_ASYM - ret =qat_asym_build_request (*ops, base_addr + tail, - tmp_qp->op_cookies[tail >> queue->trailz], -NULL , tmp_qp->qat_dev_gen); -#endif - } + tmp_qp->opaque, tmp_qp->qat_dev_gen); +if (ret !=0 ) { tmp_qp->stats.enqueue_err_count++;/* This message cannot be enqueued */
@@ -817,8 +805,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)uint16_t qat_dequeue_op_burst (void *qp,void **ops, - __rte_unused qat_op_dequeue_t qat_dequeue_process_response, -uint16_t nb_ops) +qat_op_dequeue_t qat_dequeue_process_response,uint16_t nb_ops) {struct qat_queue *rx_queue;struct qat_qp *tmp_qp = (struct qat_qp *)qp;
@@ -836,21 +823,10 @@ qat_dequeue_op_burst(void *qp, void **ops, nb_fw_responses =1 ; -if (tmp_qp->service_type ==QAT_SERVICE_SYMMETRIC ) -qat_sym_process_response (ops, resp_msg, - tmp_qp->op_cookies[head >> rx_queue->trailz], -NULL ); -else if (tmp_qp->service_type ==QAT_SERVICE_COMPRESSION ) - nb_fw_responses =qat_comp_process_response ( + nb_fw_responses =qat_dequeue_process_response ( ops, resp_msg, tmp_qp->op_cookies[head >> rx_queue->trailz], &tmp_qp->stats.dequeue_err_count); -#ifdef BUILD_QAT_ASYM - else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) - qat_asym_process_response(ops, resp_msg, - tmp_qp->op_cookies[head >> rx_queue->trailz], - NULL); -#endif head =adf_modulo (head + rx_queue->msg_size, rx_queue->modulo_mask);
drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +0 -4
@@ -146,10 +146,6 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = { .sym_session_get_size = qat_sym_session_get_private_size, .sym_session_configure = qat_sym_session_configure, .sym_session_clear = qat_sym_session_clear, - -/* Raw data-path API related operations */ - .sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size, - .sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx, }; static struct qat_capabilities_info
drivers/crypto/qat/qat_asym.c +1 -1
@@ -507,7 +507,7 @@ qat_asym_fill_arrays(struct rte_crypto_asym_op *asym_op,return 0 ; } -int +static __rte_always_inline intqat_asym_build_request (void *in_op,uint8_t *out_msg,void *op_cookie, __rte_unused uint64_t *opaque, __rte_unused enum qat_device_gen dev_gen)
drivers/crypto/qat/qat_asym.h +0 -22
@@ -103,28 +103,6 @@ void qat_asym_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_asym_session *sess); -/* -* Build PKE request to be sent to the fw, partially uses template -* request generated during session creation. -* -* @param in_op Pointer to the crypto operation, for every -* service it points to service specific struct. -* @param out_msg Message to be returned to enqueue function -* @param op_cookie Cookie pointer that holds private metadata -* @param qat_dev_gen Generation of QAT hardware -* -* @return -* This function always returns zero, -* it is because of backward compatibility. -* - 0: Always returned -* -*/ -int -qat_asym_build_request (void *in_op,uint8_t *out_msg, -void *op_cookie, - __rte_unused uint64_t *opaque, -enum qat_device_gen qat_dev_gen); -/* * Process PKE response received from outgoing queue of QAT *
drivers/crypto/qat/qat_sym.c +261 -569
896 lines not on this page · raw message
drivers/crypto/qat/qat_sym.h +0 -5
@@ -131,11 +131,6 @@ uint16_t qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops); -int -qat_sym_build_request (void *in_op,uint8_t *out_msg, -void *op_cookie,enum qat_device_gen qat_dev_gen); - -/** Encrypt a single partial block * Depends on openssl libcrypto * Uses ECB+XOR to do CFB encryption, same result, more performant
Signature · 1 line
2.30.2
PATCH v12 8/9 crypto/qat: unify raw data path functions
In reply to Kai JiThis patch unifies QAT's raw dp api implementations
to the same enqueue/dequeue methods used in crypto operations. The specific functions for different QAT generation are updated respectively. The qat_sym_hw_dp.c is removed as no longer required.
--- drivers/common/qat/meson.build | 2 +- drivers/compress/qat/qat_comp_pmd.c | 12 +- drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c | 2 + drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 214 ++++ drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 122 +++ drivers/crypto/qat/dev/qat_crypto_pmd_gens.h | 78 ++ drivers/crypto/qat/dev/qat_sym_pmd_gen1.c | 656 ++++++++++++ drivers/crypto/qat/qat_crypto.h | 3 + drivers/crypto/qat/qat_sym.c | 56 +- drivers/crypto/qat/qat_sym_hw_dp.c | 986 ------------------- 10 files changed, 1137 insertions(+), 994 deletions(-) delete mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c
drivers/common/qat/meson.build +1 -1
@@ -74,7 +74,7 @@ endif if qat_crypto foreach f: ['qat_sym.c', 'qat_sym_session.c', - 'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c', + 'qat_asym.c', 'qat_crypto.c', 'dev/qat_sym_pmd_gen1.c', 'dev/qat_asym_pmd_gen1.c', 'dev/qat_crypto_pmd_gen2.c',
drivers/compress/qat/qat_comp_pmd.c +9 -3
@@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = { .private_xform_free = qat_comp_private_xform_free }; +static uint16_t +qat_comp_dequeue_burst (void *qp,struct rte_comp_op **ops,uint16_t nb_ops) +{ +return qat_dequeue_op_burst (qp, (void **)ops, qat_comp_process_response, + nb_ops); +} +static uint16_t qat_comp_pmd_dequeue_first_op_burst (void *qp,struct rte_comp_op **ops,uint16_t nb_ops) { -uint16_t ret =qat_dequeue_op_burst (qp, (void **)ops,NULL , nb_ops); +uint16_t ret =qat_comp_dequeue_burst (qp, ops, nb_ops);struct qat_qp *tmp_qp = (struct qat_qp *)qp;if (ret) {
@@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops, }else { tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst = - (compressdev_dequeue_pkt_burst_t ) - qat_comp_pmd_enq_deq_dummy_op_burst; + qat_comp_dequeue_burst; } }return ret;
drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c +2 -0
@@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init) qat_sym_crypto_cap_get_gen2; qat_sym_gen_dev_ops[QAT_GEN2 ].set_session = qat_sym_crypto_set_session_gen2; + qat_sym_gen_dev_ops[QAT_GEN2 ].set_raw_dp_ctx = + qat_sym_configure_raw_dp_ctx_gen1; qat_sym_gen_dev_ops[QAT_GEN2 ].get_feature_flags = qat_sym_crypto_feature_flags_get_gen1;
drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +214 -0
226 lines not on this page · raw message
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +122 -0
134 lines not on this page · raw message
drivers/crypto/qat/dev/qat_crypto_pmd_gens.h +78 -0
84 lines not on this page · raw message
drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +656 -0
674 lines not on this page · raw message
drivers/crypto/qat/qat_crypto.h +3 -0
@@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);typedef int (*set_session_t )(void *cryptodev,void *session); +typedef int (*set_raw_dp_ctx_t )(void *raw_dp_ctx,void *ctx); +struct qat_crypto_gen_dev_ops {get_feature_flags_t get_feature_flags;get_capabilities_info_t get_capabilities;struct rte_cryptodev_ops *cryptodev_ops;set_session_t set_session; +set_raw_dp_ctx_t set_raw_dp_ctx;#ifdef RTE_LIB_SECURITY create_security_ctx_t create_security_ctx;#endif
drivers/crypto/qat/qat_sym.c +52 -4
80 lines not on this page · raw message
drivers/crypto/qat/qat_sym_hw_dp.c deleted +0 -986
986 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v12 9/9 crypto/qat: support out of place SG list
In reply to Kai JiThis patch adds the SGL out of place support to QAT PMD
--- drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c | 28 ++++++++-- drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c | 14 ++++- drivers/crypto/qat/dev/qat_sym_pmd_gen1.c | 55 +++++++++++++++++--- 3 files changed, 83 insertions(+), 14 deletions(-)
drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c +24 -4
@@ -468,8 +468,18 @@ qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx, (uint8_t *)tx_queue->base_addr + tail);rte_mov128 ((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); - data_len =qat_sym_build_req_set_data (req, user_data[i], cookie, - vec->src_sgl[i].vec, vec->src_sgl[i].num,NULL ,0 ); +if (vec->dest_sgl) { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, vec->src_sgl[i].num, + vec->dest_sgl[i].vec, vec->dest_sgl[i].num); + }else { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, + vec->src_sgl[i].num,NULL ,0 ); + } +if (unlikely (data_len <0 ))break ;
@@ -565,8 +575,18 @@ qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx, (uint8_t *)tx_queue->base_addr + tail);rte_mov128 ((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); - data_len =qat_sym_build_req_set_data (req, user_data[i], cookie, - vec->src_sgl[i].vec, vec->src_sgl[i].num,NULL ,0 ); +if (vec->dest_sgl) { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, vec->src_sgl[i].num, + vec->dest_sgl[i].vec, vec->dest_sgl[i].num); + }else { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, + vec->src_sgl[i].num,NULL ,0 ); + } +if (unlikely (data_len <0 ))break ; enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c +12 -2
@@ -297,8 +297,18 @@ qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx, (uint8_t *)tx_queue->base_addr + tail);rte_mov128 ((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); - data_len =qat_sym_build_req_set_data (req, user_data[i], cookie, - vec->src_sgl[i].vec, vec->src_sgl[i].num,NULL ,0 ); +if (vec->dest_sgl) { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, vec->src_sgl[i].num, + vec->dest_sgl[i].vec, vec->dest_sgl[i].num); + }else { + data_len =qat_sym_build_req_set_data (req, + user_data[i], cookie, + vec->src_sgl[i].vec, + vec->src_sgl[i].num,NULL ,0 ); + } +if (unlikely (data_len <0 ))break ;
drivers/crypto/qat/dev/qat_sym_pmd_gen1.c +47 -8
80 lines not on this page · raw message
Signature · 1 line
2.30.2
Akhil Goyal
AppliedQuoted 25 lines
v11 - v12:
- fixed a compile issuev10:
- rebase to the lastest for-main
- fix of build rerror whenRTE_LOG_DEBUGenabledv9:
- commit messages reword
- fix of unused function errorv8:
- rebase to 22.03-rc1v7:
- fix of pointer cast compile error in x86v6:
- fix of pointer cast error in x86
- rebase to the lastest for-mainv5:
- rebase to the latest for-main
- patchset reconstruct
Quoted 9 lines
v4:
- patchset break down and reconstructv3:
- sperate a single patch 6 to two different patchesv2:
- review comments addressed
Applied to dpdk-next-crypto