crypto: Driver conversions for DMA alignment
Subsystems (13)
Herbert Xu
maintainercover crypto: Driver conversions for DMA alignment
These are the rest of the driver conversions in order for arm64 to safely lower the kmalloc alignment below that required for DMA.
My criteria for inclusion are:
1) the driver can be built on arm64.
2) the driver may perform DMA on the context structure.
I have worked through all the drivers in crypto but if you think I've missed something please let me know.
Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerPATCH 1/10 crypto: cavium - Set DMA alignment explicitly
This driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/cavium/cpt/cptvf_algs.c | 10 +++++----- drivers/crypto/cavium/nitrox/nitrox_aead.c | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-)
drivers/crypto/cavium/cpt/cptvf_algs.c +5 -5
@@ -97,7 +97,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req);struct cvm_enc_ctx *ctx =crypto_skcipher_ctx (tfm); -struct cvm_req_ctx *rctx =skcipher_request_ctx (req); +struct cvm_req_ctx *rctx =skcipher_request_ctx_dma (req);struct fc_context *fctx = &rctx->fctx;u32 enc_iv_len =crypto_skcipher_ivsize (tfm);struct cpt_request_info *req_info = &rctx->cpt_req;
@@ -151,7 +151,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,static inline u32 create_input_list (struct skcipher_request *req,u32 enc,u32 enc_iv_len) { -struct cvm_req_ctx *rctx =skcipher_request_ctx (req); +struct cvm_req_ctx *rctx =skcipher_request_ctx_dma (req);struct cpt_request_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;
@@ -173,7 +173,7 @@ static inline void store_cb_info(struct skcipher_request *req,static inline void create_output_list (struct skcipher_request *req,u32 enc_iv_len) { -struct cvm_req_ctx *rctx =skcipher_request_ctx (req); +struct cvm_req_ctx *rctx =skcipher_request_ctx_dma (req);struct cpt_request_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;
@@ -193,7 +193,7 @@ static inline void create_output_list(struct skcipher_request *req,static inline int cvm_enc_dec (struct skcipher_request *req,u32 enc) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req); -struct cvm_req_ctx *rctx =skcipher_request_ctx (req); +struct cvm_req_ctx *rctx =skcipher_request_ctx_dma (req);u32 enc_iv_len =crypto_skcipher_ivsize (tfm);struct fc_context *fctx = &rctx->fctx;struct cpt_request_info *req_info = &rctx->cpt_req;
@@ -335,7 +335,7 @@ static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,static int cvm_enc_dec_init (struct crypto_skcipher *tfm) { -crypto_skcipher_set_reqsize (tfm,sizeof (struct cvm_req_ctx )); +crypto_skcipher_set_reqsize_dma (tfm,sizeof (struct cvm_req_ctx ));return 0 ; }
drivers/crypto/cavium/nitrox/nitrox_aead.c +6 -6
@@ -392,7 +392,7 @@ static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,static int nitrox_rfc4106_set_aead_rctx_sglist (struct aead_request *areq) { -struct nitrox_rfc4106_rctx *rctx =aead_request_ctx (areq); +struct nitrox_rfc4106_rctx *rctx =aead_request_ctx_dma (areq);struct nitrox_aead_rctx *aead_rctx = &rctx->base;unsigned int assoclen = areq->assoclen -GCM_RFC4106_IV_SIZE ;struct scatterlist *sg;
@@ -424,7 +424,7 @@ static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)static void nitrox_rfc4106_callback (void *arg,int err) {struct aead_request *areq = arg; -struct nitrox_rfc4106_rctx *rctx =aead_request_ctx (areq); +struct nitrox_rfc4106_rctx *rctx =aead_request_ctx_dma (areq);struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;free_src_sglist (nkreq);
@@ -441,7 +441,7 @@ static int nitrox_rfc4106_enc(struct aead_request *areq) {struct crypto_aead *aead =crypto_aead_reqtfm (areq);struct nitrox_crypto_ctx *nctx =crypto_aead_ctx (aead); -struct nitrox_rfc4106_rctx *rctx =aead_request_ctx (areq); +struct nitrox_rfc4106_rctx *rctx =aead_request_ctx_dma (areq);struct nitrox_aead_rctx *aead_rctx = &rctx->base;struct se_crypto_request *creq = &aead_rctx->nkreq.creq;int ret;
@@ -472,7 +472,7 @@ static int nitrox_rfc4106_enc(struct aead_request *areq)static int nitrox_rfc4106_dec (struct aead_request *areq) {struct crypto_aead *aead =crypto_aead_reqtfm (areq); -struct nitrox_crypto_ctx *nctx =crypto_aead_ctx (aead); +struct nitrox_crypto_ctx *nctx =crypto_aead_ctx_dma (aead);struct nitrox_rfc4106_rctx *rctx =aead_request_ctx (areq);struct nitrox_aead_rctx *aead_rctx = &rctx->base;struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
@@ -510,8 +510,8 @@ static int nitrox_rfc4106_init(struct crypto_aead *aead)if (ret)return ret; -crypto_aead_set_reqsize (aead,sizeof (struct aead_request ) + -sizeof (struct nitrox_rfc4106_rctx )); +crypto_aead_set_reqsize_dma (aead,sizeof (struct aead_request ) + +sizeof (struct nitrox_rfc4106_rctx ));return 0 ; }
Herbert Xu
maintainerPATCH 2/10 crypto: ccp - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 21 +++++++++++---------- drivers/crypto/ccp/ccp-crypto-aes-galois.c | 12 ++++++------ drivers/crypto/ccp/ccp-crypto-aes-xts.c | 20 +++++++++++--------- drivers/crypto/ccp/ccp-crypto-aes.c | 29 +++++++++++++++-------------- drivers/crypto/ccp/ccp-crypto-des3.c | 17 +++++++++-------- drivers/crypto/ccp/ccp-crypto-main.c | 4 ++-- drivers/crypto/ccp/ccp-crypto-rsa.c | 18 +++++++++--------- drivers/crypto/ccp/ccp-crypto-sha.c | 26 +++++++++++++------------- 8 files changed, 76 insertions(+), 71 deletions(-)
drivers/crypto/ccp/ccp-crypto-aes-cmac.c +11 -10
@@ -25,7 +25,7 @@ static int ccp_aes_cmac_complete(struct crypto_async_request *async_req, {struct ahash_request *req =ahash_request_cast (async_req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx_dma (req);unsigned int digest_size =crypto_ahash_digestsize (tfm);if (ret)
@@ -56,8 +56,8 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, unsigned int final) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ccp_ctx *ctx =crypto_ahash_ctx (tfm); -struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (tfm); +struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx_dma (req);struct scatterlist *sg, *cmac_key_sg =NULL ;unsigned int block_size =crypto_tfm_alg_blocksize (crypto_ahash_tfm (tfm));
@@ -182,7 +182,7 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes,static int ccp_aes_cmac_init (struct ahash_request *req) { -struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx_dma (req);memset (rctx,0 ,sizeof (*rctx));
@@ -219,7 +219,7 @@ static int ccp_aes_cmac_digest(struct ahash_request *req)static int ccp_aes_cmac_export (struct ahash_request *req,void *out) { -struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx_dma (req);struct ccp_aes_cmac_exp_ctx state;/* Don't let anything leak to 'out' */
@@ -238,7 +238,7 @@ static int ccp_aes_cmac_export(struct ahash_request *req, void *out)static int ccp_aes_cmac_import (struct ahash_request *req,const void *in) { -struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_aes_cmac_req_ctx *rctx =ahash_request_ctx_dma (req);struct ccp_aes_cmac_exp_ctx state;/* 'in' may not be aligned so memcpy to local variable */
@@ -256,7 +256,7 @@ static int ccp_aes_cmac_import(struct ahash_request *req, const void *in)static int ccp_aes_cmac_setkey (struct crypto_ahash *tfm,const u8 *key,unsigned int key_len) { -struct ccp_ctx *ctx =crypto_tfm_ctx (crypto_ahash_tfm (tfm)); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (tfm);struct ccp_crypto_ahash_alg *alg =ccp_crypto_ahash_alg (crypto_ahash_tfm (tfm));u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo;
@@ -334,13 +334,14 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,static int ccp_aes_cmac_cra_init (struct crypto_tfm *tfm) { -struct ccp_ctx *ctx =crypto_tfm_ctx (tfm); +struct ccp_ctx *ctx =crypto_tfm_ctx_dma (tfm);struct crypto_ahash *ahash =__crypto_ahash_cast (tfm); ctx->complete = ccp_aes_cmac_complete; ctx->u.aes.key_len =0 ; -crypto_ahash_set_reqsize (ahash,sizeof (struct ccp_aes_cmac_req_ctx )); +crypto_ahash_set_reqsize_dma (ahash, +sizeof (struct ccp_aes_cmac_req_ctx ));return 0 ; }
@@ -382,7 +383,7 @@ int ccp_register_aes_cmac_algs(struct list_head *head)CRYPTO_ALG_KERN_DRIVER_ONLY |CRYPTO_ALG_NEED_FALLBACK ; base->cra_blocksize =AES_BLOCK_SIZE ; - base->cra_ctxsize =sizeof (struct ccp_ctx ); + base->cra_ctxsize =sizeof (struct ccp_ctx ) +crypto_dma_padding (); base->cra_priority =CCP_CRA_PRIORITY ; base->cra_init = ccp_aes_cmac_cra_init; base->cra_module =THIS_MODULE ;
drivers/crypto/ccp/ccp-crypto-aes-galois.c +6 -6
@@ -29,7 +29,7 @@ static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret)static int ccp_aes_gcm_setkey (struct crypto_aead *tfm,const u8 *key,unsigned int key_len) { -struct ccp_ctx *ctx =crypto_aead_ctx (tfm); +struct ccp_ctx *ctx =crypto_aead_ctx_dma (tfm);switch (key_len) {case AES_KEYSIZE_128 :
@@ -76,8 +76,8 @@ static int ccp_aes_gcm_setauthsize(struct crypto_aead *tfm,static int ccp_aes_gcm_crypt (struct aead_request *req,bool encrypt) {struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct ccp_ctx *ctx =crypto_aead_ctx (tfm); -struct ccp_aes_req_ctx *rctx =aead_request_ctx (req); +struct ccp_ctx *ctx =crypto_aead_ctx_dma (tfm); +struct ccp_aes_req_ctx *rctx =aead_request_ctx_dma (req);struct scatterlist *iv_sg =NULL ;unsigned int iv_len =0 ;int i;
@@ -148,12 +148,12 @@ static int ccp_aes_gcm_decrypt(struct aead_request *req)static int ccp_aes_gcm_cra_init (struct crypto_aead *tfm) { -struct ccp_ctx *ctx =crypto_aead_ctx (tfm); +struct ccp_ctx *ctx =crypto_aead_ctx_dma (tfm); ctx->complete = ccp_aes_gcm_complete; ctx->u.aes.key_len =0 ; -crypto_aead_set_reqsize (tfm,sizeof (struct ccp_aes_req_ctx )); +crypto_aead_set_reqsize_dma (tfm,sizeof (struct ccp_aes_req_ctx ));return 0 ; }
@@ -176,7 +176,7 @@ static struct aead_alg ccp_aes_gcm_defaults = {CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, .cra_priority = CCP_CRA_PRIORITY, .cra_exit = ccp_aes_gcm_cra_exit, .cra_module = THIS_MODULE,
drivers/crypto/ccp/ccp-crypto-aes-xts.c +11 -9
@@ -62,7 +62,7 @@ static struct ccp_unit_size_map xts_unit_sizes[] = {static int ccp_aes_xts_complete (struct crypto_async_request *async_req,int ret) {struct skcipher_request *req =skcipher_request_cast (async_req); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);if (ret)return ret;
@@ -75,7 +75,7 @@ static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)static int ccp_aes_xts_setkey (struct crypto_skcipher *tfm,const u8 *key,unsigned int key_len) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);unsigned int ccpversion =ccp_version ();int ret;
@@ -105,8 +105,8 @@ static int ccp_aes_xts_crypt(struct skcipher_request *req, unsigned int encrypt) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);unsigned int ccpversion =ccp_version ();unsigned int fallback =0 ;unsigned int unit;
@@ -196,7 +196,7 @@ static int ccp_aes_xts_decrypt(struct skcipher_request *req)static int ccp_aes_xts_init_tfm (struct crypto_skcipher *tfm) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);struct crypto_skcipher *fallback_tfm; ctx->complete = ccp_aes_xts_complete;
@@ -210,15 +210,16 @@ static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm) } ctx->u.aes.tfm_skcipher = fallback_tfm; -crypto_skcipher_set_reqsize (tfm,sizeof (struct ccp_aes_req_ctx ) + -crypto_skcipher_reqsize (fallback_tfm)); +crypto_skcipher_set_reqsize_dma (tfm, +sizeof (struct ccp_aes_req_ctx ) + +crypto_skcipher_reqsize (fallback_tfm));return 0 ; }static void ccp_aes_xts_exit_tfm (struct crypto_skcipher *tfm) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);crypto_free_skcipher (ctx->u.aes.tfm_skcipher); }
@@ -246,7 +247,8 @@ static int ccp_register_aes_xts_alg(struct list_head *head,CRYPTO_ALG_KERN_DRIVER_ONLY |CRYPTO_ALG_NEED_FALLBACK ; alg->base.cra_blocksize =AES_BLOCK_SIZE ; - alg->base.cra_ctxsize =sizeof (struct ccp_ctx ); + alg->base.cra_ctxsize =sizeof (struct ccp_ctx ) + +crypto_dma_padding (); alg->base.cra_priority =CCP_CRA_PRIORITY ; alg->base.cra_module =THIS_MODULE ;
drivers/crypto/ccp/ccp-crypto-aes.c +15 -14
@@ -22,8 +22,9 @@static int ccp_aes_complete (struct crypto_async_request *async_req,int ret) {struct skcipher_request *req =skcipher_request_cast (async_req); -struct ccp_ctx *ctx =crypto_tfm_ctx (req->base.tfm); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma ( +crypto_skcipher_reqtfm (req)); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);if (ret)return ret;
@@ -38,7 +39,7 @@ static int ccp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int key_len) {struct ccp_crypto_skcipher_alg *alg =ccp_crypto_skcipher_alg (tfm); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);switch (key_len) {case AES_KEYSIZE_128 :
@@ -65,8 +66,8 @@ static int ccp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,static int ccp_aes_crypt (struct skcipher_request *req,bool encrypt) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);struct scatterlist *iv_sg =NULL ;unsigned int iv_len =0 ;
@@ -118,7 +119,7 @@ static int ccp_aes_decrypt(struct skcipher_request *req)static int ccp_aes_init_tfm (struct crypto_skcipher *tfm) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); ctx->complete = ccp_aes_complete; ctx->u.aes.key_len =0 ;
@@ -132,7 +133,7 @@ static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req, int ret) {struct skcipher_request *req =skcipher_request_cast (async_req); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);/* Restore the original pointer */ req->iv = rctx->rfc3686_info;
@@ -143,7 +144,7 @@ static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req,static int ccp_aes_rfc3686_setkey (struct crypto_skcipher *tfm,const u8 *key,unsigned int key_len) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);if (key_len <CTR_RFC3686_NONCE_SIZE )return -EINVAL ;
@@ -157,8 +158,8 @@ static int ccp_aes_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,static int ccp_aes_rfc3686_crypt (struct skcipher_request *req,bool encrypt) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); -struct ccp_aes_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); +struct ccp_aes_req_ctx *rctx =skcipher_request_ctx_dma (req);u8 *iv;/* Initialize the CTR block */
@@ -190,12 +191,12 @@ static int ccp_aes_rfc3686_decrypt(struct skcipher_request *req)static int ccp_aes_rfc3686_init_tfm (struct crypto_skcipher *tfm) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); ctx->complete = ccp_aes_rfc3686_complete; ctx->u.aes.key_len =0 ; -crypto_skcipher_set_reqsize (tfm,sizeof (struct ccp_aes_req_ctx )); +crypto_skcipher_set_reqsize_dma (tfm,sizeof (struct ccp_aes_req_ctx ));return 0 ; }
@@ -213,7 +214,7 @@ static const struct skcipher_alg ccp_aes_defaults = {CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ccp_ctx), + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, .base.cra_priority = CCP_CRA_PRIORITY, .base.cra_module = THIS_MODULE, };
@@ -231,7 +232,7 @@ static const struct skcipher_alg ccp_aes_rfc3686_defaults = {CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .base.cra_blocksize = CTR_RFC3686_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ccp_ctx), + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, .base.cra_priority = CCP_CRA_PRIORITY, .base.cra_module = THIS_MODULE, };
drivers/crypto/ccp/ccp-crypto-des3.c +9 -8
@@ -21,8 +21,9 @@static int ccp_des3_complete (struct crypto_async_request *async_req,int ret) {struct skcipher_request *req =skcipher_request_cast (async_req); -struct ccp_ctx *ctx =crypto_tfm_ctx (req->base.tfm); -struct ccp_des3_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma ( +crypto_skcipher_reqtfm (req)); +struct ccp_des3_req_ctx *rctx =skcipher_request_ctx_dma (req);if (ret)return ret;
@@ -37,7 +38,7 @@ static int ccp_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int key_len) {struct ccp_crypto_skcipher_alg *alg =ccp_crypto_skcipher_alg (tfm); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm);int err; err =verify_skcipher_des3_key (tfm, key);
@@ -60,8 +61,8 @@ static int ccp_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,static int ccp_des3_crypt (struct skcipher_request *req,bool encrypt) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req); -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); -struct ccp_des3_req_ctx *rctx =skcipher_request_ctx (req); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); +struct ccp_des3_req_ctx *rctx =skcipher_request_ctx_dma (req);struct scatterlist *iv_sg =NULL ;unsigned int iv_len =0 ;
@@ -114,12 +115,12 @@ static int ccp_des3_decrypt(struct skcipher_request *req)static int ccp_des3_init_tfm (struct crypto_skcipher *tfm) { -struct ccp_ctx *ctx =crypto_skcipher_ctx (tfm); +struct ccp_ctx *ctx =crypto_skcipher_ctx_dma (tfm); ctx->complete = ccp_des3_complete; ctx->u.des3.key_len =0 ; -crypto_skcipher_set_reqsize (tfm,sizeof (struct ccp_des3_req_ctx )); +crypto_skcipher_set_reqsize_dma (tfm,sizeof (struct ccp_des3_req_ctx ));return 0 ; }
@@ -137,7 +138,7 @@ static const struct skcipher_alg ccp_des3_defaults = {CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ccp_ctx), + .base.cra_ctxsize = sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, .base.cra_priority = CCP_CRA_PRIORITY, .base.cra_module = THIS_MODULE, };
drivers/crypto/ccp/ccp-crypto-main.c +2 -2
@@ -139,7 +139,7 @@ static void ccp_crypto_complete(void *data, int err)struct ccp_crypto_cmd *crypto_cmd = data;struct ccp_crypto_cmd *held, *next, *backlog;struct crypto_async_request *req = crypto_cmd->req; -struct ccp_ctx *ctx =crypto_tfm_ctx (req->tfm); +struct ccp_ctx *ctx =crypto_tfm_ctx_dma (req->tfm);int ret;if (err == -EINPROGRESS ) {
@@ -183,7 +183,7 @@ static void ccp_crypto_complete(void *data, int err)break ;/* Error occurred, report it and get the next entry */ - ctx =crypto_tfm_ctx (held->req->tfm); + ctx =crypto_tfm_ctx_dma (held->req->tfm);if (ctx->complete) ret = ctx->complete (held->req, ret); held->req->complete (held->req, ret);
drivers/crypto/ccp/ccp-crypto-rsa.c +9 -9
@@ -44,7 +44,7 @@ static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen,static int ccp_rsa_complete (struct crypto_async_request *async_req,int ret) {struct akcipher_request *req =akcipher_request_cast (async_req); -struct ccp_rsa_req_ctx *rctx =akcipher_request_ctx (req); +struct ccp_rsa_req_ctx *rctx =akcipher_request_ctx_dma (req);if (ret)return ret;
@@ -56,7 +56,7 @@ static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret)static unsigned int ccp_rsa_maxsize (struct crypto_akcipher *tfm) { -struct ccp_ctx *ctx =akcipher_tfm_ctx (tfm); +struct ccp_ctx *ctx =akcipher_tfm_ctx_dma (tfm);return ctx->u.rsa.n_len; }
@@ -64,8 +64,8 @@ static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm)static int ccp_rsa_crypt (struct akcipher_request *req,bool encrypt) {struct crypto_akcipher *tfm =crypto_akcipher_reqtfm (req); -struct ccp_ctx *ctx =akcipher_tfm_ctx (tfm); -struct ccp_rsa_req_ctx *rctx =akcipher_request_ctx (req); +struct ccp_ctx *ctx =akcipher_tfm_ctx_dma (tfm); +struct ccp_rsa_req_ctx *rctx =akcipher_request_ctx_dma (req);int ret =0 ;memset (&rctx->cmd,0 ,sizeof (rctx->cmd));
@@ -126,7 +126,7 @@ static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx)static int ccp_rsa_setkey (struct crypto_akcipher *tfm,const void *key,unsigned int keylen,bool private) { -struct ccp_ctx *ctx =akcipher_tfm_ctx (tfm); +struct ccp_ctx *ctx =akcipher_tfm_ctx_dma (tfm);struct rsa_key raw_key;int ret;
@@ -192,9 +192,9 @@ static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,static int ccp_rsa_init_tfm (struct crypto_akcipher *tfm) { -struct ccp_ctx *ctx =akcipher_tfm_ctx (tfm); +struct ccp_ctx *ctx =akcipher_tfm_ctx_dma (tfm); -akcipher_set_reqsize (tfm,sizeof (struct ccp_rsa_req_ctx )); +akcipher_set_reqsize_dma (tfm,sizeof (struct ccp_rsa_req_ctx )); ctx->complete = ccp_rsa_complete;return 0 ;
@@ -202,7 +202,7 @@ static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm)static void ccp_rsa_exit_tfm (struct crypto_akcipher *tfm) { -struct ccp_ctx *ctx =crypto_tfm_ctx (&tfm->base); +struct ccp_ctx *ctx =akcipher_tfm_ctx_dma (tfm);ccp_rsa_free_key_bufs (ctx); }
@@ -220,7 +220,7 @@ static struct akcipher_alg ccp_rsa_defaults = { .cra_driver_name = "rsa-ccp", .cra_priority = CCP_CRA_PRIORITY, .cra_module = THIS_MODULE, - .cra_ctxsize = 2 * sizeof(struct ccp_ctx), + .cra_ctxsize = 2 * sizeof(struct ccp_ctx) + CRYPTO_DMA_PADDING, }, };
drivers/crypto/ccp/ccp-crypto-sha.c +13 -13
@@ -28,7 +28,7 @@ static int ccp_sha_complete(struct crypto_async_request *async_req, int ret) {struct ahash_request *req =ahash_request_cast (async_req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ccp_sha_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_sha_req_ctx *rctx =ahash_request_ctx_dma (req);unsigned int digest_size =crypto_ahash_digestsize (tfm);if (ret)
@@ -59,8 +59,8 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, unsigned int final) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ccp_ctx *ctx =crypto_ahash_ctx (tfm); -struct ccp_sha_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (tfm); +struct ccp_sha_req_ctx *rctx =ahash_request_ctx_dma (req);struct scatterlist *sg;unsigned int block_size =crypto_tfm_alg_blocksize (crypto_ahash_tfm (tfm));
@@ -182,8 +182,8 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes,static int ccp_sha_init (struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ccp_ctx *ctx =crypto_ahash_ctx (tfm); -struct ccp_sha_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (tfm); +struct ccp_sha_req_ctx *rctx =ahash_request_ctx_dma (req);struct ccp_crypto_ahash_alg *alg =ccp_crypto_ahash_alg (crypto_ahash_tfm (tfm)); unsigned int block_size =
@@ -231,7 +231,7 @@ static int ccp_sha_digest(struct ahash_request *req)static int ccp_sha_export (struct ahash_request *req,void *out) { -struct ccp_sha_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_sha_req_ctx *rctx =ahash_request_ctx_dma (req);struct ccp_sha_exp_ctx state;/* Don't let anything leak to 'out' */
@@ -252,7 +252,7 @@ static int ccp_sha_export(struct ahash_request *req, void *out)static int ccp_sha_import (struct ahash_request *req,const void *in) { -struct ccp_sha_req_ctx *rctx =ahash_request_ctx (req); +struct ccp_sha_req_ctx *rctx =ahash_request_ctx_dma (req);struct ccp_sha_exp_ctx state;/* 'in' may not be aligned so memcpy to local variable */
@@ -272,7 +272,7 @@ static int ccp_sha_import(struct ahash_request *req, const void *in)static int ccp_sha_setkey (struct crypto_ahash *tfm,const u8 *key,unsigned int key_len) { -struct ccp_ctx *ctx =crypto_tfm_ctx (crypto_ahash_tfm (tfm)); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (tfm);struct crypto_shash *shash = ctx->u.sha.hmac_tfm;unsigned int block_size =crypto_shash_blocksize (shash);unsigned int digest_size =crypto_shash_digestsize (shash);
@@ -313,13 +313,13 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,static int ccp_sha_cra_init (struct crypto_tfm *tfm) { -struct ccp_ctx *ctx =crypto_tfm_ctx (tfm);struct crypto_ahash *ahash =__crypto_ahash_cast (tfm); +struct ccp_ctx *ctx =crypto_ahash_ctx_dma (ahash); ctx->complete = ccp_sha_complete; ctx->u.sha.key_len =0 ; -crypto_ahash_set_reqsize (ahash,sizeof (struct ccp_sha_req_ctx )); +crypto_ahash_set_reqsize_dma (ahash,sizeof (struct ccp_sha_req_ctx ));return 0 ; }
@@ -330,7 +330,7 @@ static void ccp_sha_cra_exit(struct crypto_tfm *tfm)static int ccp_hmac_sha_cra_init (struct crypto_tfm *tfm) { -struct ccp_ctx *ctx =crypto_tfm_ctx (tfm); +struct ccp_ctx *ctx =crypto_tfm_ctx_dma (tfm);struct ccp_crypto_ahash_alg *alg =ccp_crypto_ahash_alg (tfm);struct crypto_shash *hmac_tfm;
@@ -348,7 +348,7 @@ static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm)static void ccp_hmac_sha_cra_exit (struct crypto_tfm *tfm) { -struct ccp_ctx *ctx =crypto_tfm_ctx (tfm); +struct ccp_ctx *ctx =crypto_tfm_ctx_dma (tfm);if (ctx->u.sha.hmac_tfm)crypto_free_shash (ctx->u.sha.hmac_tfm);
@@ -492,7 +492,7 @@ static int ccp_register_sha_alg(struct list_head *head,CRYPTO_ALG_KERN_DRIVER_ONLY |CRYPTO_ALG_NEED_FALLBACK ; base->cra_blocksize = def->block_size; - base->cra_ctxsize =sizeof (struct ccp_ctx ); + base->cra_ctxsize =sizeof (struct ccp_ctx ) +crypto_dma_padding (); base->cra_priority =CCP_CRA_PRIORITY ; base->cra_init = ccp_sha_cra_init; base->cra_exit = ccp_sha_cra_exit;
Herbert Xu
maintainerPATCH 3/10 crypto: ccree - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/ccree/cc_aead.c | 62 ++++++++++++------------- drivers/crypto/ccree/cc_buffer_mgr.c | 18 +++---- drivers/crypto/ccree/cc_hash.c | 86 +++++++++++++++++------------------ 3 files changed, 83 insertions(+), 83 deletions(-)
drivers/crypto/ccree/cc_aead.c +31 -31
@@ -138,7 +138,7 @@ static int cc_aead_init(struct crypto_aead *tfm) ctx->flow_mode = cc_alg->flow_mode; ctx->auth_mode = cc_alg->auth_mode; ctx->drvdata = cc_alg->drvdata; -crypto_aead_set_reqsize (tfm,sizeof (struct aead_req_ctx )); +crypto_aead_set_reqsize_dma (tfm,sizeof (struct aead_req_ctx ));/* Allocate key buffer, cache line aligned */ ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
@@ -208,7 +208,7 @@ static int cc_aead_init(struct crypto_aead *tfm)static void cc_aead_complete (struct device *dev,void *cc_req,int err) {struct aead_request *areq = (struct aead_request *)cc_req; -struct aead_req_ctx *areq_ctx =aead_request_ctx (areq); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (areq);struct crypto_aead *tfm =crypto_aead_reqtfm (cc_req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm);
@@ -723,7 +723,7 @@ static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, {struct crypto_aead *tfm =crypto_aead_reqtfm (areq);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *areq_ctx =aead_request_ctx (areq); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (areq);enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;unsigned int idx = *seq_size;struct device *dev =drvdata_to_dev (ctx->drvdata);
@@ -762,7 +762,7 @@ static void cc_proc_authen_desc(struct aead_request *areq,struct cc_hw_desc desc[], unsigned int *seq_size, int direct) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq);enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;unsigned int idx = *seq_size;struct crypto_aead *tfm =crypto_aead_reqtfm (areq);
@@ -827,7 +827,7 @@ static void cc_proc_cipher_desc(struct aead_request *areq, unsigned int *seq_size) {unsigned int idx = *seq_size; -struct aead_req_ctx *areq_ctx =aead_request_ctx (areq); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (areq);enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;struct crypto_aead *tfm =crypto_aead_reqtfm (areq);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm);
@@ -873,7 +873,7 @@ static void cc_proc_digest_desc(struct aead_request *req, {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int idx = *seq_size;unsigned int hash_mode = (ctx->auth_mode ==DRV_HASH_SHA1 ) ?DRV_HASH_HW_SHA1 :DRV_HASH_HW_SHA256 ;
@@ -923,7 +923,7 @@ static void cc_set_cipher_desc(struct aead_request *req, {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int hw_iv_size = req_ctx->hw_iv_size;unsigned int idx = *seq_size;int direct = req_ctx->gen_ctx.op_type;
@@ -965,7 +965,7 @@ static void cc_set_cipher_desc(struct aead_request *req,static void cc_proc_cipher (struct aead_request *req,struct cc_hw_desc desc[],unsigned int *seq_size,unsigned int data_flow_mode) { -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);int direct = req_ctx->gen_ctx.op_type;unsigned int idx = *seq_size;
@@ -1082,7 +1082,7 @@ static void cc_proc_header_desc(struct aead_request *req,struct cc_hw_desc desc[], unsigned int *seq_size) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(req); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);unsigned int idx = *seq_size;/* Hash associated data */
@@ -1158,7 +1158,7 @@ static void cc_proc_scheme_desc(struct aead_request *req,static void cc_mlli_to_sram (struct aead_request *req,struct cc_hw_desc desc[],unsigned int *seq_size) { -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm);struct device *dev =drvdata_to_dev (ctx->drvdata);
@@ -1212,7 +1212,7 @@ static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);int direct = req_ctx->gen_ctx.op_type;unsigned int data_flow_mode = cc_get_data_flow(direct, ctx->flow_mode,
@@ -1265,7 +1265,7 @@ cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);int direct = req_ctx->gen_ctx.op_type;unsigned int data_flow_mode = cc_get_data_flow(direct, ctx->flow_mode,
@@ -1312,7 +1312,7 @@ static int validate_data_size(struct cc_aead_ctx *ctx,enum drv_crypto_direction direct, struct aead_request *req) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(req); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);struct device *dev =drvdata_to_dev (ctx->drvdata);unsigned int assoclen = areq_ctx->assoclen; unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
@@ -1411,7 +1411,7 @@ static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int idx = *seq_size;unsigned int cipher_flow_mode;dma_addr_t mac_result;
@@ -1533,7 +1533,7 @@ static int config_ccm_adata(struct aead_request *req)struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm);struct device *dev =drvdata_to_dev (ctx->drvdata); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);//unsigned int size_of_a = 0, rem_a_size = 0; unsigned int lp = req->iv[0 ];/* Note: The code assume that req->iv[0] already contains the value
@@ -1591,7 +1591,7 @@ static void cc_proc_rfc4309_ccm(struct aead_request *req) {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);/* L' */ memset (areq_ctx->ctr_iv,0 ,AES_BLOCK_SIZE );
@@ -1615,7 +1615,7 @@ static void cc_set_ghash_desc(struct aead_request *req, {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int idx = *seq_size;/* load key to AES*/
@@ -1693,7 +1693,7 @@ static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int idx = *seq_size;/* load key to AES*/
@@ -1730,7 +1730,7 @@ static void cc_proc_gcm_result(struct aead_request *req, {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);dma_addr_t mac_result;unsigned int idx = *seq_size;
@@ -1792,7 +1792,7 @@ static void cc_proc_gcm_result(struct aead_request *req,static int cc_gcm (struct aead_request *req,struct cc_hw_desc desc[],unsigned int *seq_size) { -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);unsigned int cipher_flow_mode;//in RFC4543 no data to encrypt. just copy data from src to dest.
@@ -1830,7 +1830,7 @@ static int config_gcm_context(struct aead_request *req) {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *req_ctx =aead_request_ctx (req); +struct aead_req_ctx *req_ctx =aead_request_ctx_dma (req);struct device *dev =drvdata_to_dev (ctx->drvdata); unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
@@ -1879,7 +1879,7 @@ static void cc_proc_rfc4_gcm(struct aead_request *req) {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);memcpy (areq_ctx->ctr_iv +GCM_BLOCK_RFC4_NONCE_OFFSET , ctx->ctr_nonce,GCM_BLOCK_RFC4_NONCE_SIZE );
@@ -1896,7 +1896,7 @@ static int cc_proc_aead(struct aead_request *req,struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ ];struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct cc_aead_ctx *ctx =crypto_aead_ctx (tfm); -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);struct device *dev =drvdata_to_dev (ctx->drvdata);struct cc_crypto_req cc_req = {};
@@ -2019,7 +2019,7 @@ static int cc_proc_aead(struct aead_request *req,static int cc_aead_encrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc;memset (areq_ctx,0 ,sizeof (*areq_ctx));
@@ -2039,7 +2039,7 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req) {/* Very similar to cc_aead_encrypt() above. */ -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
@@ -2063,7 +2063,7 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req)static int cc_aead_decrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc;memset (areq_ctx,0 ,sizeof (*areq_ctx));
@@ -2081,7 +2081,7 @@ static int cc_aead_decrypt(struct aead_request *req)static int cc_rfc4309_ccm_decrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
@@ -2193,7 +2193,7 @@ static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,static int cc_rfc4106_gcm_encrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
@@ -2217,7 +2217,7 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req)static int cc_rfc4543_gcm_encrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
@@ -2244,7 +2244,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)static int cc_rfc4106_gcm_decrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
@@ -2268,7 +2268,7 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req)static int cc_rfc4543_gcm_decrypt (struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);int rc; rc =crypto_ipsec_check_assoclen (req->assoclen);
drivers/crypto/ccree/cc_buffer_mgr.c +9 -9
@@ -52,7 +52,7 @@ static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type)static void cc_copy_mac (struct device *dev,struct aead_request *req,enum cc_sg_cpy_direct dir) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);u32 skip = req->assoclen + req->cryptlen; cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src,
@@ -456,7 +456,7 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,void cc_unmap_aead_request (struct device *dev,struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);unsigned int hw_iv_size = areq_ctx->hw_iv_size;struct cc_drvdata *drvdata =dev_get_drvdata (dev);int src_direction = (req->src != req->dst ?DMA_TO_DEVICE :DMA_BIDIRECTIONAL );
@@ -546,7 +546,7 @@ static int cc_aead_chain_iv(struct cc_drvdata *drvdata,struct buffer_array *sg_data, bool is_last, bool do_chain) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(req); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);unsigned int hw_iv_size = areq_ctx->hw_iv_size;struct device *dev =drvdata_to_dev (drvdata);gfp_t flags =cc_gfp_flags (&req->base);
@@ -586,7 +586,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,struct buffer_array *sg_data, bool is_last, bool do_chain) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(req); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);int rc =0 ;int mapped_nents =0 ;struct device *dev =drvdata_to_dev (drvdata);
@@ -652,7 +652,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,static void cc_prepare_aead_data_dlli (struct aead_request *req,u32 *src_last_bytes,u32 *dst_last_bytes) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;unsigned int authsize = areq_ctx->req_authsize;struct scatterlist *sg;
@@ -678,7 +678,7 @@ static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, u32 *src_last_bytes, u32 *dst_last_bytes, bool is_last_table) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;unsigned int authsize = areq_ctx->req_authsize;struct device *dev =drvdata_to_dev (drvdata);
@@ -790,7 +790,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata,struct buffer_array *sg_data, bool is_last_table, bool do_chain) { - struct aead_req_ctx *areq_ctx = aead_request_ctx(req); + struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req);struct device *dev =drvdata_to_dev (drvdata);enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;unsigned int authsize = areq_ctx->req_authsize;
@@ -895,7 +895,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata,static void cc_update_aead_mlli_nents (struct cc_drvdata *drvdata,struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);u32 curr_mlli_size =0 ;if (areq_ctx->assoc_buff_type ==CC_DMA_BUF_MLLI ) {
@@ -945,7 +945,7 @@ static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,int cc_map_aead_request (struct cc_drvdata *drvdata,struct aead_request *req) { -struct aead_req_ctx *areq_ctx =aead_request_ctx (req); +struct aead_req_ctx *areq_ctx =aead_request_ctx_dma (req);struct mlli_params *mlli_params = &areq_ctx->mlli_params;struct device *dev =drvdata_to_dev (drvdata);struct buffer_array sg_data;
drivers/crypto/ccree/cc_hash.c +43 -43
252 lines not on this page · raw message
Herbert Xu
maintainerPATCH 4/10 crypto: chelsio - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/chelsio/chcr_algo.c | 43 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-)
drivers/crypto/chelsio/chcr_algo.c +22 -21
@@ -210,7 +210,7 @@ static inline int chcr_handle_aead_resp(struct aead_request *req, unsigned char *input, int err) { -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_dev *dev =a_ctx (tfm)->dev;
@@ -718,7 +718,7 @@ static inline int get_qidxs(struct crypto_async_request *req, {struct aead_request *aead_req = container_of(req, struct aead_request, base); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (aead_req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (aead_req); *txqidx = reqctx->txqidx; *rxqidx = reqctx->rxqidx;break ;
@@ -2362,7 +2362,7 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)inline void chcr_aead_common_exit (struct aead_request *req) { -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct uld_ctx *u_ctx =ULD_CTX (a_ctx (tfm));
@@ -2373,7 +2373,7 @@ static int chcr_aead_common_init(struct aead_request *req) {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_aead_ctx *aeadctx =AEAD_CTX (a_ctx (tfm)); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);unsigned int authsize =crypto_aead_authsize (tfm);int error = -EINVAL ;
@@ -2417,7 +2417,7 @@ static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_aead_ctx *aeadctx =AEAD_CTX (a_ctx (tfm)); -struct aead_request *subreq =aead_request_ctx (req); +struct aead_request *subreq =aead_request_ctx_dma (req);aead_request_set_tfm (subreq, aeadctx->sw_cipher); aead_request_set_callback(subreq, req->base.flags,
@@ -2438,7 +2438,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,struct uld_ctx *u_ctx =ULD_CTX (ctx);struct chcr_aead_ctx *aeadctx =AEAD_CTX (ctx);struct chcr_authenc_ctx *actx =AUTHENC_CTX (aeadctx); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct sk_buff *skb =NULL ;struct chcr_wr *chcr_req;struct cpl_rx_phys_dsgl *phys_cpl;
@@ -2576,7 +2576,7 @@ int chcr_aead_dma_map(struct device *dev, unsigned short op_type) {int error; -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);unsigned int authsize =crypto_aead_authsize (tfm);int src_len, dst_len;
@@ -2637,7 +2637,7 @@ void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req, unsigned short op_type) { -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);unsigned int authsize =crypto_aead_authsize (tfm);int src_len, dst_len;
@@ -2678,7 +2678,7 @@ void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx) {struct ulptx_walk ulp_walk; -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);if (reqctx->imm) {u8 *buf = (u8 *)ulptx;
@@ -2704,7 +2704,7 @@ void chcr_add_aead_dst_ent(struct aead_request *req, struct cpl_rx_phys_dsgl *phys_cpl, unsigned short qid) { -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct dsgl_walk dsgl_walk;unsigned int authsize =crypto_aead_authsize (tfm);
@@ -2894,7 +2894,7 @@ static int generate_b0(struct aead_request *req, u8 *ivptr,unsigned int l, lp, m;int rc;struct crypto_aead *aead =crypto_aead_reqtfm (req); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);u8 *b0 = reqctx->scratch_pad; m =crypto_aead_authsize (aead);
@@ -2932,7 +2932,7 @@ static int ccm_format_packet(struct aead_request *req, unsigned short op_type, unsigned int assoclen) { -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_aead_ctx *aeadctx =AEAD_CTX (a_ctx (tfm));int rc =0 ;
@@ -2963,7 +2963,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,struct chcr_context *ctx =a_ctx (tfm);struct uld_ctx *u_ctx =ULD_CTX (ctx);struct chcr_aead_ctx *aeadctx =AEAD_CTX (ctx); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);unsigned int cipher_mode =CHCR_SCMD_CIPHER_MODE_AES_CCM ;unsigned int mac_mode =CHCR_SCMD_AUTH_MODE_CBCMAC ;unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
@@ -3036,7 +3036,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, {struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_aead_ctx *aeadctx =AEAD_CTX (a_ctx (tfm)); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct sk_buff *skb =NULL ;struct chcr_wr *chcr_req;struct cpl_rx_phys_dsgl *phys_cpl;
@@ -3135,7 +3135,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,struct chcr_context *ctx =a_ctx (tfm);struct uld_ctx *u_ctx =ULD_CTX (ctx);struct chcr_aead_ctx *aeadctx =AEAD_CTX (ctx); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct sk_buff *skb =NULL ;struct chcr_wr *chcr_req;struct cpl_rx_phys_dsgl *phys_cpl;
@@ -3255,9 +3255,10 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) CRYPTO_ALG_ASYNC);if (IS_ERR (aeadctx->sw_cipher))return PTR_ERR (aeadctx->sw_cipher); -crypto_aead_set_reqsize (tfm,max (sizeof (struct chcr_aead_reqctx ), -sizeof (struct aead_request ) + -crypto_aead_reqsize (aeadctx->sw_cipher))); +crypto_aead_set_reqsize_dma ( + tfm,max (sizeof (struct chcr_aead_reqctx ), +sizeof (struct aead_request ) + +crypto_aead_reqsize (aeadctx->sw_cipher)));return chcr_device_init (a_ctx (tfm)); }
@@ -3735,7 +3736,7 @@ static int chcr_aead_op(struct aead_request *req, create_wr_t create_wr_fn) {struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct chcr_context *ctx =a_ctx (tfm);struct uld_ctx *u_ctx =ULD_CTX (ctx);struct sk_buff *skb;
@@ -3785,7 +3786,7 @@ static int chcr_aead_op(struct aead_request *req,static int chcr_aead_encrypt (struct aead_request *req) {struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);struct chcr_context *ctx =a_ctx (tfm);unsigned int cpu;
@@ -3816,7 +3817,7 @@ static int chcr_aead_decrypt(struct aead_request *req)struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct chcr_context *ctx =a_ctx (tfm);struct chcr_aead_ctx *aeadctx =AEAD_CTX (ctx); -struct chcr_aead_reqctx *reqctx =aead_request_ctx (req); +struct chcr_aead_reqctx *reqctx =aead_request_ctx_dma (req);int size;unsigned int cpu;
Herbert Xu
maintainerPATCH 5/10 crypto: hisilicon/hpre - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/hisilicon/hpre/hpre_crypto.c | 40 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-)
drivers/crypto/hisilicon/hpre/hpre_crypto.c +25 -15
@@ -147,6 +147,16 @@ struct hpre_asym_request {struct timespec64 req_time; }; +static inline unsigned int hpre_align_sz (void ) +{ +return ((crypto_dma_align () -1 ) | (HPRE_ALIGN_SZ -1 )) +1 ; +} + +static inline unsigned int hpre_align_pd (void ) +{ +return (hpre_align_sz () -1 ) & ~(crypto_tfm_ctx_alignment () -1 ); +} +static int hpre_alloc_req_id (struct hpre_ctx *ctx) {unsigned long flags;
@@ -517,7 +527,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) } tmp =akcipher_request_ctx (akreq); - h_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); + h_req =PTR_ALIGN (tmp,hpre_align_sz ()); h_req->cb = hpre_rsa_cb; h_req->areq.rsa = akreq; msg = &h_req->req;
@@ -531,7 +541,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) } tmp =kpp_request_ctx (kreq); - h_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); + h_req =PTR_ALIGN (tmp,hpre_align_sz ()); h_req->cb = hpre_dh_cb; h_req->areq.dh = kreq; msg = &h_req->req;
@@ -582,7 +592,7 @@ static int hpre_dh_compute_value(struct kpp_request *req)struct crypto_kpp *tfm =crypto_kpp_reqtfm (req);struct hpre_ctx *ctx =kpp_tfm_ctx (tfm);void *tmp =kpp_request_ctx (req); -struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); +struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,hpre_align_sz ());struct hpre_sqe *msg = &hpre_req->req;int ret;
@@ -740,7 +750,7 @@ static int hpre_dh_init_tfm(struct crypto_kpp *tfm) {struct hpre_ctx *ctx =kpp_tfm_ctx (tfm); -kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +HPRE_ALIGN_SZ ); +kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +hpre_align_pd ());return hpre_ctx_init (ctx,HPRE_V2_ALG_TYPE ); }
@@ -785,7 +795,7 @@ static int hpre_rsa_enc(struct akcipher_request *req)struct crypto_akcipher *tfm =crypto_akcipher_reqtfm (req);struct hpre_ctx *ctx =akcipher_tfm_ctx (tfm);void *tmp =akcipher_request_ctx (req); -struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); +struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,hpre_align_sz ());struct hpre_sqe *msg = &hpre_req->req;int ret;
@@ -833,7 +843,7 @@ static int hpre_rsa_dec(struct akcipher_request *req)struct crypto_akcipher *tfm =crypto_akcipher_reqtfm (req);struct hpre_ctx *ctx =akcipher_tfm_ctx (tfm);void *tmp =akcipher_request_ctx (req); -struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); +struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,hpre_align_sz ());struct hpre_sqe *msg = &hpre_req->req;int ret;
@@ -1168,7 +1178,7 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) }akcipher_set_reqsize (tfm,sizeof (struct hpre_asym_request ) + -HPRE_ALIGN_SZ ); +hpre_align_pd ()); ret =hpre_ctx_init (ctx,HPRE_V2_ALG_TYPE );if (ret)
@@ -1490,7 +1500,7 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, } tmp =kpp_request_ctx (req); - h_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); + h_req =PTR_ALIGN (tmp,hpre_align_sz ()); h_req->cb = hpre_ecdh_cb; h_req->areq.ecdh = req; msg = &h_req->req;
@@ -1571,7 +1581,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)struct hpre_ctx *ctx =kpp_tfm_ctx (tfm);struct device *dev = ctx->dev;void *tmp =kpp_request_ctx (req); -struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); +struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,hpre_align_sz ());struct hpre_sqe *msg = &hpre_req->req;int ret;
@@ -1622,7 +1632,7 @@ static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) ctx->curve_id =ECC_CURVE_NIST_P192 ; -kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +HPRE_ALIGN_SZ ); +kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +hpre_align_pd ());return hpre_ctx_init (ctx,HPRE_V3_ECC_ALG_TYPE ); }
@@ -1633,7 +1643,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ctx->curve_id =ECC_CURVE_NIST_P256 ; -kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +HPRE_ALIGN_SZ ); +kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +hpre_align_pd ());return hpre_ctx_init (ctx,HPRE_V3_ECC_ALG_TYPE ); }
@@ -1644,7 +1654,7 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ctx->curve_id =ECC_CURVE_NIST_P384 ; -kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +HPRE_ALIGN_SZ ); +kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +hpre_align_pd ());return hpre_ctx_init (ctx,HPRE_V3_ECC_ALG_TYPE ); }
@@ -1802,7 +1812,7 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, } tmp =kpp_request_ctx (req); - h_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); + h_req =PTR_ALIGN (tmp,hpre_align_sz ()); h_req->cb = hpre_curve25519_cb; h_req->areq.curve25519 = req; msg = &h_req->req;
@@ -1923,7 +1933,7 @@ static int hpre_curve25519_compute_value(struct kpp_request *req)struct hpre_ctx *ctx =kpp_tfm_ctx (tfm);struct device *dev = ctx->dev;void *tmp =kpp_request_ctx (req); -struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,HPRE_ALIGN_SZ ); +struct hpre_asym_request *hpre_req =PTR_ALIGN (tmp,hpre_align_sz ());struct hpre_sqe *msg = &hpre_req->req;int ret;
@@ -1972,7 +1982,7 @@ static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) {struct hpre_ctx *ctx =kpp_tfm_ctx (tfm); -kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +HPRE_ALIGN_SZ ); +kpp_set_reqsize (tfm,sizeof (struct hpre_asym_request ) +hpre_align_pd ());return hpre_ctx_init (ctx,HPRE_V3_ECC_ALG_TYPE ); }
Herbert Xu
maintainerPATCH 6/10 crypto: safexcel - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/inside-secure/safexcel_hash.c | 99 +++++++++++++-------------- 1 file changed, 50 insertions(+), 49 deletions(-)
drivers/crypto/inside-secure/safexcel_hash.c +50 -49
@@ -231,7 +231,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,struct safexcel_result_desc *rdesc;struct ahash_request *areq =ahash_request_cast (async);struct crypto_ahash *ahash =crypto_ahash_reqtfm (areq); -struct safexcel_ahash_req *sreq =ahash_request_ctx (areq); +struct safexcel_ahash_req *sreq =ahash_request_ctx_dma (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (ahash);u64 cache_len;
@@ -312,7 +312,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, int *commands, int *results) { struct ahash_request *areq = ahash_request_cast(async); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq));struct safexcel_crypto_priv *priv = ctx->base.priv;struct safexcel_command_desc *cdesc, *first_cdesc =NULL ;
@@ -569,7 +569,7 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, bool *should_complete, int *ret) { struct ahash_request *areq = ahash_request_cast(async); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);int err;BUG_ON (!(priv->flags &EIP197_TRC_CACHE ) && req->needs_inv);
@@ -608,7 +608,7 @@ static int safexcel_ahash_send(struct crypto_async_request *async, int ring, int *commands, int *results) { struct ahash_request *areq = ahash_request_cast(async); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);int ret;if (req->needs_inv)
@@ -624,7 +624,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)struct safexcel_ahash_ctx *ctx =crypto_tfm_ctx (tfm);struct safexcel_crypto_priv *priv = ctx->base.priv;EIP197_REQUEST_ON_STACK (req, ahash,EIP197_AHASH_REQ_SIZE ); -struct safexcel_ahash_req *rctx =ahash_request_ctx (req); +struct safexcel_ahash_req *rctx =ahash_request_ctx_dma (req);struct safexcel_inv_result result = {};int ring = ctx->base.ring;
@@ -663,7 +663,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)*/ static int safexcel_ahash_cache (struct ahash_request *areq) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);u64 cache_len;/* cache_len: everything accepted by the driver but not sent yet,
@@ -689,7 +689,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq)static int safexcel_ahash_enqueue (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);struct safexcel_crypto_priv *priv = ctx->base.priv;int ret, ring;
@@ -741,7 +741,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)static int safexcel_ahash_update (struct ahash_request *areq) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);int ret;/* If the request is 0 length, do nothing */
@@ -766,7 +766,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)static int safexcel_ahash_final (struct ahash_request *areq) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); req->finish = true;
@@ -870,7 +870,7 @@ static int safexcel_ahash_final(struct ahash_request *areq)static int safexcel_ahash_finup (struct ahash_request *areq) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq); req->finish = true;
@@ -880,7 +880,7 @@ static int safexcel_ahash_finup(struct ahash_request *areq)static int safexcel_ahash_export (struct ahash_request *areq,void *out) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);struct safexcel_ahash_export_state *export = out; export->len = req->len;
@@ -896,7 +896,7 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)static int safexcel_ahash_import (struct ahash_request *areq,const void *in) { -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);const struct safexcel_ahash_export_state *export = in;int ret;
@@ -927,15 +927,15 @@ static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) ctx->base.handle_result = safexcel_handle_result; ctx->fb_do_setkey = false; -crypto_ahash_set_reqsize (__crypto_ahash_cast (tfm), -sizeof (struct safexcel_ahash_req )); +crypto_ahash_set_reqsize_dma (__crypto_ahash_cast (tfm), +sizeof (struct safexcel_ahash_req ));return 0 ; }static int safexcel_sha1_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1012,7 +1012,7 @@ struct safexcel_alg_template safexcel_alg_sha1 = {static int safexcel_hmac_sha1_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1124,7 +1124,7 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq,if (ret)return ret; - req =ahash_request_ctx (areq); + req =ahash_request_ctx_dma (areq); req->hmac = true; req->last_req = true;
@@ -1264,7 +1264,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha1 = {static int safexcel_sha256_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1321,7 +1321,7 @@ struct safexcel_alg_template safexcel_alg_sha256 = {static int safexcel_sha224_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1385,7 +1385,7 @@ static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,static int safexcel_hmac_sha224_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1457,7 +1457,7 @@ static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,static int safexcel_hmac_sha256_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1522,7 +1522,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha256 = {static int safexcel_sha512_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1579,7 +1579,7 @@ struct safexcel_alg_template safexcel_alg_sha512 = {static int safexcel_sha384_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1643,7 +1643,7 @@ static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,static int safexcel_hmac_sha512_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1715,7 +1715,7 @@ static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,static int safexcel_hmac_sha384_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1780,7 +1780,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha384 = {static int safexcel_md5_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1837,7 +1837,7 @@ struct safexcel_alg_template safexcel_alg_md5 = {static int safexcel_hmac_md5_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1920,7 +1920,7 @@ static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)static int safexcel_crc32_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -1992,7 +1992,7 @@ struct safexcel_alg_template safexcel_alg_crc32 = {static int safexcel_cbcmac_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2252,7 +2252,7 @@ struct safexcel_alg_template safexcel_alg_cmac = {static int safexcel_sm3_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2316,7 +2316,7 @@ static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,static int safexcel_hmac_sm3_init (struct ahash_request *areq) {struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (crypto_ahash_reqtfm (areq)); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2382,7 +2382,7 @@ static int safexcel_sha3_224_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2400,7 +2400,7 @@ static int safexcel_sha3_fbcheck(struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req);int ret =0 ;if (ctx->do_fallback) {
@@ -2437,7 +2437,7 @@ static int safexcel_sha3_update(struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback = true;return safexcel_sha3_fbcheck (req) ?:crypto_ahash_update (subreq);
@@ -2447,7 +2447,7 @@ static int safexcel_sha3_final(struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback = true;return safexcel_sha3_fbcheck (req) ?:crypto_ahash_final (subreq);
@@ -2457,7 +2457,7 @@ static int safexcel_sha3_finup(struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback |= !req->nbytes;if (ctx->do_fallback)
@@ -2472,7 +2472,7 @@ static int safexcel_sha3_digest_fallback(struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback = true; ctx->fb_init_done = false;
@@ -2492,7 +2492,7 @@ static int safexcel_sha3_export(struct ahash_request *req, void *out) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback = true;return safexcel_sha3_fbcheck (req) ?:crypto_ahash_export (subreq, out);
@@ -2502,7 +2502,7 @@ static int safexcel_sha3_import(struct ahash_request *req, const void *in) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct ahash_request *subreq =ahash_request_ctx (req); +struct ahash_request *subreq =ahash_request_ctx_dma (req); ctx->do_fallback = true;return safexcel_sha3_fbcheck (req) ?:crypto_ahash_import (subreq, in);
@@ -2526,9 +2526,10 @@ static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)/* Update statesize from fallback algorithm! */ crypto_hash_alg_common (ahash)->statesize =crypto_ahash_statesize (ctx->fback); -crypto_ahash_set_reqsize (ahash,max (sizeof (struct safexcel_ahash_req ), -sizeof (struct ahash_request ) + -crypto_ahash_reqsize (ctx->fback))); +crypto_ahash_set_reqsize_dma ( + ahash,max (sizeof (struct safexcel_ahash_req ), +sizeof (struct ahash_request ) + +crypto_ahash_reqsize (ctx->fback)));return 0 ; }
@@ -2575,7 +2576,7 @@ static int safexcel_sha3_256_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2633,7 +2634,7 @@ static int safexcel_sha3_384_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2691,7 +2692,7 @@ static int safexcel_sha3_512_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2841,7 +2842,7 @@ static int safexcel_hmac_sha3_224_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2912,7 +2913,7 @@ static int safexcel_hmac_sha3_256_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -2983,7 +2984,7 @@ static int safexcel_hmac_sha3_384_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
@@ -3054,7 +3055,7 @@ static int safexcel_hmac_sha3_512_init(struct ahash_request *areq) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (areq);struct safexcel_ahash_ctx *ctx =crypto_ahash_ctx (tfm); -struct safexcel_ahash_req *req =ahash_request_ctx (areq); +struct safexcel_ahash_req *req =ahash_request_ctx_dma (areq);memset (req,0 ,sizeof (*req));
Herbert Xu
maintainerPATCH 7/10 crypto: keembay - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/keembay/keembay-ocs-hcu-core.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-)
drivers/crypto/keembay/keembay-ocs-hcu-core.c +13 -13
@@ -226,7 +226,7 @@ static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req,*/ static int kmb_ocs_dma_prepare (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);struct device *dev = rctx->hcu_dev->dev;unsigned int remainder =0 ;unsigned int total;
@@ -356,7 +356,7 @@ static int kmb_ocs_dma_prepare(struct ahash_request *req)static void kmb_ocs_hcu_secure_cleanup (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);/* Clear buffer of any data. */ memzero_explicit (rctx->buffer,sizeof (rctx->buffer));
@@ -374,7 +374,7 @@ static int kmb_ocs_hcu_handle_queue(struct ahash_request *req)static int prepare_ipad (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct ocs_hcu_ctx *ctx =crypto_ahash_ctx (tfm);int i;
@@ -414,7 +414,7 @@ static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) base);struct ocs_hcu_dev *hcu_dev =kmb_ocs_hcu_find_dev (req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);struct ocs_hcu_ctx *tctx =crypto_ahash_ctx (tfm);int rc;int i;
@@ -561,7 +561,7 @@ static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)static int kmb_ocs_hcu_init (struct ahash_request *req) {struct ocs_hcu_dev *hcu_dev =kmb_ocs_hcu_find_dev (req); -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct ocs_hcu_ctx *ctx =crypto_ahash_ctx (tfm);
@@ -614,7 +614,7 @@ static int kmb_ocs_hcu_init(struct ahash_request *req)static int kmb_ocs_hcu_update (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);int rc;if (!req->nbytes)
@@ -650,7 +650,7 @@ static int kmb_ocs_hcu_update(struct ahash_request *req)/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ static int kmb_ocs_hcu_fin_common (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);struct crypto_ahash *tfm =crypto_ahash_reqtfm (req);struct ocs_hcu_ctx *ctx =crypto_ahash_ctx (tfm);int rc;
@@ -687,7 +687,7 @@ static int kmb_ocs_hcu_fin_common(struct ahash_request *req)static int kmb_ocs_hcu_final (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req); rctx->sg_data_total =0 ; rctx->sg_data_offset =0 ;
@@ -698,7 +698,7 @@ static int kmb_ocs_hcu_final(struct ahash_request *req)static int kmb_ocs_hcu_finup (struct ahash_request *req) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req); rctx->sg_data_total = req->nbytes; rctx->sg_data_offset =0 ;
@@ -726,7 +726,7 @@ static int kmb_ocs_hcu_digest(struct ahash_request *req)static int kmb_ocs_hcu_export (struct ahash_request *req,void *out) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);/* Intermediate data is always stored and applied per request. */ memcpy (out, rctx,sizeof (*rctx));
@@ -736,7 +736,7 @@ static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)static int kmb_ocs_hcu_import (struct ahash_request *req,const void *in) { -struct ocs_hcu_rctx *rctx =ahash_request_ctx (req); +struct ocs_hcu_rctx *rctx =ahash_request_ctx_dma (req);/* Intermediate data is always stored and applied per request. */ memcpy (rctx, in,sizeof (*rctx));
@@ -822,8 +822,8 @@ static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key,/* Set request size and initialize tfm context. */ static void __cra_init (struct crypto_tfm *tfm,struct ocs_hcu_ctx *ctx) { -crypto_ahash_set_reqsize (__crypto_ahash_cast (tfm), -sizeof (struct ocs_hcu_rctx )); +crypto_ahash_set_reqsize_dma (__crypto_ahash_cast (tfm), +sizeof (struct ocs_hcu_rctx ));/* Init context to 0. */ memzero_explicit (ctx,sizeof (*ctx));
Herbert Xu
maintainerPATCH 8/10 crypto: octeontx - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 69 +++++++++++------------ 1 file changed, 35 insertions(+), 34 deletions(-)
drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +35 -34
@@ -103,7 +103,7 @@ static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req) req = container_of(cpt_req->areq, struct aead_request, base); tfm =crypto_aead_reqtfm (req); - rctx =aead_request_ctx (req); + rctx =aead_request_ctx_dma (req);if (memcmp (rctx->fctx.hmac.s.hmac_calc, rctx->fctx.hmac.s.hmac_recv,crypto_aead_authsize (tfm)) !=0 )
@@ -155,7 +155,7 @@ static void output_iv_copyback(struct crypto_async_request *areq) ctx =crypto_skcipher_ctx (stfm);if (ctx->cipher_type ==OTX_CPT_AES_CBC || ctx->cipher_type ==OTX_CPT_DES3_CBC ) { - rctx =skcipher_request_ctx (sreq); + rctx =skcipher_request_ctx_dma (sreq); req_info = &rctx->cpt_req; ivsize =crypto_skcipher_ivsize (stfm); start = sreq->cryptlen - ivsize;
@@ -233,7 +233,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, u32 *argcnt) {struct crypto_skcipher *stfm =crypto_skcipher_reqtfm (req); -struct otx_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;struct crypto_tfm *tfm =crypto_skcipher_tfm (stfm);struct otx_cpt_enc_ctx *ctx =crypto_tfm_ctx (tfm);
@@ -303,7 +303,7 @@ static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,static inline u32 create_input_list (struct skcipher_request *req,u32 enc,u32 enc_iv_len) { -struct otx_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;int ret;
@@ -321,7 +321,7 @@ static inline u32 create_input_list(struct skcipher_request *req, u32 enc,static inline void create_output_list (struct skcipher_request *req,u32 enc_iv_len) { -struct otx_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;
@@ -340,7 +340,7 @@ static inline void create_output_list(struct skcipher_request *req,static inline int cpt_enc_dec (struct skcipher_request *req,u32 enc) {struct crypto_skcipher *stfm =crypto_skcipher_reqtfm (req); -struct otx_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 enc_iv_len =crypto_skcipher_ivsize (stfm);struct pci_dev *pdev;
@@ -501,15 +501,16 @@ static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm)* allocated since the cryptd daemon uses * this memory for request_ctx information */ -crypto_skcipher_set_reqsize (tfm,sizeof (struct otx_cpt_req_ctx ) + -sizeof (struct skcipher_request )); +crypto_skcipher_set_reqsize_dma ( + tfm,sizeof (struct otx_cpt_req_ctx ) + +sizeof (struct skcipher_request ));return 0 ; }static int cpt_aead_init (struct crypto_aead *tfm,u8 cipher_type,u8 mac_type) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm); ctx->cipher_type = cipher_type; ctx->mac_type = mac_type;
@@ -551,7 +552,7 @@ static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type) } } -crypto_aead_set_reqsize (tfm,sizeof (struct otx_cpt_req_ctx )); +crypto_aead_set_reqsize_dma (tfm,sizeof (struct otx_cpt_req_ctx ));return 0 ; }
@@ -603,7 +604,7 @@ static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)static void otx_cpt_aead_exit (struct crypto_aead *tfm) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);kfree (ctx->ipad);kfree (ctx->opad);
@@ -619,7 +620,7 @@ static void otx_cpt_aead_exit(struct crypto_aead *tfm)static int otx_cpt_aead_set_authsize (struct crypto_aead *tfm,unsigned int authsize) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);switch (ctx->mac_type) {case OTX_CPT_SHA1 :
@@ -739,7 +740,7 @@ static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)static int aead_hmac_init (struct crypto_aead *cipher) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);int state_size =crypto_shash_statesize (ctx->hashalg);int ds =crypto_shash_digestsize (ctx->hashalg);int bs =crypto_shash_blocksize (ctx->hashalg);
@@ -837,7 +838,7 @@ static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);struct crypto_authenc_key_param *param;int enckeylen =0 , authkeylen =0 ;struct rtattr *rta = (void *)key;
@@ -896,7 +897,7 @@ static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);struct crypto_authenc_key_param *param;struct rtattr *rta = (void *)key;int enckeylen =0 ;
@@ -932,7 +933,7 @@ static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);/* * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
@@ -965,9 +966,9 @@ static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,static inline u32 create_aead_ctx_hdr (struct aead_request *req,u32 enc,u32 *argcnt) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);struct otx_cpt_req_info *req_info = &rctx->cpt_req;struct otx_cpt_fc_ctx *fctx = &rctx->fctx;int mac_len =crypto_aead_authsize (tfm);
@@ -1050,9 +1051,9 @@ static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,static inline u32 create_hmac_ctx_hdr (struct aead_request *req,u32 *argcnt,u32 enc) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);struct otx_cpt_req_info *req_info = &rctx->cpt_req; req_info->ctrl.s.dma_mode =OTX_CPT_DMA_GATHER_SCATTER ;
@@ -1076,7 +1077,7 @@ static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,static inline u32 create_aead_input_list (struct aead_request *req,u32 enc) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 inputlen = req->cryptlen + req->assoclen;u32 status, argcnt =0 ;
@@ -1093,7 +1094,7 @@ static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)static inline u32 create_aead_output_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 , outputlen =0 ;
@@ -1111,7 +1112,7 @@ static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,static inline u32 create_aead_null_input_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;u32 inputlen, argcnt =0 ;
@@ -1130,7 +1131,7 @@ static inline u32 create_aead_null_input_list(struct aead_request *req,static inline u32 create_aead_null_output_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;struct scatterlist *dst;u8 *ptr =NULL ;
@@ -1217,7 +1218,7 @@ static inline u32 create_aead_null_output_list(struct aead_request *req,static u32 cpt_aead_enc_dec (struct aead_request *req,u8 reg_type,u8 enc) { -struct otx_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx_cpt_req_info *req_info = &rctx->cpt_req;struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct pci_dev *pdev;
@@ -1409,7 +1410,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha1_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1428,7 +1429,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha256_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1447,7 +1448,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha384_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1466,7 +1467,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha512_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1485,7 +1486,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha1_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1504,7 +1505,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha256_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1523,7 +1524,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha384_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1542,7 +1543,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha512_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1561,7 +1562,7 @@ static struct aead_alg otx_cpt_aeads[] = { { .cra_driver_name = "cpt_rfc4106_gcm_aes", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
Herbert Xu
maintainerPATCH 9/10 crypto: octeontx2 - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c | 79 ++++++++++----------- 1 file changed, 40 insertions(+), 39 deletions(-)
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c +40 -39
@@ -87,7 +87,7 @@ static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) req = container_of(cpt_req->areq, struct aead_request, base); tfm =crypto_aead_reqtfm (req); - rctx =aead_request_ctx (req); + rctx =aead_request_ctx_dma (req);if (memcmp (rctx->fctx.hmac.s.hmac_calc, rctx->fctx.hmac.s.hmac_recv,crypto_aead_authsize (tfm)) !=0 )
@@ -137,7 +137,7 @@ static void output_iv_copyback(struct crypto_async_request *areq) ctx =crypto_skcipher_ctx (stfm);if (ctx->cipher_type ==OTX2_CPT_AES_CBC || ctx->cipher_type ==OTX2_CPT_DES3_CBC ) { - rctx =skcipher_request_ctx (sreq); + rctx =skcipher_request_ctx_dma (sreq); req_info = &rctx->cpt_req; ivsize =crypto_skcipher_ivsize (stfm); start = sreq->cryptlen - ivsize;
@@ -219,7 +219,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, u32 *argcnt) {struct crypto_skcipher *stfm =crypto_skcipher_reqtfm (req); -struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx2_cpt_enc_ctx *ctx =crypto_skcipher_ctx (stfm);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
@@ -288,7 +288,7 @@ static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,static inline int create_input_list (struct skcipher_request *req,u32 enc,u32 enc_iv_len) { -struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;int ret;
@@ -306,7 +306,7 @@ static inline int create_input_list(struct skcipher_request *req, u32 enc,static inline void create_output_list (struct skcipher_request *req,u32 enc_iv_len) { -struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 ;
@@ -325,7 +325,7 @@ static inline void create_output_list(struct skcipher_request *req,static int skcipher_do_fallback (struct skcipher_request *req,bool is_enc) {struct crypto_skcipher *stfm =crypto_skcipher_reqtfm (req); -struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx2_cpt_enc_ctx *ctx =crypto_skcipher_ctx (stfm);int ret;
@@ -348,7 +348,7 @@ static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)static inline int cpt_enc_dec (struct skcipher_request *req,u32 enc) {struct crypto_skcipher *stfm =crypto_skcipher_reqtfm (req); -struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =skcipher_request_ctx_dma (req);struct otx2_cpt_enc_ctx *ctx =crypto_skcipher_ctx (stfm);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 enc_iv_len =crypto_skcipher_ivsize (stfm);
@@ -537,8 +537,9 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)* allocated since the cryptd daemon uses * this memory for request_ctx information */ -crypto_skcipher_set_reqsize (stfm,sizeof (struct otx2_cpt_req_ctx ) + -sizeof (struct skcipher_request )); +crypto_skcipher_set_reqsize_dma ( + stfm,sizeof (struct otx2_cpt_req_ctx ) + +sizeof (struct skcipher_request ));return cpt_skcipher_fallback_init (ctx, alg); }
@@ -572,7 +573,7 @@ static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,static int cpt_aead_init (struct crypto_aead *atfm,u8 cipher_type,u8 mac_type) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (atfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (atfm);struct crypto_tfm *tfm =crypto_aead_tfm (atfm);struct crypto_alg *alg = tfm->__crt_alg;
@@ -629,7 +630,7 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) ctx->enc_align_len =1 ;break ; } -crypto_aead_set_reqsize (atfm,sizeof (struct otx2_cpt_req_ctx )); +crypto_aead_set_reqsize_dma (atfm,sizeof (struct otx2_cpt_req_ctx ));return cpt_aead_fallback_init (ctx, alg); }
@@ -681,7 +682,7 @@ static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)static void otx2_cpt_aead_exit (struct crypto_aead *tfm) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);kfree (ctx->ipad);kfree (ctx->opad);
@@ -698,7 +699,7 @@ static void otx2_cpt_aead_exit(struct crypto_aead *tfm)static int otx2_cpt_aead_gcm_set_authsize (struct crypto_aead *tfm,unsigned int authsize) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);if (crypto_rfc4106_check_authsize (authsize))return -EINVAL ;
@@ -722,7 +723,7 @@ static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,static int otx2_cpt_aead_null_set_authsize (struct crypto_aead *tfm,unsigned int authsize) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm); ctx->is_trunc_hmac = true; tfm->authsize = authsize;
@@ -794,7 +795,7 @@ static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)static int aead_hmac_init (struct crypto_aead *cipher) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);int state_size =crypto_shash_statesize (ctx->hashalg);int ds =crypto_shash_digestsize (ctx->hashalg);int bs =crypto_shash_blocksize (ctx->hashalg);
@@ -892,7 +893,7 @@ static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);struct crypto_authenc_key_param *param;int enckeylen =0 , authkeylen =0 ;struct rtattr *rta = (void *)key;
@@ -944,7 +945,7 @@ static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);struct crypto_authenc_key_param *param;struct rtattr *rta = (void *)key;int enckeylen =0 ;
@@ -979,7 +980,7 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, const unsigned char *key, unsigned int keylen) { -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (cipher); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (cipher);/* * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
@@ -1012,9 +1013,9 @@ static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,static inline int create_aead_ctx_hdr (struct aead_request *req,u32 enc,u32 *argcnt) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;int mac_len =crypto_aead_authsize (tfm);
@@ -1103,9 +1104,9 @@ static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,static inline void create_hmac_ctx_hdr (struct aead_request *req,u32 *argcnt,u32 enc) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);struct otx2_cpt_req_info *req_info = &rctx->cpt_req; req_info->ctrl.s.dma_mode =OTX2_CPT_DMA_MODE_SG ;
@@ -1127,7 +1128,7 @@ static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,static inline int create_aead_input_list (struct aead_request *req,u32 enc) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 inputlen = req->cryptlen + req->assoclen;u32 status, argcnt =0 ;
@@ -1144,7 +1145,7 @@ static inline int create_aead_input_list(struct aead_request *req, u32 enc)static inline void create_aead_output_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 argcnt =0 , outputlen =0 ;
@@ -1160,7 +1161,7 @@ static inline void create_aead_output_list(struct aead_request *req, u32 enc,static inline void create_aead_null_input_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;u32 inputlen, argcnt =0 ;
@@ -1177,7 +1178,7 @@ static inline void create_aead_null_input_list(struct aead_request *req,static inline int create_aead_null_output_list (struct aead_request *req,u32 enc,u32 mac_len) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;struct scatterlist *dst;u8 *ptr =NULL ;
@@ -1257,9 +1258,9 @@ static inline int create_aead_null_output_list(struct aead_request *req,static int aead_do_fallback (struct aead_request *req,bool is_enc) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *aead =crypto_aead_reqtfm (req); -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (aead); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (aead);int ret;if (ctx->fbk_cipher) {
@@ -1281,10 +1282,10 @@ static int aead_do_fallback(struct aead_request *req, bool is_enc)static int cpt_aead_enc_dec (struct aead_request *req,u8 reg_type,u8 enc) { -struct otx2_cpt_req_ctx *rctx =aead_request_ctx (req); +struct otx2_cpt_req_ctx *rctx =aead_request_ctx_dma (req);struct otx2_cpt_req_info *req_info = &rctx->cpt_req;struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx (tfm); +struct otx2_cpt_aead_ctx *ctx =crypto_aead_ctx_dma (tfm);struct pci_dev *pdev;int status, cpu_num;
@@ -1458,7 +1459,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha1_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1477,7 +1478,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha256_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1496,7 +1497,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha384_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1515,7 +1516,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha512_cbc_aes", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1534,7 +1535,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha1_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1553,7 +1554,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha256_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1572,7 +1573,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha384_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1591,7 +1592,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_hmac_sha512_ecb_null", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
@@ -1610,7 +1611,7 @@ static struct aead_alg otx2_cpt_aeads[] = { { .cra_driver_name = "cpt_rfc4106_gcm_aes", .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, .cra_priority = 4001, .cra_alignmask = 0, .cra_module = THIS_MODULE,
Herbert Xu
maintainerPATCH 10/10 crypto: qce - Set DMA alignment explicitly
In reply to Herbert XuThis driver has been implicitly relying on kmalloc alignment to be sufficient for DMA. This may no longer be the case with upcoming arm64 changes.
This patch changes it to explicitly request DMA alignment from the Crypto API.
---
drivers/crypto/qce/aead.c | 22 +++++++++++----------- drivers/crypto/qce/common.c | 5 +++-- drivers/crypto/qce/sha.c | 18 +++++++++--------- 3 files changed, 23 insertions(+), 22 deletions(-)
drivers/crypto/qce/aead.c +11 -11
@@ -24,7 +24,7 @@ static void qce_aead_done(void *data) {struct crypto_async_request *async_req = data;struct aead_request *req =aead_request_cast (async_req); -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_aead_ctx *ctx =crypto_tfm_ctx (async_req->tfm);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));struct qce_device *qce = tmpl->qce;
@@ -92,7 +92,7 @@ static void qce_aead_done(void *data)static struct scatterlist *qce_aead_prepare_result_buf (struct sg_table *tbl,struct aead_request *req) { -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));struct qce_device *qce = tmpl->qce;
@@ -103,7 +103,7 @@ qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req)static struct scatterlist *qce_aead_prepare_ccm_result_buf (struct sg_table *tbl,struct aead_request *req) { -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);sg_init_one (&rctx->result_sg, rctx->ccmresult_buf,QCE_BAM_BURST_SIZE );return qce_sgtable_add (tbl, &rctx->result_sg,QCE_BAM_BURST_SIZE );
@@ -112,7 +112,7 @@ qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req)static struct scatterlist *qce_aead_prepare_dst_buf (struct aead_request *req) { -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));struct qce_device *qce = tmpl->qce;struct scatterlist *sg, *msg_sg, __sg[2 ];
@@ -186,7 +186,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) {struct scatterlist *sg, *msg_sg, __sg[2 ];struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_aead_ctx *ctx =crypto_aead_ctx (tfm);unsigned int assoclen = rctx->assoclen;unsigned int adata_header_len, cryptlen, totallen;
@@ -300,7 +300,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req)static int qce_aead_prepare_buf (struct aead_request *req) { -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));struct qce_device *qce = tmpl->qce;struct scatterlist *sg;
@@ -328,7 +328,7 @@ static int qce_aead_prepare_buf(struct aead_request *req)static int qce_aead_ccm_prepare_buf (struct aead_request *req) { -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct qce_aead_ctx *ctx =crypto_aead_ctx (tfm);struct scatterlist *sg;
@@ -408,7 +408,7 @@ static int qce_aead_async_req_handle(struct crypto_async_request *async_req) {struct aead_request *req =aead_request_cast (async_req); -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct crypto_aead *tfm =crypto_aead_reqtfm (req);struct qce_aead_ctx *ctx =crypto_tfm_ctx (async_req->tfm);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));
@@ -502,7 +502,7 @@ qce_aead_async_req_handle(struct crypto_async_request *async_req)static int qce_aead_crypt (struct aead_request *req,int encrypt) {struct crypto_aead *tfm =crypto_aead_reqtfm (req); -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_aead_ctx *ctx =crypto_aead_ctx (tfm);struct qce_alg_template *tmpl =to_aead_tmpl (tfm);unsigned int blocksize =crypto_aead_blocksize (tfm);
@@ -675,8 +675,8 @@ static int qce_aead_init(struct crypto_aead *tfm)if (IS_ERR (ctx->fallback))return PTR_ERR (ctx->fallback); -crypto_aead_set_reqsize (tfm,sizeof (struct qce_aead_reqctx ) + -crypto_aead_reqsize (ctx->fallback)); +crypto_aead_set_reqsize_dma (tfm,sizeof (struct qce_aead_reqctx ) + +crypto_aead_reqsize (ctx->fallback));return 0 ; }
drivers/crypto/qce/common.c +3 -2
@@ -3,6 +3,7 @@* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. */ +#include <crypto/internal/hash.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/types.h>
@@ -147,7 +148,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) {struct ahash_request *req =ahash_request_cast (async_req);struct crypto_ahash *ahash =__crypto_ahash_cast (async_req->tfm); -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (async_req->tfm);struct qce_device *qce = tmpl->qce;unsigned int digestsize =crypto_ahash_digestsize (ahash);
@@ -419,7 +420,7 @@ static unsigned int qce_be32_to_cpu_array(u32 *dst, const u8 *src, unsigned intstatic int qce_setup_regs_aead (struct crypto_async_request *async_req) {struct aead_request *req =aead_request_cast (async_req); -struct qce_aead_reqctx *rctx =aead_request_ctx (req); +struct qce_aead_reqctx *rctx =aead_request_ctx_dma (req);struct qce_aead_ctx *ctx =crypto_tfm_ctx (async_req->tfm);struct qce_alg_template *tmpl =to_aead_tmpl (crypto_aead_reqtfm (req));struct qce_device *qce = tmpl->qce;
drivers/crypto/qce/sha.c +9 -9
@@ -38,7 +38,7 @@ static void qce_ahash_done(void *data)struct crypto_async_request *async_req = data;struct ahash_request *req =ahash_request_cast (async_req);struct crypto_ahash *ahash =crypto_ahash_reqtfm (req); -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (async_req->tfm);struct qce_device *qce = tmpl->qce;struct qce_result_dump *result = qce->dma.result_buf;
@@ -75,7 +75,7 @@ static void qce_ahash_done(void *data)static int qce_ahash_async_req_handle (struct crypto_async_request *async_req) {struct ahash_request *req =ahash_request_cast (async_req); -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_sha_ctx *ctx =crypto_tfm_ctx (async_req->tfm);struct qce_alg_template *tmpl =to_ahash_tmpl (async_req->tfm);struct qce_device *qce = tmpl->qce;
@@ -132,7 +132,7 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req)static int qce_ahash_init (struct ahash_request *req) { -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (req->base.tfm);const u32 *std_iv = tmpl->std_iv;
@@ -147,7 +147,7 @@ static int qce_ahash_init(struct ahash_request *req)static int qce_ahash_export (struct ahash_request *req,void *out) { -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_sha_saved_state *export_state = out;memcpy (export_state->pending_buf, rctx->buf, rctx->buflen);
@@ -164,7 +164,7 @@ static int qce_ahash_export(struct ahash_request *req, void *out)static int qce_ahash_import (struct ahash_request *req,const void *in) { -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);const struct qce_sha_saved_state *import_state = in;memset (rctx,0 ,sizeof (*rctx));
@@ -183,7 +183,7 @@ static int qce_ahash_import(struct ahash_request *req, const void *in)static int qce_ahash_update (struct ahash_request *req) {struct crypto_ahash *tfm =crypto_ahash_reqtfm (req); -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (req->base.tfm);struct qce_device *qce = tmpl->qce;struct scatterlist *sg_last, *sg;
@@ -275,7 +275,7 @@ static int qce_ahash_update(struct ahash_request *req)static int qce_ahash_final (struct ahash_request *req) { -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (req->base.tfm);struct qce_device *qce = tmpl->qce;
@@ -302,7 +302,7 @@ static int qce_ahash_final(struct ahash_request *req)static int qce_ahash_digest (struct ahash_request *req) { -struct qce_sha_reqctx *rctx =ahash_request_ctx (req); +struct qce_sha_reqctx *rctx =ahash_request_ctx_dma (req);struct qce_alg_template *tmpl =to_ahash_tmpl (req->base.tfm);struct qce_device *qce = tmpl->qce;int ret;
@@ -395,7 +395,7 @@ static int qce_ahash_cra_init(struct crypto_tfm *tfm)struct crypto_ahash *ahash =__crypto_ahash_cast (tfm);struct qce_sha_ctx *ctx =crypto_tfm_ctx (tfm); -crypto_ahash_set_reqsize (ahash,sizeof (struct qce_sha_reqctx )); +crypto_ahash_set_reqsize_dma (ahash,sizeof (struct qce_sha_reqctx ));memset (ctx,0 ,sizeof (*ctx));return 0 ; }
liulongfang
maintainerQuoted 11 lines · Herbert Xu
These are the rest of the driver conversions in order for arm64 to safely lower the kmalloc alignment below that required for DMA.
My criteria for inclusion are:
1) the driver can be built on arm64.
2) the driver may perform DMA on the context structure.I have worked through all the drivers in crypto but if you think I've missed something please let me know.
Hi, Herbert:
There are also skcipher_request_ctx and aead_request_ctx in the hisilicon/sec2 driver that need to be updated.
Thanks,
Longfang.
Quoted 2 lines
Thanks,
Herbert Xu
maintainerOn Fri, Dec 02, 2022 at 05:49:21PM +0800, liulongfang wrote:
.
Quoted 3 lines
Hi, Herbert:
There are alsoskcipher_request_ctxandaead_request_ctxin the hisilicon/sec2 driver that need to be updated.
Please show me where they do DMA directly on the context structure. I wasn't able to find such a place.
Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Quoted 2 lines · Herbert Xu
These are the rest of the driver conversions in order for arm64 to safely lower the kmalloc alignment below that required for DMA.
Btw, drivers/crypto/ has a lot of weird and most likely uses of GFP_DMA. Can someone look into those while we're doing DMA audits of the cryto drivers?
Herbert Xu
maintainerQuoted 7 lines · Christoph Hellwig
Quoted 2 lines · Herbert Xu
These are the rest of the driver conversions in order for arm64 to safely lower the kmalloc alignment below that required for DMA.
Btw,
drivers/crypto/has a lot of weird and most likely uses ofGFP_DMA. Can someone look into those while we're doing DMA audits of the cryto drivers?
Yes they're clearly bogus. Basically they are saying they want memory that is aligned sufficiently for DMA. So if Catalin's patch-set will break this assumption, then all the GFP_DMA allocations in drivers/crypto will need to be enlarged to take this into account.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Quoted 5 lines · Herbert Xu
Yes they're clearly bogus. Basically they are saying they want memory that is aligned sufficiently for DMA. So if Catalin's patch-set will break this assumption, then all the
GFP_DMAallocations indrivers/cryptowill need to be enlarged to take this into account.
But GFP_DMA never did do anything at all about alignment. It picks allocations from ZONE_DMA (which on x86 is the first 16MB only).
Herbert Xu
maintainerOn Mon, Dec 05, 2022 at 10:27:58PM -0800, Christoph Hellwig wrote:
.
Quoted 2 lines
But
GFP_DMAnever did do anything at all about alignment. It picks allocations fromZONE_DMA(which on x86 is the first 16MB only).
Right. I'm not arguing that they are correct or anything. I'm just saying that they are currently working on arm64 because of the large minimum kmalloc alignment, and they will all be broken afterwards.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerPATCH 1/1 crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
GFP_DMA does not guarantee that the returned memory is aligned for DMA. In fact for sun8i-ss it is superfluous and can be removed.
However, kmalloc may start returning DMA-unaligned memory in future so fix this by adding the alignment by hand.
drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +2 -2
@@ -452,7 +452,7 @@ int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, }kfree_sensitive (op->key); op->keylen = keylen; - op->key =kmemdup (key, keylen,GFP_KERNEL |GFP_DMA ); + op->key =kmemdup (key, keylen,GFP_KERNEL );if (!op->key)return -ENOMEM ;
@@ -475,7 +475,7 @@ int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,kfree_sensitive (op->key); op->keylen = keylen; - op->key =kmemdup (key, keylen,GFP_KERNEL |GFP_DMA ); + op->key =kmemdup (key, keylen,GFP_KERNEL );if (!op->key)return -ENOMEM ;
drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +8 -5
@@ -16,6 +16,7 @@#include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h>
@@ -527,7 +528,7 @@ static int allocate_flows(struct sun8i_ss_dev *ss)init_completion (&ss->flows[i].complete); ss->flows[i].biv =devm_kmalloc (ss->dev,AES_BLOCK_SIZE , -GFP_KERNEL |GFP_DMA ); +GFP_KERNEL );if (!ss->flows[i].biv) { err = -ENOMEM ; goto error_engine;
@@ -535,7 +536,7 @@ static int allocate_flows(struct sun8i_ss_dev *ss)for (j =0 ; j <MAX_SG ; j++) { ss->flows[i].iv[j] =devm_kmalloc (ss->dev,AES_BLOCK_SIZE , -GFP_KERNEL |GFP_DMA ); +GFP_KERNEL );if (!ss->flows[i].iv[j]) { err = -ENOMEM ; goto error_engine;
@@ -544,13 +545,15 @@ static int allocate_flows(struct sun8i_ss_dev *ss)/* the padding could be up to two block. */ ss->flows[i].pad =devm_kmalloc (ss->dev,MAX_PAD_SIZE , -GFP_KERNEL |GFP_DMA ); +GFP_KERNEL );if (!ss->flows[i].pad) { err = -ENOMEM ; goto error_engine; } - ss->flows[i].result =devm_kmalloc (ss->dev,SHA256_DIGEST_SIZE , -GFP_KERNEL |GFP_DMA ); + ss->flows[i].result = +devm_kmalloc (ss->dev,max (SHA256_DIGEST_SIZE , +dma_get_cache_alignment ()), +GFP_KERNEL );if (!ss->flows[i].result) { err = -ENOMEM ; goto error_engine;
drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +2 -2
@@ -79,10 +79,10 @@ int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,memcpy (tfmctx->key, key, keylen); } - tfmctx->ipad =kzalloc (bs,GFP_KERNEL |GFP_DMA ); + tfmctx->ipad =kzalloc (bs,GFP_KERNEL );if (!tfmctx->ipad)return -ENOMEM ; - tfmctx->opad =kzalloc (bs,GFP_KERNEL |GFP_DMA ); + tfmctx->opad =kzalloc (bs,GFP_KERNEL );if (!tfmctx->opad) { ret = -ENOMEM ; goto err_opad;
drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c +9 -2
@@ -11,6 +11,8 @@*/ #include "sun8i-ss.h" #include <linux/dma-mapping.h> +#include <linux/kernel.h> +#include <linux/mm.h> #include <linux/pm_runtime.h> #include <crypto/internal/rng.h>
@@ -25,7 +27,7 @@ int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, ctx->seed =NULL ; }if (!ctx->seed) - ctx->seed =kmalloc (slen,GFP_KERNEL |GFP_DMA ); + ctx->seed =kmalloc (slen,GFP_KERNEL );if (!ctx->seed)return -ENOMEM ;
@@ -58,6 +60,7 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,struct sun8i_ss_rng_tfm_ctx *ctx =crypto_rng_ctx (tfm);struct rng_alg *alg =crypto_rng_alg (tfm);struct sun8i_ss_alg_template *algt; +unsigned int todo_with_padding;struct sun8i_ss_dev *ss;dma_addr_t dma_iv, dma_dst;unsigned int todo;
@@ -81,7 +84,11 @@ int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, todo = dlen +PRNG_SEED_SIZE +PRNG_DATA_SIZE ; todo -= todo %PRNG_DATA_SIZE ; - d =kzalloc (todo,GFP_KERNEL |GFP_DMA ); + todo_with_padding =ALIGN (todo,dma_get_cache_alignment ()); +if (todo_with_padding < todo || todo < dlen) +return -EOVERFLOW ; + + d =kzalloc (todo_with_padding,GFP_KERNEL );if (!d)return -ENOMEM ;
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerPATCH 1/1 crypto: caam - Remove GFP_DMA and add DMA alignment padding
GFP_DMA does not guarantee that the returned memory is aligned for DMA. It should be removed where it is superfluous.
However, kmalloc may start returning DMA-unaligned memory in future so fix this by adding the alignment by hand.
drivers/crypto/caam/blob_gen.c +1 -1
@@ -83,7 +83,7 @@ int caam_process_blob(struct caam_blob_priv *priv, output_len = info->input_len -CAAM_BLOB_OVERHEAD ; } - desc =kzalloc (CAAM_BLOB_DESC_BYTES_MAX ,GFP_KERNEL |GFP_DMA ); + desc =kzalloc (CAAM_BLOB_DESC_BYTES_MAX ,GFP_KERNEL );if (!desc)return -ENOMEM ;
drivers/crypto/caam/caamalg.c +10 -6
@@ -59,6 +59,8 @@#include <crypto/engine.h> #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> /* * crypto alg
@@ -1379,8 +1381,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_bytes = sec4_sg_len *sizeof (struct sec4_sg_entry );/* allocate space for base edesc and hw desc commands, link tables */ - edesc =kzalloc (sizeof (*edesc) + desc_bytes + sec4_sg_bytes, -GFP_DMA | flags); + edesc =kzalloc (sizeof (*edesc) + desc_bytes + sec4_sg_bytes, flags);if (!edesc) {caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );
@@ -1608,6 +1609,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,u8 *iv;int ivsize =crypto_skcipher_ivsize (skcipher);int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; +unsigned int aligned_size; src_nents =sg_nents_for_len (req->src, req->cryptlen);if (unlikely (src_nents <0 )) {
@@ -1681,15 +1683,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* * allocate space for base edesc and hw desc commands, link tables, IV */ - edesc =kzalloc (sizeof (*edesc) + desc_bytes + sec4_sg_bytes + ivsize, -GFP_DMA | flags); -if (!edesc) { + aligned_size = ALIGN(ivsize, __alignof__(*edesc)); + aligned_size +=sizeof (*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size =ALIGN (aligned_size,dma_get_cache_alignment ()); + iv =kzalloc (aligned_size, flags); +if (!iv) {dev_err (jrdev,"could not allocate extended descriptor\n" );caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );return ERR_PTR (-ENOMEM ); } + edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->mapped_src_nents = mapped_src_nents;
@@ -1701,7 +1706,6 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* Make sure IV is located in a DMAable area */ if (ivsize) { - iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (jrdev, iv, ivsize,DMA_BIDIRECTIONAL );
drivers/crypto/caam/caamalg_qi.c +10 -6
@@ -20,6 +20,8 @@#include "caamalg_desc.h" #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> /* * crypto alg
@@ -959,7 +961,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,return (struct aead_edesc *)drv_ctx;/* allocate space for base edesc and hw desc commands, link tables */ - edesc =qi_cache_alloc (GFP_DMA | flags); + edesc =qi_cache_alloc (flags);if (unlikely (!edesc)) {dev_err (qidev,"could not allocate extended descriptor\n" );return ERR_PTR (-ENOMEM );
@@ -1317,8 +1319,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents =1 +pad_sg_nents (qm_sg_ents); qm_sg_bytes = qm_sg_ents *sizeof (struct qm_sg_entry ); -if (unlikely (offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes + - ivsize >CAAM_QI_MEMCACHE_SIZE )) { + if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) + + offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes > +CAAM_QI_MEMCACHE_SIZE )) {dev_err (qidev,"No space for %d S/G entries and/or %dB IV\n" , qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1327,17 +1330,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, }/* allocate space for base edesc, link tables and IV */ - edesc =qi_cache_alloc (GFP_DMA | flags); -if (unlikely (!edesc)) { + iv =qi_cache_alloc (flags); +if (unlikely (!iv)) {dev_err (qidev,"could not allocate extended descriptor\n" );caam_unmap (qidev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,DMA_NONE ,0 ,0 );return ERR_PTR (-ENOMEM ); } + edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); +/* Make sure IV is located in a DMAable area */ sg_table = &edesc->sgt[0 ]; - iv = (u8 *)(sg_table + qm_sg_ents);memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (qidev, iv, ivsize,DMA_BIDIRECTIONAL );
drivers/crypto/caam/caamalg_qi2.c +29 -19
160 lines not on this page · raw message
drivers/crypto/caam/caamalg_qi2.h +4 -6
@@ -7,13 +7,14 @@#ifndef _CAAMALG_QI2_H_#define _CAAMALG_QI2_H_ +#include <crypto/internal/skcipher.h> +#include <linux/compiler_attributes.h> #include <soc/fsl/dpaa2-io.h> #include <soc/fsl/dpaa2-fd.h> #include <linux/threads.h> #include <linux/netdevice.h> #include "dpseci.h" #include "desc_constr.h" -#include <crypto/skcipher.h> #define DPAA2_CAAM_STORE_SIZE 16/* NAPI weight *must* be a multiple of the store size. */
@@ -36,8 +37,6 @@* @tx_queue_attr: array of Tx queue attributes * @cscn_mem: pointer to memory region containing the congestion SCN * it's size is larger than to accommodate alignment -* @cscn_mem_aligned: pointer to congestion SCN; it is computed as -* PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN) * @cscn_dma: dma address used by the QMAN to write CSCN messages * @dev: device associated with the DPSECI object * @mc_io: pointer to MC portal's I/O object
@@ -58,7 +57,6 @@ struct dpaa2_caam_priv {/* congestion */ void *cscn_mem; -void *cscn_mem_aligned;dma_addr_t cscn_dma;struct device *dev;
@@ -158,7 +156,7 @@ struct ahash_edesc {struct caam_flc {u32 flc[16 ];u32 sh_desc[MAX_SDLEN ]; -}____cacheline_aligned ; +}__aligned(CRYPTO_DMA_ALIGN) ;enum optype {ENCRYPT =0 ,
@@ -180,7 +178,7 @@ enum optype {* @edesc: extended descriptor; points to one of {skcipher,aead}_edesc */ struct caam_request { -struct dpaa2_fl_entry fd_flt[2 ]; +struct dpaa2_fl_entry fd_flt[2 ]__aligned(CRYPTO_DMA_ALIGN) ;dma_addr_t fd_flt_dma;struct caam_flc *flc;dma_addr_t flc_dma;
drivers/crypto/caam/caamhash.c +11 -3
@@ -66,6 +66,8 @@#include "key_gen.h" #include "caamhash_desc.h" #include <crypto/engine.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #define CAAM_CRA_PRIORITY 3000
@@ -365,7 +367,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,dma_addr_t key_dma;int ret; - desc =kmalloc (CAAM_CMD_SZ *8 +CAAM_PTR_SZ *2 ,GFP_KERNEL |GFP_DMA ); + desc =kmalloc (CAAM_CMD_SZ *8 +CAAM_PTR_SZ *2 ,GFP_KERNEL );if (!desc) {dev_err (jrdev,"unable to allocate key input memory\n" );return -ENOMEM ;
@@ -432,7 +434,13 @@ static int ahash_setkey(struct crypto_ahash *ahash,dev_dbg (jrdev,"keylen %d\n" , keylen);if (keylen > blocksize) { - hashed_key =kmemdup (key, keylen,GFP_KERNEL |GFP_DMA ); +unsigned int aligned_len = +ALIGN (keylen,dma_get_cache_alignment ()); + +if (aligned_len < keylen) +return -EOVERFLOW ; + + hashed_key =kmemdup (key, keylen,GFP_KERNEL );if (!hashed_key)return -ENOMEM ; ret =hash_digest_key (ctx, &keylen, hashed_key, digestsize);
@@ -702,7 +710,7 @@ static struct ahash_edesc *ahash_edesc_alloc(struct ahash_request *req,struct ahash_edesc *edesc;unsigned int sg_size = sg_num *sizeof (struct sec4_sg_entry ); - edesc =kzalloc (sizeof (*edesc) + sg_size,GFP_DMA | flags); + edesc =kzalloc (sizeof (*edesc) + sg_size, flags);if (!edesc) {dev_err (ctx->jrdev,"could not allocate extended descriptor\n" );return NULL ;
drivers/crypto/caam/caampkc.c +17 -14
@@ -16,6 +16,8 @@#include "desc_constr.h" #include "sg_sw_sec4.h" #include "caampkc.h" +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB)#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
@@ -310,8 +312,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, sec4_sg_bytes = sec4_sg_len *sizeof (struct sec4_sg_entry );/* allocate space for base edesc, hw desc commands and link tables */ - edesc =kzalloc (sizeof (*edesc) + desclen + sec4_sg_bytes, -GFP_DMA | flags); + edesc =kzalloc (sizeof (*edesc) + desclen + sec4_sg_bytes, flags);if (!edesc) goto dst_fail;
@@ -898,7 +899,7 @@ static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)if (!nbytes)return NULL ; - dst =kzalloc (dstlen,GFP_DMA |GFP_KERNEL ); + dst =kzalloc (dstlen,GFP_KERNEL );if (!dst)return NULL ;
@@ -910,7 +911,7 @@ static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)/** * caam_read_raw_data - Read a raw byte stream as a positive integer. * The function skips buffer's leading zeros, copies the remained data -* to a buffer allocated in the GFP_DMA | GFP_KERNEL zone and returns +* to a buffer allocated in the GFP_KERNEL zone and returns * the address of the new buffer. * * @buf : The data to read
@@ -923,7 +924,7 @@ static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes)if (!*nbytes)return NULL ; -return kmemdup (buf, *nbytes,GFP_DMA |GFP_KERNEL ); +return kmemdup (buf, *nbytes,GFP_KERNEL ); }static int caam_rsa_check_key_length (unsigned int len)
@@ -949,13 +950,13 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,return ret;/* Copy key in DMA zone */ - rsa_key->e =kmemdup (raw_key.e, raw_key.e_sz,GFP_DMA |GFP_KERNEL ); + rsa_key->e =kmemdup (raw_key.e, raw_key.e_sz,GFP_KERNEL );if (!rsa_key->e) goto err;/* * Skip leading zeros and copy the positive integer to a buffer -* allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor +* allocated in the GFP_KERNEL zone. The decryption descriptor * expects a positive integer for the RSA modulus and uses its length as * decryption output length. */
@@ -983,6 +984,7 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx,struct caam_rsa_key *rsa_key = &ctx->key;size_t p_sz = raw_key->p_sz;size_t q_sz = raw_key->q_sz; +unsigned aligned_size; rsa_key->p =caam_read_raw_data (raw_key->p, &p_sz);if (!rsa_key->p)
@@ -994,11 +996,13 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, goto free_p; rsa_key->q_sz = q_sz; - rsa_key->tmp1 =kzalloc (raw_key->p_sz,GFP_DMA |GFP_KERNEL ); + aligned_size =ALIGN (raw_key->p_sz,dma_get_cache_alignment ()); + rsa_key->tmp1 =kzalloc (aligned_size,GFP_KERNEL );if (!rsa_key->tmp1) goto free_q; - rsa_key->tmp2 =kzalloc (raw_key->q_sz,GFP_DMA |GFP_KERNEL ); + aligned_size =ALIGN (raw_key->q_sz,dma_get_cache_alignment ()); + rsa_key->tmp2 =kzalloc (aligned_size,GFP_KERNEL );if (!rsa_key->tmp2) goto free_tmp1;
@@ -1051,17 +1055,17 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,return ret;/* Copy key in DMA zone */ - rsa_key->d =kmemdup (raw_key.d, raw_key.d_sz,GFP_DMA |GFP_KERNEL ); + rsa_key->d =kmemdup (raw_key.d, raw_key.d_sz,GFP_KERNEL );if (!rsa_key->d) goto err; - rsa_key->e =kmemdup (raw_key.e, raw_key.e_sz,GFP_DMA |GFP_KERNEL ); + rsa_key->e =kmemdup (raw_key.e, raw_key.e_sz,GFP_KERNEL );if (!rsa_key->e) goto err;/* * Skip leading zeros and copy the positive integer to a buffer -* allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor +* allocated in the GFP_KERNEL zone. The decryption descriptor * expects a positive integer for the RSA modulus and uses its length as * decryption output length. */
@@ -1185,8 +1189,7 @@ int caam_pkc_init(struct device *ctrldev)return 0 ;/* allocate zero buffer, used for padding input */ - zero_buffer =kzalloc (CAAM_RSA_MAX_INPUT_SIZE -1 ,GFP_DMA | -GFP_KERNEL ); + zero_buffer =kzalloc (CAAM_RSA_MAX_INPUT_SIZE -1 ,GFP_KERNEL );if (!zero_buffer)return -ENOMEM ;
drivers/crypto/caam/caamprng.c +9 -3
@@ -8,6 +8,8 @@#include <linux/completion.h> #include <crypto/internal/rng.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include "compat.h" #include "regs.h" #include "intern.h"
@@ -75,6 +77,7 @@ static int caam_prng_generate(struct crypto_rng *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int dlen) { + unsigned int aligned_dlen = ALIGN(dlen, dma_get_cache_alignment());struct caam_prng_ctx ctx;struct device *jrdev;dma_addr_t dst_dma;
@@ -82,7 +85,10 @@ static int caam_prng_generate(struct crypto_rng *tfm,u8 *buf;int ret; - buf =kzalloc (dlen,GFP_KERNEL ); +if (aligned_dlen < dlen) +return -EOVERFLOW ; + + buf =kzalloc (aligned_dlen,GFP_KERNEL );if (!buf)return -ENOMEM ;
@@ -94,7 +100,7 @@ static int caam_prng_generate(struct crypto_rng *tfm,return ret; } - desc =kzalloc (CAAM_PRNG_MAX_DESC_LEN ,GFP_KERNEL |GFP_DMA ); + desc =kzalloc (CAAM_PRNG_MAX_DESC_LEN ,GFP_KERNEL );if (!desc) { ret = -ENOMEM ; goto out1;
@@ -156,7 +162,7 @@ static int caam_prng_seed(struct crypto_rng *tfm,return ret; } - desc =kzalloc (CAAM_PRNG_MAX_DESC_LEN ,GFP_KERNEL |GFP_DMA ); + desc =kzalloc (CAAM_PRNG_MAX_DESC_LEN ,GFP_KERNEL );if (!desc) {caam_jr_free (jrdev);return -ENOMEM ;
drivers/crypto/caam/caamrng.c +7 -4
@@ -12,6 +12,8 @@#include <linux/hw_random.h> #include <linux/completion.h> #include <linux/atomic.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/kfifo.h> #include "compat.h"
@@ -176,17 +178,18 @@ static int caam_init(struct hwrng *rng)int err; ctx->desc_sync =devm_kzalloc (ctx->ctrldev,CAAM_RNG_DESC_LEN , -GFP_DMA |GFP_KERNEL ); +GFP_KERNEL );if (!ctx->desc_sync)return -ENOMEM ; ctx->desc_async =devm_kzalloc (ctx->ctrldev,CAAM_RNG_DESC_LEN , -GFP_DMA |GFP_KERNEL ); +GFP_KERNEL );if (!ctx->desc_async)return -ENOMEM ; -if (kfifo_alloc (&ctx->fifo,CAAM_RNG_MAX_FIFO_STORE_SIZE , -GFP_DMA |GFP_KERNEL )) +if (kfifo_alloc (&ctx->fifo,ALIGN (CAAM_RNG_MAX_FIFO_STORE_SIZE , +dma_get_cache_alignment ()), +GFP_KERNEL ))return -ENOMEM ;INIT_WORK (&ctx->worker, caam_rng_worker);
drivers/crypto/caam/ctrl.c +2 -2
@@ -199,7 +199,7 @@ static int deinstantiate_rng(struct device *ctrldev, int state_handle_mask)u32 *desc, status;int sh_idx, ret =0 ; - desc =kmalloc (CAAM_CMD_SZ *3 ,GFP_KERNEL |GFP_DMA ); + desc =kmalloc (CAAM_CMD_SZ *3 ,GFP_KERNEL );if (!desc)return -ENOMEM ;
@@ -276,7 +276,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,int ret =0 , sh_idx; ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; - desc =kmalloc (CAAM_CMD_SZ *7 ,GFP_KERNEL |GFP_DMA ); + desc =kmalloc (CAAM_CMD_SZ *7 ,GFP_KERNEL );if (!desc)return -ENOMEM ;
drivers/crypto/caam/key_gen.c +1 -1
@@ -64,7 +64,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,if (local_max > max_keylen)return -EINVAL ; - desc =kmalloc (CAAM_CMD_SZ *6 +CAAM_PTR_SZ *2 ,GFP_KERNEL |GFP_DMA ); + desc =kmalloc (CAAM_CMD_SZ *6 +CAAM_PTR_SZ *2 ,GFP_KERNEL );if (!desc) {dev_err (jrdev,"unable to allocate key input memory\n" );return ret;
drivers/crypto/caam/qi.c +2 -2
@@ -614,7 +614,7 @@ static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)struct qman_fq *fq;int ret; - fq =kzalloc (sizeof (*fq),GFP_KERNEL |GFP_DMA ); + fq =kzalloc (sizeof (*fq),GFP_KERNEL );if (!fq)return -ENOMEM ;
@@ -756,7 +756,7 @@ int caam_qi_init(struct platform_device *caam_pdev) } qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE ,0 , -SLAB_CACHE_DMA ,NULL ); +0 ,NULL );if (!qi_cache) {dev_err (qidev,"Can't allocate CAAM cache\n" );free_rsp_fqs ();
drivers/crypto/caam/qi.h +8 -4
@@ -9,6 +9,8 @@#ifndef __QI_H__#define __QI_H__ +#include <crypto/algapi.h> +#include <linux/compiler_attributes.h> #include <soc/fsl/qman.h> #include "compat.h" #include "desc.h"
@@ -58,8 +60,10 @@ enum optype {* @qidev: device pointer for CAAM/QI backend */ struct caam_drv_ctx { -u32 prehdr[2 ]; -u32 sh_desc[MAX_SDLEN ]; +struct { +u32 prehdr[2 ]; +u32 sh_desc[MAX_SDLEN ]; + }__aligned(CRYPTO_DMA_ALIGN) ;dma_addr_t context_a;struct qman_fq *req_fq;struct qman_fq *rsp_fq;
@@ -67,7 +71,7 @@ struct caam_drv_ctx {int cpu;enum optype op_type;struct device *qidev; -}____cacheline_aligned ; +};/** * caam_drv_req - The request structure the driver application should fill while
@@ -88,7 +92,7 @@ struct caam_drv_req {struct caam_drv_ctx *drv_ctx;caam_qi_cbk cbk;void *app_ctx; -}____cacheline_aligned ; +}__aligned(CRYPTO_DMA_ALIGN) ;/** * caam_drv_ctx_init - Initialise a CAAM/QI driver context
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Herbert Xu
maintainerPATCH 1/1 crypto: talitos - Remove GFP_DMA and add DMA alignment padding
GFP_DMA does not guarantee that the returned memory is aligned for DMA. It should be removed where it is superfluous.
However, kmalloc may start returning DMA-unaligned memory in future so fix this by adding the alignment by hand.
drivers/crypto/talitos.c +1 -1
@@ -1393,7 +1393,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len +=sizeof (struct talitos_desc ); alloc_len += ivsize; - edesc =kmalloc (alloc_len,GFP_DMA | flags); + edesc =kmalloc (ALIGN (alloc_len,dma_get_cache_alignment ()), flags);if (!edesc)return ERR_PTR (-ENOMEM );if (ivsize) {
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Re: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
Looks good:
Re: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
In reply to Herbert XuLe 30/12/2022 à 08:31, Herbert Xu a écrit :
Quoted 2 lines
GFP_DMAdoes not guarantee that the returned memory is aligned for DMA. It should be removed where it is superfluous.
Doesn't GFP_DMA guarantees that the provided memory is addressable for DMA ? Or do we assume that all memory returned by kmalloc can be used for DMA ?
Quoted 3 lines
However, kmalloc may start returning DMA-unaligned memory in future so fix this by adding the alignment by hand.
kmalloc() already returns not DMA aligned memory, why does it becomes a problem now ?
Ok, that may be suboptimal, but is that a problem at all ?
By the way, I'm not sure I understand the solution, what's the added value of aligning allocation length to the cache alignment ?
Quoted 16 lines
Signed-off-by Herbert Xu <[email protected]>drivers/crypto/talitos.c +1
-1@@ -1393,7 +1393,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len +=sizeof (struct talitos_desc ); alloc_len += ivsize; - edesc =kmalloc (alloc_len,GFP_DMA | flags); + edesc =kmalloc (ALIGN (alloc_len,dma_get_cache_alignment ()), flags);if (!edesc)return ERR_PTR (-ENOMEM );if (ivsize) {
Herbert Xu
maintainerRe: [PATCH] crypto: talitos - Remove GFP_DMA and add DMA alignment padding
Quoted 4 lines · Christophe Leroy
Doesn't
GFP_DMAguarantees that the provided memory is addressable for DMA ? Or do we assume that all memory returned by kmalloc can be used for DMA ?
No it does not. Please refer to the DMI API documentation.
Quoted 2 lines
kmalloc()already returns not DMA aligned memory, why does it becomes a problem now ?
As it stands kmalloc on arm64 returns DMA-aligned memory. This will soon no longer be the case.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Re: [PATCH] crypto: caam - Remove GFP_DMA and add DMA alignment padding
In reply to Herbert XuLooks good:
A message that is not in the archive (sent off-list or not yet received): DU0PR04MB95635D72885111458C50DB6F8EA89@DU0PR04MB9563.eurprd04.prod.outlook.com
Herbert Xu
maintainerPATCH 1/1 crypto: caam - Fix edesc/iv ordering mixup
Hi Meenakshi:
Quoted 6 lines · Meenakshi Aggarwal
with this change, edesc is following IV but we need IV to follow edesc.
Also, we are freeing edesc pointer in function, returning edesc pointer and calling function also free edesc pointer but you have allocated IV . So we are facing kernel crash.We need to fix this, please share why are you allocating IV in place of edesc ?
Sorry, my patch was completely broken. I was trying to place the IV at the front in a vain effort to reduce the total allocation size.
Anyhow, please let me know if this patch fixes the problem, and I will push it to Linus.
BTW, should we add you to the list of maintainers for caam? Perhaps next time we can spot the problem earlier. Thanks!
---8<---
The attempt to add DMA alignment padding by moving IV to the front of edesc was completely broken as it didn't change the places where edesc was freed.
It's also wrong as the IV may still share a cache-line with the edesc.
Fix this by restoring the original layout and simply reserving enough memmory so that the IV is on a DMA cache-line by itself.
drivers/crypto/caam/caamalg.c +12 -5
@@ -60,7 +60,11 @@#include <crypto/xts.h> #include <asm/unaligned.h> #include <linux/dma-mapping.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> /* * crypto alg
@@ -1683,18 +1687,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* * allocate space for base edesc and hw desc commands, link tables, IV */ - aligned_size = ALIGN(ivsize, __alignof__(*edesc)); - aligned_size +=sizeof (*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size =sizeof (*edesc) + desc_bytes + sec4_sg_bytes; aligned_size =ALIGN (aligned_size,dma_get_cache_alignment ()); - iv =kzalloc (aligned_size, flags); -if (!iv) { + aligned_size += ~(ARCH_KMALLOC_MINALIGN -1 ) & + (dma_get_cache_alignment () -1 ); + aligned_size +=ALIGN (ivsize,dma_get_cache_alignment ()); + edesc =kzalloc (aligned_size, flags); +if (!edesc) {dev_err (jrdev,"could not allocate extended descriptor\n" );caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->mapped_src_nents = mapped_src_nents;
@@ -1706,6 +1711,8 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* Make sure IV is located in a DMAable area */ if (ivsize) { + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; + iv =PTR_ALIGN (iv,dma_get_cache_alignment ());memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (jrdev, iv, ivsize,DMA_BIDIRECTIONAL );
drivers/crypto/caam/caamalg_qi.c +14 -7
@@ -20,8 +20,11 @@#include "caamalg_desc.h" #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/dma-mapping.h> #include <linux/kernel.h> +#include <linux/string.h> /* * crypto alg
@@ -1259,6 +1262,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,int dst_sg_idx, qm_sg_ents, qm_sg_bytes;struct qm_sg_entry *sg_table, *fd_sgt;struct caam_drv_ctx *drv_ctx; +unsigned int len; drv_ctx =get_drv_ctx (ctx, encrypt ?ENCRYPT :DECRYPT );if (IS_ERR (drv_ctx))
@@ -1319,9 +1323,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents =1 +pad_sg_nents (qm_sg_ents); qm_sg_bytes = qm_sg_ents *sizeof (struct qm_sg_entry ); - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) + - offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes > -CAAM_QI_MEMCACHE_SIZE )) { + + len = offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes; + len =ALIGN (len,dma_get_cache_alignment ()); + len += ivsize; + +if (unlikely (len >CAAM_QI_MEMCACHE_SIZE )) {dev_err (qidev,"No space for %d S/G entries and/or %dB IV\n" , qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1330,18 +1337,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, }/* allocate space for base edesc, link tables and IV */ - iv =qi_cache_alloc (flags); -if (unlikely (!iv)) { + edesc =qi_cache_alloc (flags); +if (unlikely (!edesc)) {dev_err (qidev,"could not allocate extended descriptor\n" );caam_unmap (qidev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,DMA_NONE ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); -/* Make sure IV is located in a DMAable area */ sg_table = &edesc->sgt[0 ]; + iv = (u8 *)(sg_table + qm_sg_ents); + iv =PTR_ALIGN (iv,dma_get_cache_alignment ());memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (qidev, iv, ivsize,DMA_BIDIRECTIONAL );
drivers/crypto/caam/qi.c +8 -2
@@ -8,7 +8,13 @@*/ #include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/slab.h> +#include <linux/string.h> #include <soc/fsl/qman.h> #include "debugfs.h"
@@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)napi_enable (irqtask); } - qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE ,0 , -0 ,NULL ); + qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE , +dma_get_cache_alignment (),0 ,NULL );if (!qi_cache) {dev_err (qidev,"Can't allocate CAAM cache\n" );free_rsp_fqs ();
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
RE: [PATCH] crypto: caam - Fix edesc/iv ordering mixup
Hi Herbert,
I have tested your changes, not facing a kernel crash now but still kernel warning messages are coming:
[ 6.359772] alg: skcipher: ctr-aes-caam encryption test failed (wrong output IV) on test vector 0, cfg="in-place (one sglist)" [ 6.371179] 00000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 [ 6.377645] alg: self-tests for ctr(aes) using ctr-aes-caam failed (rc=-22) [ 6.377649] ------------[ cut here ]------------ [ 6.389248] alg: self-tests for ctr(aes) using ctr-aes-caam failed (rc=-22) [ 6.389269] WARNING: CPU: 0 PID: 246 at crypto/testmgr.c:6101 alg_test.part.0+0x3c8/0x3d0 [ 6.404400] Modules linked in: [ 6.407446] CPU: 0 PID: 246 Comm: cryptomgr_test Not tainted 6.2.0-rc4-next-20230119-04770-gb8b0d08d8447 #3 [ 6.417181] Hardware name: LS1046A RDB Board (DT) [ 6.421876] pstate: 60000005 (nZCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 6.428833] pc : alg_test.part.0+0x3c8/0x3d0 [ 6.433095] lr : alg_test.part.0+0x3c8/0x3d0 [ 6.437358] sp : ffff80000a51bd40 [ 6.440662] x29: ffff80000a51bd40 x28: 0000000000000000 x27: 0000000000000000 [ 6.447795] x26: 00000000ffffffff x25: 0000000000000400 x24: ffff125a1d7ad280 [ 6.454926] x23: ffff125a1d7ad200 x22: ffff125a1e24d100 x21: 0000000000011085 [ 6.462057] x20: 00000000ffffffea x19: ffffb57461018490 x18: 0000000000000001 [ 6.469188] x17: 74757074756f2067 x16: 6e6f727728206465 x15: ffff125a1e24d568 [ 6.476319] x14: 0000000000000000 x13: ffffb57462e8e300 x12: 00000000ffffefff [ 6.483450] x11: 0000000000000003 x10: ffffb5746298ecd8 x9 : ffffb5745f902b34 [ 6.490581] x8 : 0000000000017fe8 x7 : c0000000ffffefff x6 : 000000000000bff4 [ 6.497712] x5 : 0000000000057fa8 x4 : 0000000000000000 x3 : ffff80000a51bb08 [ 6.504842] x2 : ffffb57462936a60 x1 : 78bd64a75e317900 x0 : 0000000000000000 [ 6.511973] Call trace: [ 6.514409] alg_test.part.0+0x3c8/0x3d0 [ 6.518325] alg_test+0x24/0x68 [ 6.521458] cryptomgr_test+0x28/0x48 [ 6.525112] kthread+0x114/0x120 [ 6.528334] ret_from_fork+0x10/0x20 [ 6.531903] ---[ end trace 0000000000000000 ]--- [ 6.536608] alg: skcipher: cbc-des-caam encryption test failed (wrong output IV) on test vector 0, cfg="in-place (one sglist)" [ 6.548010] 00000000: 00 00 00 00 00 00 00 00 [ 6.552370] alg: self-tests for cbc(des) using cbc-des-caam failed (rc=-22)
Thanks,
Meenakshi
Quoted 210 lines
-----Original Message-----
From: Herbert Xu <[email protected]>
Sent: Friday, February 24, 2023 3:48 PM
To: Meenakshi Aggarwal <[email protected]>
Cc: Linux Crypto Mailing List <[email protected]>; Christoph Hellwig <[email protected]>; Horia Geanta <[email protected]>; Pankaj Gupta <[email protected]>; Gaurav Jain <[email protected]>
Subject: [PATCH] crypto: caam - Fix edesc/iv ordering mixupHi Meenakshi:
Quoted 4 lines · Meenakshi Aggarwal
with this change, edesc is following IV but we need IV to follow edesc. Also, we are freeing edesc pointer in function, returning edesc pointer and calling function also free edesc pointer but you have allocated IV .
So we are facing kernel crash.
Quoted 2 lines
We need to fix this, please share why are you allocating IV in place of edesc ?
Sorry, my patch was completely broken. I was trying to place the IV at the front in a vain effort to reduce the total allocation size.
Anyhow, please let me know if this patch fixes the problem, and I will push it to Linus.
BTW, should we add you to the list of maintainers for caam? Perhaps next time we can spot the problem earlier. Thanks!
---8<---
The attempt to add DMA alignment padding by moving IV to the front of edesc was completely broken as it didn't change the places where edesc was freed.It's also wrong as the IV may still share a cache-line with the edesc.
Fix this by restoring the original layout and simply reserving enough memmory so that the IV is on a DMA cache-line by itself.
Reported-by Meenakshi Aggarwal <[email protected]>Fixes 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMAalignment padding")
Signed-off-by Herbert Xu <[email protected]>drivers/crypto/caam/caamalg.c +10
-5@@ -60,7 +60,11 @@#include <crypto/xts.h> #include <asm/unaligned.h> #include <linux/dma-mapping.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> /* * crypto alg @@ -1683,18 +1687,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* * allocate space for base edesc and hw desc commands, link tables, IV */ - aligned_size = ALIGN(ivsize, __alignof__(*edesc)); - aligned_size +=sizeof (*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes; aligned_size =ALIGN (aligned_size,dma_get_cache_alignment ()); - iv =kzalloc (aligned_size, flags); -if (!iv) { + aligned_size += ~(ARCH_KMALLOC_MINALIGN -1 ) & + (dma_get_cache_alignment () -1 ); + aligned_size +=ALIGN (ivsize,dma_get_cache_alignment ()); + edesc =kzalloc (aligned_size, flags); +if (!edesc) {dev_err (jrdev,"could not allocate extended descriptor\n" );caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents;
edesc->mapped_src_nents=mapped_src_nents; @@ -1706,6 +1711,8
@@ static structskcipher_edesc*skcipher_edesc_alloc(structskcipher_request*req,drivers/crypto/caam/caamalg.c +2
-0/* Make sure IV is located in a DMAable area */ if (ivsize) { + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; + iv =PTR_ALIGN (iv,dma_get_cache_alignment ());memcpy (iv, req->iv, ivsize); iv_dma = dma_map_single(jrdev, iv, ivsize,
DMA_BIDIRECTIONAL); diff --gita/drivers/crypto/caam/caamalg_qi.cb/drivers/crypto/caam/caamalg_qi.c
index5e218bf20d5b..5d17f5862b93100644drivers/crypto/caam/caamalg_qi.c +4
-0@@ -20,8 +20,11 @@#include "caamalg_desc.h" #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/dma-mapping.h> #include <linux/kernel.h> +#include <linux/string.h> /* * crypto alg @@ -1259,6 +1262,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, int dst_sg_idx, qm_sg_ents, qm_sg_bytes;struct qm_sg_entry *sg_table, *fd_sgt;struct caam_drv_ctx *drv_ctx; +unsigned int len; drv_ctx =get_drv_ctx (ctx, encrypt ?ENCRYPT :DECRYPT );drivers/crypto/caam/caamalg_qi.c +6
-3if (IS_ERR (drv_ctx))@@ -1319,9 +1323,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); qm_sg_bytes = qm_sg_ents *sizeof (struct qm_sg_entry ); - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) + - offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes > -CAAM_QI_MEMCACHE_SIZE )) { + + len = offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes; + len =ALIGN (len,dma_get_cache_alignment ()); + len += ivsize; + +if (unlikely (len >CAAM_QI_MEMCACHE_SIZE )) {dev_err (qidev,"No space for %d S/G entries and/or %dB IV\n" , qm_sg_ents, ivsize);drivers/crypto/caam/caamalg_qi.c +4
-4caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,@@ -1330,18 +1337,18 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, }/* allocate space for base edesc, link tables and IV */ - iv = qi_cache_alloc(flags); -if (unlikely (!iv)) { + edesc = qi_cache_alloc(flags); +if (unlikely (!edesc)) {dev_err (qidev,"could not allocate extended descriptor\n" );caam_unmap (qidev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,DMA_NONE ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); -/* Make sure IV is located in a DMAable area */ sg_table = &edesc->sgt[0 ]; + iv = (u8 *)(sg_table + qm_sg_ents); + iv =PTR_ALIGN (iv,dma_get_cache_alignment ());memcpy (iv, req->iv, ivsize);drivers/crypto/caam/caamalg_qi.c +0
-1iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); diff - -git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index
4c52c9365558..2ad2c1035856100644drivers/crypto/caam/qi.c +8
-2@@ -8,7 +8,13 @@*/ #include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/slab.h> +#include <linux/string.h> #include <soc/fsl/qman.h> #include "debugfs.h" @@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)napi_enable (irqtask); } - qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE ,0 , -0 ,NULL ); + qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE , +dma_get_cache_alignment (),0 ,NULL );if (!qi_cache) {drivers/crypto/caam/qi.c +0
-1dev_err (qidev,"Can't allocate CAAM cache\n" );free_rsp_fqs (); --Email: Herbert Xu <[email protected]> Home Page:
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
ana.org.au%2F~herbert%2F&data=05%7C01%7Cmeenakshi.aggarwal%40nxp.co m%7C02a5c1fbb5f54a893c0a08db165073cb%7C686ea1d3bc2b4c6fa92cd99c5c 301635%7C0%7C0%7C638128307024161720%7CUnknown%7CTWFpbGZsb3d8e yJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7 C3000%7C%7C%7C&sdata=51qldsOV%2FNUO5VEMSd4kKbQdqIOJdEWF99Us%2 FR01sUA%3D&reserved=0
PGP Key:
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
ana.org.au%2F~herbert%2Fpubkey.txt&data=05%7C01%7Cmeenakshi.aggarwal %40nxp.com%7C02a5c1fbb5f54a893c0a08db165073cb%7C686ea1d3bc2b4c6fa 92cd99c5c301635%7C0%7C0%7C638128307024161720%7CUnknown%7CTWFp bGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6 Mn0%3D%7C3000%7C%7C%7C&sdata=ME%2BROVEsFMovZ1P4Y3KOONWHwA ezgFgxOOe0y5fToCA%3D&reserved=0
Herbert Xu
maintainerPATCH v2 1/1 crypto: caam - Fix edesc/iv ordering mixup
Quoted 3 lines · Meenakshi Aggarwal
Hi Herbert,
I have tested your changes, not facing a kernel crash now but still kernel warning messages are coming:
Thanks for testing! Indeed, I forgot to update the IV calculations on the way out:
---8<---
The attempt to add DMA alignment padding by moving IV to the front of edesc was completely broken as it didn't change the places where edesc was freed.
It's also wrong as the IV may still share a cache-line with the edesc.
Fix this by restoring the original layout and simply reserving enough memmory so that the IV is on a DMA cache-line by itself.
drivers/crypto/caam/caamalg.c +19 -7
@@ -60,7 +60,11 @@#include <crypto/xts.h> #include <asm/unaligned.h> #include <linux/dma-mapping.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> /* * crypto alg
@@ -1000,6 +1004,13 @@ static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,crypto_finalize_aead_request (jrp->engine, req, ecode); } +static inline u8 *skcipher_edesc_iv (struct skcipher_edesc *edesc) +{ + +return PTR_ALIGN ((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, +dma_get_cache_alignment ()); +} +static void skcipher_crypt_done (struct device *jrdev,u32 *desc,u32 err,void *context) {
@@ -1027,8 +1038,7 @@ static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,* This is used e.g. by the CTS mode. */ if (ivsize && !ecode) { -memcpy (req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, - ivsize); +memcpy (req->iv,skcipher_edesc_iv (edesc), ivsize); print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
@@ -1683,18 +1693,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* * allocate space for base edesc and hw desc commands, link tables, IV */ - aligned_size = ALIGN(ivsize, __alignof__(*edesc)); - aligned_size +=sizeof (*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size =sizeof (*edesc) + desc_bytes + sec4_sg_bytes; aligned_size =ALIGN (aligned_size,dma_get_cache_alignment ()); - iv =kzalloc (aligned_size, flags); -if (!iv) { + aligned_size += ~(ARCH_KMALLOC_MINALIGN -1 ) & + (dma_get_cache_alignment () -1 ); + aligned_size +=ALIGN (ivsize,dma_get_cache_alignment ()); + edesc =kzalloc (aligned_size, flags); +if (!edesc) {dev_err (jrdev,"could not allocate extended descriptor\n" );caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->mapped_src_nents = mapped_src_nents;
@@ -1706,6 +1717,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,/* Make sure IV is located in a DMAable area */ if (ivsize) { + iv =skcipher_edesc_iv (edesc);memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (jrdev, iv, ivsize,DMA_BIDIRECTIONAL );
drivers/crypto/caam/caamalg_qi.c +26 -14
@@ -20,8 +20,11 @@#include "caamalg_desc.h" #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/dma-mapping.h> #include <linux/kernel.h> +#include <linux/string.h> /* * crypto alg
@@ -1204,6 +1207,12 @@ static int ipsec_gcm_decrypt(struct aead_request *req) false); } +static inline u8 *skcipher_edesc_iv (struct skcipher_edesc *edesc) +{ +return PTR_ALIGN ((u8 *)&edesc->sgt[0 ] + edesc->qm_sg_bytes, +dma_get_cache_alignment ()); +} +static void skcipher_done (struct caam_drv_req *drv_req,u32 status) {struct skcipher_edesc *edesc;
@@ -1236,8 +1245,7 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)* This is used e.g. by the CTS mode. */ if (!ecode) -memcpy (req->iv, (u8 *)&edesc->sgt[0 ] + edesc->qm_sg_bytes, - ivsize); +memcpy (req->iv,skcipher_edesc_iv (edesc), ivsize);qi_cache_free (edesc);skcipher_request_complete (req, ecode);
@@ -1259,6 +1267,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,int dst_sg_idx, qm_sg_ents, qm_sg_bytes;struct qm_sg_entry *sg_table, *fd_sgt;struct caam_drv_ctx *drv_ctx; +unsigned int len; drv_ctx =get_drv_ctx (ctx, encrypt ?ENCRYPT :DECRYPT );if (IS_ERR (drv_ctx))
@@ -1319,9 +1328,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents =1 +pad_sg_nents (qm_sg_ents); qm_sg_bytes = qm_sg_ents *sizeof (struct qm_sg_entry ); - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) + - offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes > -CAAM_QI_MEMCACHE_SIZE )) { + + len = offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes; + len =ALIGN (len,dma_get_cache_alignment ()); + len += ivsize; + +if (unlikely (len >CAAM_QI_MEMCACHE_SIZE )) {dev_err (qidev,"No space for %d S/G entries and/or %dB IV\n" , qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
@@ -1330,18 +1342,24 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, }/* allocate space for base edesc, link tables and IV */ - iv =qi_cache_alloc (flags); -if (unlikely (!iv)) { + edesc =qi_cache_alloc (flags); +if (unlikely (!edesc)) {dev_err (qidev,"could not allocate extended descriptor\n" );caam_unmap (qidev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,DMA_NONE ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->qm_sg_bytes = qm_sg_bytes; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = skcipher_done; + edesc->drv_req.drv_ctx = drv_ctx;/* Make sure IV is located in a DMAable area */ sg_table = &edesc->sgt[0 ]; + iv =skcipher_edesc_iv (edesc);memcpy (iv, req->iv, ivsize); iv_dma =dma_map_single (qidev, iv, ivsize,DMA_BIDIRECTIONAL );
@@ -1353,13 +1371,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,return ERR_PTR (-ENOMEM ); } - edesc->src_nents = src_nents; - edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; - edesc->qm_sg_bytes = qm_sg_bytes; - edesc->drv_req.app_ctx = req; - edesc->drv_req.cbk = skcipher_done; - edesc->drv_req.drv_ctx = drv_ctx;dma_to_qm_sg_one (sg_table, iv_dma, ivsize,0 );sg_to_qm_sg (req->src, req->cryptlen, sg_table +1 ,0 );
drivers/crypto/caam/qi.c +8 -2
@@ -8,7 +8,13 @@*/ #include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/slab.h> +#include <linux/string.h> #include <soc/fsl/qman.h> #include "debugfs.h"
@@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)napi_enable (irqtask); } - qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE ,0 , -0 ,NULL ); + qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE , +dma_get_cache_alignment (),0 ,NULL );if (!qi_cache) {dev_err (qidev,"Can't allocate CAAM cache\n" );free_rsp_fqs ();
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
RE: [v2 PATCH] crypto: caam - Fix edesc/iv ordering mixup
Hi Herbert,
Changes are working for me.
Thanks,
Meenakshi
Quoted 263 lines
-----Original Message-----
From: Herbert Xu <[email protected]>
Sent: Monday, February 27, 2023 1:56 PM
To: Meenakshi Aggarwal <[email protected]>
Cc: Linux Crypto Mailing List <[email protected]>; Christoph Hellwig <[email protected]>; Horia Geanta <[email protected]>; Pankaj Gupta <[email protected]>; Gaurav Jain <[email protected]>
Subject: [v2 PATCH] crypto: caam - Fix edesc/iv ordering mixupQuoted 3 lines · Meenakshi Aggarwal
Hi Herbert,
I have tested your changes, not facing a kernel crash now but still kernel
warning messages are coming:
Thanks for testing! Indeed, I forgot to update the IV calculations on the way out:
---8<---
The attempt to add DMA alignment padding by moving IV to the front of edesc was completely broken as it didn't change the places where edesc was freed.It's also wrong as the IV may still share a cache-line with the edesc.
Fix this by restoring the original layout and simply reserving enough memmory so that the IV is on a DMA cache-line by itself.
Reported-by Meenakshi Aggarwal <[email protected]>Fixes 199354d7fb6e ("crypto: caam - Remove GFP_DMA and add DMAalignment padding")
Signed-off-by Herbert Xu <[email protected]>drivers/crypto/caam/caamalg.c +11
-2@@ -60,7 +60,11 @@#include <crypto/xts.h> #include <asm/unaligned.h> #include <linux/dma-mapping.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> /* * crypto alg @@ -1000,6 +1004,13 @@ static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, crypto_finalize_aead_request(jrp->engine, req, ecode); } +static inline u8 *skcipher_edesc_iv (struct skcipher_edesc *edesc) { + +return PTR_ALIGN ((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, +dma_get_cache_alignment ()); +} +static void skcipher_crypt_done (struct device *jrdev,u32 *desc,u32 err,void *context) {@@ -1027,8 +1038,7 @@ static void skcipher_crypt_done(struct device *jrdev,u32 *desc, u32 err, * This is used e.g. by the CTS mode. */ if (ivsize && !ecode) { -memcpy (req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, - ivsize); +memcpy (req->iv,skcipher_edesc_iv (edesc), ivsize); print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4,req->iv, @@
-1683,18 +1693,19 @@ static structskcipher_edesc*skcipher_edesc_alloc(structskcipher_request*req,drivers/crypto/caam/caamalg.c +6
-5/* * allocate space for base edesc and hw desc commands, link tables, IV */ - aligned_size = ALIGN(ivsize, __alignof__(*edesc)); - aligned_size +=sizeof (*edesc) + desc_bytes + sec4_sg_bytes; + aligned_size =sizeof (*edesc) + desc_bytes + sec4_sg_bytes; aligned_size =ALIGN (aligned_size,dma_get_cache_alignment ()); - iv =kzalloc (aligned_size, flags); -if (!iv) { + aligned_size += ~(ARCH_KMALLOC_MINALIGN -1 ) & + (dma_get_cache_alignment () -1 ); + aligned_size +=ALIGN (ivsize,dma_get_cache_alignment ()); + edesc =kzalloc (aligned_size, flags); +if (!edesc) {dev_err (jrdev,"could not allocate extended descriptor\n" );caam_unmap (jrdev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->mapped_src_nents = mapped_src_nents; @@ -1706,6 +1717,7@@ static struct
skcipher_edesc*skcipher_edesc_alloc(structskcipher_request*req,drivers/crypto/caam/caamalg.c +1
-0/* Make sure IV is located in a DMAable area */ if (ivsize) { + iv =skcipher_edesc_iv (edesc);memcpy (iv, req->iv, ivsize); iv_dma = dma_map_single(jrdev, iv, ivsize,
DMA_BIDIRECTIONAL); diff --gita/drivers/crypto/caam/caamalg_qi.cb/drivers/crypto/caam/caamalg_qi.c
index5e218bf20d5b..743ce50c14f2100644drivers/crypto/caam/caamalg_qi.c +9
-2@@ -20,8 +20,11 @@#include "caamalg_desc.h" #include <crypto/xts.h> #include <asm/unaligned.h> +#include <linux/device.h> +#include <linux/err.h> #include <linux/dma-mapping.h> #include <linux/kernel.h> +#include <linux/string.h> /* * crypto alg @@ -1204,6 +1207,12 @@ static int ipsec_gcm_decrypt(struct aead_request *req) false); } +static inline u8 *skcipher_edesc_iv (struct skcipher_edesc *edesc) { +return PTR_ALIGN ((u8 *)&edesc->sgt[0 ] + edesc->qm_sg_bytes, +dma_get_cache_alignment ()); +} +static void skcipher_done (struct caam_drv_req *drv_req,u32 status) {struct skcipher_edesc *edesc;@@ -1236,8 +1245,7 @@ static void skcipher_done(struct caam_drv_req*drv_req, u32 status) * This is used e.g. by the CTS mode. */ if (!ecode) -memcpy (req->iv, (u8 *)&edesc->sgt[0 ] + edesc->qm_sg_bytes, - ivsize); +memcpy (req->iv,skcipher_edesc_iv (edesc), ivsize);qi_cache_free (edesc);
skcipher_request_complete(req, ecode); @@ -1259,6 +1267,7 @@
static structskcipher_edesc*skcipher_edesc_alloc(structskcipher_request*req,drivers/crypto/caam/caamalg_qi.c +7
-3int dst_sg_idx, qm_sg_ents, qm_sg_bytes;struct qm_sg_entry *sg_table, *fd_sgt;struct caam_drv_ctx *drv_ctx; +unsigned int len; drv_ctx =get_drv_ctx (ctx, encrypt ?ENCRYPT :DECRYPT );if (IS_ERR (drv_ctx))@@ -1319,9 +1328,12 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); qm_sg_bytes = qm_sg_ents *sizeof (struct qm_sg_entry ); - if (unlikely(ALIGN(ivsize, __alignof__(*edesc)) + - offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes > -CAAM_QI_MEMCACHE_SIZE )) { + + len = offsetof(struct skcipher_edesc , sgt) + qm_sg_bytes; + len =ALIGN (len,dma_get_cache_alignment ()); + len += ivsize; + +if (unlikely (len >CAAM_QI_MEMCACHE_SIZE )) {dev_err (qidev,"No space for %d S/G entries and/or %dB IV\n" , qm_sg_ents, ivsize);drivers/crypto/caam/caamalg_qi.c +9
-3caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,@@ -1330,18 +1342,24 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, }/* allocate space for base edesc, link tables and IV */ - iv = qi_cache_alloc(flags); -if (unlikely (!iv)) { + edesc = qi_cache_alloc(flags); +if (unlikely (!edesc)) {dev_err (qidev,"could not allocate extended descriptor\n" );caam_unmap (qidev, req->src, req->dst, src_nents, dst_nents,0 ,0 ,DMA_NONE ,0 ,0 );return ERR_PTR (-ENOMEM ); } - edesc = (void *)(iv + ALIGN(ivsize, __alignof__(*edesc))); + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->qm_sg_bytes = qm_sg_bytes; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = skcipher_done; + edesc->drv_req.drv_ctx = drv_ctx;/* Make sure IV is located in a DMAable area */ sg_table = &edesc->sgt[0 ]; + iv =skcipher_edesc_iv (edesc);memcpy (iv, req->iv, ivsize);
iv_dma=dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); @@
-1353,13 +1371,7 @@ static structskcipher_edesc*skcipher_edesc_alloc(structskcipher_request*req,drivers/crypto/caam/caamalg_qi.c +0
-6return ERR_PTR (-ENOMEM ); } - edesc->src_nents = src_nents; - edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; - edesc->qm_sg_bytes = qm_sg_bytes; - edesc->drv_req.app_ctx = req; - edesc->drv_req.cbk = skcipher_done; - edesc->drv_req.drv_ctx = drv_ctx;dma_to_qm_sg_one (sg_table, iv_dma, ivsize,0 ); sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); diff --git
a/drivers/crypto/caam/qi.cb/drivers/crypto/caam/qi.cindex4c52c9365558..2ad2c1035856100644drivers/crypto/caam/qi.c +8
-2@@ -8,7 +8,13 @@*/ #include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/kthread.h> +#include <linux/netdevice.h> +#include <linux/slab.h> +#include <linux/string.h> #include <soc/fsl/qman.h> #include "debugfs.h" @@ -755,8 +761,8 @@ int caam_qi_init(struct platform_device *caam_pdev)napi_enable (irqtask); } - qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE ,0 , -0 ,NULL ); + qi_cache =kmem_cache_create ("caamqicache" ,CAAM_QI_MEMCACHE_SIZE , +dma_get_cache_alignment (),0 ,NULL );if (!qi_cache) {drivers/crypto/caam/qi.c +0
-1dev_err (qidev,"Can't allocate CAAM cache\n" );free_rsp_fqs (); --Email: Herbert Xu <[email protected]> Home Page:
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
ana.org.au%2F~herbert%2F&data=05%7C01%7Cmeenakshi.aggarwal%40nxp.co m%7Cfa91dc99b1b147f0b13008db189c4c71%7C686ea1d3bc2b4c6fa92cd99c5c 301635%7C0%7C0%7C638130831812121500%7CUnknown%7CTWFpbGZsb3d8e yJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7 C3000%7C%7C%7C&sdata=UScavImC4%2FcLjGiqTDANWb%2FG1cOg4J18Ijfoicl %2FRSg%3D&reserved=0
PGP Key:
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fgondor.ap
ana.org.au%2F~herbert%2Fpubkey.txt&data=05%7C01%7Cmeenakshi.aggarwal %40nxp.com%7Cfa91dc99b1b147f0b13008db189c4c71%7C686ea1d3bc2b4c6fa 92cd99c5c301635%7C0%7C0%7C638130831812121500%7CUnknown%7CTWFp bGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6 Mn0%3D%7C3000%7C%7C%7C&sdata=ub4VeaCOKFt4iDVcxy3jnN0EXJBPJZEs%2 B1CrnK5gimg%3D&reserved=0
Re: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
In reply to Herbert XuQuoted 7 lines · Herbert Xu
GFP_DMAdoes not guarantee that the returned memory is aligned for DMA. In fact for sun8i-ss it is superfluous and can be removed.However, kmalloc may start returning DMA-unaligned memory in future so fix this by adding the alignment by hand.
Signed-off-by Herbert Xu <[email protected]>
Hello
Tested-on: sun8i-a83t-bananapi-m3
This means that lot of other crypto driver (sun8i-ce, amlogic, etc...) I maintain need the same fix, right ?
Regards
Herbert Xu
maintainerRe: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
On Tue, Jan 03, 2023 at 07:51:24AM +0100, Corentin Labbe wrote:
.
Quoted 3 lines
Tested-by Corentin Labbe <[email protected]>Tested-on: sun8i-a83t-bananapi-m3
Acked-by Corentin Labbe <[email protected]>
Thanks,
Quoted 1 line
This means that lot of other crypto driver (sun8i-ce, amlogic, etc...) I maintain need the same fix, right ?
I checked the other drivers and none of them seem to be broken. Only DMA from the device (DMA_FROM_DEVICE or DMA_BIDIRECTIONAL) need to be aligned appropriately.
However, please do remove any unnecessary uses of GFP_DMA in those drivers.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Re: [PATCH] crypto: sun8i-ss - Remove GFP_DMA and add DMA alignment padding
In reply to Herbert XuLooks good: