crypto: qce - use cryptlen when adding extra sgl
Subsystems (2)
PATCH v4 1/3 crypto: qce - use cryptlen when adding extra sgl
The qce crypto driver appends an extra entry to the dst sgl, to maintain private state information.
When the gcm driver sends requests to the ctr skcipher, it passes the authentication tag after the actual crypto payload, but it must not be touched.
Commit 1336c2221bee ("crypto: qce - save a sg table slot for result buf") limited the destination sgl to avoid overwriting the authentication tag but it assumed the tag would be in a separate sgl entry.
This is not always the case, so it is better to limit the length of the destination buffer to req->cryptlen before appending the result buf.
--
v1 -> v4: no change
drivers/crypto/qce/dma.c +6 -5
@@ -48,9 +48,10 @@ void qce_dma_release(struct qce_dma_data *dma)struct scatterlist *qce_sgtable_add (struct sg_table *sgt,struct scatterlist *new_sgl, -int max_ents) +unsigned int max_len) {struct scatterlist *sg = sgt->sgl, *sg_last =NULL ; +unsigned int new_len;while (sg) {if (!sg_page (sg))
@@ -61,13 +62,13 @@ qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl,if (!sg)return ERR_PTR (-EINVAL ); -while (new_sgl && sg && max_ents) { -sg_set_page (sg,sg_page (new_sgl), new_sgl->length, - new_sgl->offset); +while (new_sgl && sg && max_len) { + new_len = new_sgl->length > max_len ? max_len : new_sgl->length; +sg_set_page (sg,sg_page (new_sgl), new_len, new_sgl->offset); sg_last = sg; sg =sg_next (sg); new_sgl =sg_next (new_sgl); - max_ents--; + max_len -= new_len; }return sg_last;
drivers/crypto/qce/dma.h +1 -1
@@ -43,6 +43,6 @@ void qce_dma_issue_pending(struct qce_dma_data *dma);int qce_dma_terminate_all (struct qce_dma_data *dma);struct scatterlist *qce_sgtable_add (struct sg_table *sgt,struct scatterlist *sg_add, -int max_ents); +unsigned int max_len);#endif /* _DMA_H_ */
drivers/crypto/qce/skcipher.c +3 -2
@@ -97,13 +97,14 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req)sg_init_one (&rctx->result_sg, qce->dma.result_buf,QCE_RESULT_BUF_SZ ); - sg =qce_sgtable_add (&rctx->dst_tbl, req->dst, rctx->dst_nents -1 ); + sg =qce_sgtable_add (&rctx->dst_tbl, req->dst, req->cryptlen);if (IS_ERR (sg)) { ret =PTR_ERR (sg); goto error_free; } - sg =qce_sgtable_add (&rctx->dst_tbl, &rctx->result_sg,1 ); + sg =qce_sgtable_add (&rctx->dst_tbl, &rctx->result_sg, +QCE_RESULT_BUF_SZ );if (IS_ERR (sg)) { ret =PTR_ERR (sg); goto error_free;
PATCH v4 2/3 crypto: qce - use AES fallback for small requests
Process small blocks using the fallback cipher, as a workaround for an observed failure (DMA-related, apparently) when computing the GCM ghash key. This brings a speed gain as well, since it avoids the latency of using the hardware engine to process small blocks.
Using software for all 16-byte requests would be enough to make GCM work, but to increase performance, a larger threshold would be better. Measuring the performance of supported ciphers with openssl speed, software matches hardware at around 768-1024 bytes.
Considering the 256-bit ciphers, software is 2-3 times faster than qce at 256-bytes, 30% faster at 512, and about even at 768-bytes. With 128-bit keys, the break-even point would be around 1024-bytes.
This adds the 'aes_sw_max_len' parameter, to set the largest request length processed by the software fallback. Its default is being set to 512 bytes, a little lower than the break-even point, to balance the cost in CPU usage.
--
v3 -> v4:
Corrected a missing 'static' declaration of aes_sw_max_len
v2 -> v3:
Corrected style issues pointed out by checkpatch.pl
v1 -> v2:
Changed the threshold from a fixed number to a module parameter
drivers/crypto/Kconfig +23 -0
@@ -685,6 +685,29 @@ choice endchoice +config CRYPTO_DEV_QCE_SW_MAX_LEN + int "Default maximum request size to use software for AES" + depends on CRYPTO_DEV_QCE && CRYPTO_DEV_QCE_SKCIPHER + default 512 + help + This sets the default maximum request size to perform AES requests + using software instead of the crypto engine. It can be changed by + setting the aes_sw_max_len parameter. + + Small blocks are processed faster in software than hardware. + Considering the 256-bit ciphers, software is 2-3 times faster than + qce at 256-bytes, 30% faster at 512, and about even at 768-bytes. + With 128-bit keys, the break-even point would be around 1024-bytes. + + The default is set a little lower, to 512 bytes, to balance the + cost in CPU usage. The minimum recommended setting is 16-bytes + (1 AES block), since AES-GCM will fail if you set it lower. + Setting this to zero will send all requests to the hardware. + + Note that 192-bit keys are not supported by the hardware and are + always processed by the software fallback, and all DES requests + are done by the hardware. + config CRYPTO_DEV_QCOM_RNG tristate "Qualcomm Random Number Generator Driver" depends on ARCH_QCOM || COMPILE_TEST
drivers/crypto/qce/skcipher.c +12 -8
@@ -5,6 +5,7 @@#include <linux/device.h> #include <linux/interrupt.h> +#include <linux/moduleparam.h> #include <linux/types.h> #include <crypto/aes.h> #include <crypto/internal/des.h>
@@ -12,6 +13,13 @@#include "cipher.h" +static unsigned int aes_sw_max_len =CONFIG_CRYPTO_DEV_QCE_SW_MAX_LEN ; +module_param (aes_sw_max_len, uint,0644 ); +MODULE_PARM_DESC (aes_sw_max_len, +"Only use hardware for AES requests larger than this " +"[0=always use hardware; anything <16 breaks AES-GCM; default=" + __stringify(CONFIG_CRYPTO_DEV_QCE_SOFT_THRESHOLD)"]"); +static LIST_HEAD (skcipher_algs);static void qce_skcipher_done (void *data)
@@ -166,15 +174,10 @@ static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key,switch (IS_XTS (flags) ? keylen >>1 : keylen) {case AES_KEYSIZE_128 :case AES_KEYSIZE_256 : +memcpy (ctx->enc_key, key, keylen);break ; -default : - goto fallback; } - ctx->enc_keylen = keylen; -memcpy (ctx->enc_key, key, keylen); -return 0 ; -fallback: ret =crypto_sync_skcipher_setkey (ctx->fallback, key, keylen);if (!ret) ctx->enc_keylen = keylen;
@@ -224,8 +227,9 @@ static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) rctx->flags |= encrypt ?QCE_ENCRYPT :QCE_DECRYPT ; keylen =IS_XTS (rctx->flags) ? ctx->enc_keylen >>1 : ctx->enc_keylen; -if (IS_AES (rctx->flags) && keylen !=AES_KEYSIZE_128 && - keylen !=AES_KEYSIZE_256 ) { +if (IS_AES (rctx->flags) && + ((keylen !=AES_KEYSIZE_128 && keylen !=AES_KEYSIZE_256 ) + || req->cryptlen <= aes_sw_max_len)) {SYNC_SKCIPHER_REQUEST_ON_STACK (subreq, ctx->fallback);skcipher_request_set_sync_tfm (subreq, ctx->fallback);
PATCH v4 3/3 crypto: qce - handle AES-XTS cases that qce fails
In reply to Eneas U de QueirozQCE hangs when presented with an AES-XTS request whose length is larger than QCE_SECTOR_SIZE (512-bytes), and is not a multiple of it. Let the fallback cipher handle them.
--
v3 -> v4
no change
v2 -> v3
Corrected style issues pointed out by checkpatch.pl
v1 -> v2
Patch was first added to the series
drivers/crypto/qce/common.c +0 -2
@@ -15,8 +15,6 @@#include "regs-v5.h" #include "sha.h" -#define QCE_SECTOR_SIZE 512 -static inline u32 qce_read (struct qce_device *qce,u32 offset) {return readl (qce->base + offset);
drivers/crypto/qce/common.h +3 -0
@@ -12,6 +12,9 @@#include <crypto/hash.h> #include <crypto/internal/skcipher.h> +/* xts du size */ +#define QCE_SECTOR_SIZE 512 +/* key size in bytes */ #define QCE_SHA_HMAC_KEY_SIZE 64#define QCE_MAX_CIPHER_KEY_SIZE AES_KEYSIZE_256
drivers/crypto/qce/skcipher.c +6 -1
@@ -227,9 +227,14 @@ static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) rctx->flags |= encrypt ?QCE_ENCRYPT :QCE_DECRYPT ; keylen =IS_XTS (rctx->flags) ? ctx->enc_keylen >>1 : ctx->enc_keylen; +/* qce is hanging when AES-XTS request len > QCE_SECTOR_SIZE and +* is not a multiple of it; pass such requests to the fallback +*/ if (IS_AES (rctx->flags) && ((keylen !=AES_KEYSIZE_128 && keylen !=AES_KEYSIZE_256 ) - || req->cryptlen <= aes_sw_max_len)) { + || req->cryptlen <= aes_sw_max_len) + || (IS_XTS (rctx->flags) && req->cryptlen >QCE_SECTOR_SIZE && + req->cryptlen %QCE_SECTOR_SIZE )) {SYNC_SKCIPHER_REQUEST_ON_STACK (subreq, ctx->fallback);skcipher_request_set_sync_tfm (subreq, ctx->fallback);