crypto: Add AES-NI support for more modes
Subsystems (2)
PATCH 3/3 crypto: Add AES-NI support for more modes
Because kernel_fpu_begin() and kernel_fpu_end() operations are too slow, the performance gain of general mode implementation + aes-aesni is almost all compensated.
The AES-NI support for more modes are implemented as follow:
- Add a new AES algorithm implementation named __aes-aesni without kernel_fpu_begin/end()
- Use fpu(<mode>(AES)) to provide kenrel_fpu_begin/end() invoking
- Add <mode>(AES) ablkcipher, which uses cryptd(fpu(<mode>(AES))) to defer cryption to cryptd context in soft_irq context.
Now the ctr, lrw, pcbc and xts support are added.
Performance testing based on dm-crypt shows that cryption time can be reduced to 50% of general mode implementation + aes-aesni implementation.
v2: Add description of mode acceleration support in Kconfig
v3: Fix some bugs of CTR block size, LRW and XTS min/max key size.
--- arch/x86/crypto/aesni-intel_glue.c | 267 ++++++++++++++++++++++++++++++++++++- crypto/Kconfig | 5 2 files changed, 271 insertions(+), 1 deletion(-)
arch/x86/crypto/aesni-intel_glue.c +266 -1
@@ -21,6 +21,22 @@#include <asm/i387.h> #include <asm/aes.h> +#if defined(CONFIG_CRYPTO_CTR ) || defined(CONFIG_CRYPTO_CTR_MODULE ) +#define HAS_CTR +#endif + +#if defined(CONFIG_CRYPTO_LRW ) || defined(CONFIG_CRYPTO_LRW_MODULE ) +#define HAS_LRW +#endif + +#if defined(CONFIG_CRYPTO_PCBC ) || defined(CONFIG_CRYPTO_PCBC_MODULE ) +#define HAS_PCBC +#endif + +#if defined(CONFIG_CRYPTO_XTS ) || defined(CONFIG_CRYPTO_XTS_MODULE ) +#define HAS_XTS +#endif +struct async_aes_ctx {struct cryptd_ablkcipher *cryptd_tfm; };
@@ -137,6 +153,41 @@ static struct crypto_alg aesni_alg = { } }; +static void __aes_encrypt (struct crypto_tfm *tfm,u8 *dst,const u8 *src) +{ +struct crypto_aes_ctx *ctx =aes_ctx (crypto_tfm_ctx (tfm)); + +aesni_enc (ctx, dst, src); +} + +static void __aes_decrypt (struct crypto_tfm *tfm,u8 *dst,const u8 *src) +{ +struct crypto_aes_ctx *ctx =aes_ctx (crypto_tfm_ctx (tfm)); + +aesni_dec (ctx, dst, src); +} + +static struct crypto_alg __aesni_alg = { + .cra_name ="__aes-aesni" , + .cra_driver_name ="__driver-aes-aesni" , + .cra_priority =0 , + .cra_flags =CRYPTO_ALG_TYPE_CIPHER , + .cra_blocksize =AES_BLOCK_SIZE , + .cra_ctxsize =sizeof (struct crypto_aes_ctx )+AESNI_ALIGN -1 , + .cra_alignmask =0 , + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (__aesni_alg.cra_list), + .cra_u = { + .cipher = { + .cia_min_keysize =AES_MIN_KEY_SIZE , + .cia_max_keysize =AES_MAX_KEY_SIZE , + .cia_setkey = aes_set_key, + .cia_encrypt = __aes_encrypt, + .cia_decrypt = __aes_decrypt + } + } +}; +static int ecb_encrypt (struct blkcipher_desc *desc,struct scatterlist *dst,struct scatterlist *src,unsigned int nbytes)
@@ -277,8 +328,16 @@ static int ablk_set_key(struct crypto_ab unsigned int key_len) {struct async_aes_ctx *ctx =crypto_ablkcipher_ctx (tfm); +struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; +int err; -return crypto_ablkcipher_setkey (&ctx->cryptd_tfm->base, key, key_len); +crypto_ablkcipher_clear_flags (child,CRYPTO_TFM_REQ_MASK ); +crypto_ablkcipher_set_flags (child,crypto_ablkcipher_get_flags (tfm) + &CRYPTO_TFM_REQ_MASK ); + err =crypto_ablkcipher_setkey (child, key, key_len); +crypto_ablkcipher_set_flags (tfm,crypto_ablkcipher_get_flags (child) + &CRYPTO_TFM_RES_MASK ); +return err; }static int ablk_encrypt (struct ablkcipher_request *req)
@@ -411,6 +470,163 @@ static struct crypto_alg ablk_cbc_alg = }, }; +#ifdef HAS_CTR +static int ablk_ctr_init (struct crypto_tfm *tfm) +{ +struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm =cryptd_alloc_ablkcipher ("fpu(ctr(__driver-aes-aesni))" , +0 ,0 ); +if (IS_ERR (cryptd_tfm)) +return PTR_ERR (cryptd_tfm); +ablk_init_common (tfm, cryptd_tfm); +return 0 ; +} + +static struct crypto_alg ablk_ctr_alg = { + .cra_name ="ctr(aes)" , + .cra_driver_name ="ctr-aes-aesni" , + .cra_priority =400 , + .cra_flags =CRYPTO_ALG_TYPE_ABLKCIPHER |CRYPTO_ALG_ASYNC , + .cra_blocksize =1 , + .cra_ctxsize =sizeof (struct async_aes_ctx ), + .cra_alignmask =0 , + .cra_type = &crypto_ablkcipher_type, + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (ablk_ctr_alg.cra_list), + .cra_init = ablk_ctr_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize =AES_MIN_KEY_SIZE , + .max_keysize =AES_MAX_KEY_SIZE , + .ivsize =AES_BLOCK_SIZE , + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + .geniv ="chainiv" , + }, + }, +}; +#endif + +#ifdef HAS_LRW +static int ablk_lrw_init (struct crypto_tfm *tfm) +{ +struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm =cryptd_alloc_ablkcipher ("fpu(lrw(__driver-aes-aesni))" , +0 ,0 ); +if (IS_ERR (cryptd_tfm)) +return PTR_ERR (cryptd_tfm); +ablk_init_common (tfm, cryptd_tfm); +return 0 ; +} + +static struct crypto_alg ablk_lrw_alg = { + .cra_name ="lrw(aes)" , + .cra_driver_name ="lrw-aes-aesni" , + .cra_priority =400 , + .cra_flags =CRYPTO_ALG_TYPE_ABLKCIPHER |CRYPTO_ALG_ASYNC , + .cra_blocksize =AES_BLOCK_SIZE , + .cra_ctxsize =sizeof (struct async_aes_ctx ), + .cra_alignmask =0 , + .cra_type = &crypto_ablkcipher_type, + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (ablk_lrw_alg.cra_list), + .cra_init = ablk_lrw_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize =AES_MIN_KEY_SIZE +AES_BLOCK_SIZE , + .max_keysize =AES_MAX_KEY_SIZE +AES_BLOCK_SIZE , + .ivsize =AES_BLOCK_SIZE , + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}; +#endif + +#ifdef HAS_PCBC +static int ablk_pcbc_init (struct crypto_tfm *tfm) +{ +struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm =cryptd_alloc_ablkcipher ("fpu(pcbc(__driver-aes-aesni))" , +0 ,0 ); +if (IS_ERR (cryptd_tfm)) +return PTR_ERR (cryptd_tfm); +ablk_init_common (tfm, cryptd_tfm); +return 0 ; +} + +static struct crypto_alg ablk_pcbc_alg = { + .cra_name ="pcbc(aes)" , + .cra_driver_name ="pcbc-aes-aesni" , + .cra_priority =400 , + .cra_flags =CRYPTO_ALG_TYPE_ABLKCIPHER |CRYPTO_ALG_ASYNC , + .cra_blocksize =AES_BLOCK_SIZE , + .cra_ctxsize =sizeof (struct async_aes_ctx ), + .cra_alignmask =0 , + .cra_type = &crypto_ablkcipher_type, + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (ablk_pcbc_alg.cra_list), + .cra_init = ablk_pcbc_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize =AES_MIN_KEY_SIZE , + .max_keysize =AES_MAX_KEY_SIZE , + .ivsize =AES_BLOCK_SIZE , + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}; +#endif + +#ifdef HAS_XTS +static int ablk_xts_init (struct crypto_tfm *tfm) +{ +struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm =cryptd_alloc_ablkcipher ("fpu(xts(__driver-aes-aesni))" , +0 ,0 ); +if (IS_ERR (cryptd_tfm)) +return PTR_ERR (cryptd_tfm); +ablk_init_common (tfm, cryptd_tfm); +return 0 ; +} + +static struct crypto_alg ablk_xts_alg = { + .cra_name ="xts(aes)" , + .cra_driver_name ="xts-aes-aesni" , + .cra_priority =400 , + .cra_flags =CRYPTO_ALG_TYPE_ABLKCIPHER |CRYPTO_ALG_ASYNC , + .cra_blocksize =AES_BLOCK_SIZE , + .cra_ctxsize =sizeof (struct async_aes_ctx ), + .cra_alignmask =0 , + .cra_type = &crypto_ablkcipher_type, + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (ablk_xts_alg.cra_list), + .cra_init = ablk_xts_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize =2 *AES_MIN_KEY_SIZE , + .max_keysize =2 *AES_MAX_KEY_SIZE , + .ivsize =AES_BLOCK_SIZE , + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}; +#endif +static int __init aesni_init (void ) {int err;
@@ -421,6 +637,8 @@ static int __init aesni_init(void) }if ((err =crypto_register_alg (&aesni_alg))) goto aes_err; +if ((err =crypto_register_alg (&__aesni_alg))) + goto __aes_err;if ((err =crypto_register_alg (&blk_ecb_alg))) goto blk_ecb_err;if ((err =crypto_register_alg (&blk_cbc_alg)))
@@ -429,9 +647,41 @@ static int __init aesni_init(void) goto ablk_ecb_err;if ((err =crypto_register_alg (&ablk_cbc_alg))) goto ablk_cbc_err; +#ifdef HAS_CTR +if ((err =crypto_register_alg (&ablk_ctr_alg))) + goto ablk_ctr_err; +#endif +#ifdef HAS_LRW +if ((err =crypto_register_alg (&ablk_lrw_alg))) + goto ablk_lrw_err; +#endif +#ifdef HAS_PCBC +if ((err =crypto_register_alg (&ablk_pcbc_alg))) + goto ablk_pcbc_err; +#endif +#ifdef HAS_XTS +if ((err =crypto_register_alg (&ablk_xts_alg))) + goto ablk_xts_err; +#endif return err; +#ifdef HAS_XTS +ablk_xts_err: +#endif +#ifdef HAS_PCBC +crypto_unregister_alg (&ablk_pcbc_alg); +ablk_pcbc_err: +#endif +#ifdef HAS_LRW +crypto_unregister_alg (&ablk_lrw_alg); +ablk_lrw_err: +#endif +#ifdef HAS_CTR +crypto_unregister_alg (&ablk_ctr_alg); +ablk_ctr_err: +#endif +crypto_unregister_alg (&ablk_cbc_alg); ablk_cbc_err:crypto_unregister_alg (&ablk_ecb_alg); ablk_ecb_err:
@@ -439,6 +689,8 @@ ablk_ecb_err: blk_cbc_err:crypto_unregister_alg (&blk_ecb_alg); blk_ecb_err: +crypto_unregister_alg (&__aesni_alg); +__aes_err:crypto_unregister_alg (&aesni_alg); aes_err:return err;
@@ -446,10 +698,23 @@ aes_err: static void __exit aesni_exit(void) { +#ifdef HAS_XTS +crypto_unregister_alg (&ablk_xts_alg); +#endif +#ifdef HAS_PCBC +crypto_unregister_alg (&ablk_pcbc_alg); +#endif +#ifdef HAS_LRW +crypto_unregister_alg (&ablk_lrw_alg); +#endif +#ifdef HAS_CTR +crypto_unregister_alg (&ablk_ctr_alg); +#endif crypto_unregister_alg (&ablk_cbc_alg);crypto_unregister_alg (&ablk_ecb_alg);crypto_unregister_alg (&blk_cbc_alg);crypto_unregister_alg (&blk_ecb_alg); +crypto_unregister_alg (&__aesni_alg);crypto_unregister_alg (&aesni_alg); }
crypto/Kconfig +5 -0
@@ -491,6 +491,7 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_AES_X86_64 select CRYPTO_CRYPTD select CRYPTO_ALGAPI + select CRYPTO_FPU help Use Intel AES-NI instructions for AES algorithm.
@@ -510,6 +511,10 @@ config CRYPTO_AES_NI_INTEL See <http://csrc.nist.gov/encryption/aes/> for more information. + In addition to AES cipher algorithm support, the + acceleration for some popular block cipher mode is supported + too, including ECB, CBC, CTR, LRW, PCBC, XTS. + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI