crypto: x86/aesni - implement accelerated CBCMAC ahash
Subsystems (2)
PATCH v2 1/1 crypto: x86/aesni - implement accelerated CBCMAC ahash
Ben reports that CCM using AES-NI instructions performs pathologically poorly, which is due to the overhead of preserving/restoring the SIMD state, which is repeated after every 16 bytes of input when executing the CBCMAC portion of the algorithm.
So let's adopt the arm64 implementation of cbcmac(aes), which takes care to only preserve/restore the SIMD state after processing the whole input, and convert it into an ahash implementation, which gives us more control over the way FPU preserve/restore can be amortized over the entire request.
Cc Ben Greear <[email protected]>
Signed-off-by Ard Biesheuvel <[email protected]>
---
v2:
- use a ahash not shash, so we can process most of the cbcmac input in one go
- drop cmac and xcbc, which are only used in shash form by other parts of the kernel
arch/x86/crypto/Makefile | 2 +- arch/x86/crypto/aesni-intel.h | 21 ++ arch/x86/crypto/aesni-intel_asm.S | 38 ++++ arch/x86/crypto/aesni-intel_glue.c | 24 +-- arch/x86/crypto/aesni-intel_mac.c | 206 ++++++++++++++++++++ 5 files changed, 275 insertions(+), 16 deletions(-)
arch/x86/crypto/Makefile +1 -1
@@ -51,7 +51,7 @@ chacha-x86_64-y := chacha-avx2-x86_64.o chacha-ssse3-x86_64.o chacha_glue.o chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o -aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o +aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o aesni-intel_mac.o aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
arch/x86/crypto/aesni-intel.h new file +21 -0
@@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/internal/hash.h> + +#define AESNI_ALIGN 16 +#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) +#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) + +extern struct ahash_alg aesni_cbcmac_alg; + +static inline struct crypto_aes_ctx *aes_ctx (void *raw_ctx) +{ +unsigned long addr = (unsigned long )raw_ctx; +unsigned long align =AESNI_ALIGN ; + +if (align <=crypto_tfm_ctx_alignment ()) + align =1 ; +return (struct crypto_aes_ctx *)ALIGN (addr, align); +}
arch/x86/crypto/aesni-intel_asm.S +38 -0
@@ -2578,6 +2578,44 @@ SYM_FUNC_START(aesni_cbc_dec) ret SYM_FUNC_END(aesni_cbc_dec) +/* + * void aesni_cbc_mac(struct crypto_aes_ctx *ctx, u8 *dgst, const u8 *src, + * size_t len) + */ +SYM_FUNC_START(aesni_cbc_mac) + FRAME_BEGIN +#ifndef __x86_64__ + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+16)(%esp), KEYP # ctx + movl (FRAME_OFFSET+20)(%esp), OUTP # dgst + movl (FRAME_OFFSET+24)(%esp), INP # src + movl (FRAME_OFFSET+28)(%esp), LEN # len +#endif + sub $1, LEN + jb .Lcbc_mac_ret + mov 480(KEYP), KLEN + movups (OUTP), STATE # load digest as initial state +.align 4 +.Lcbc_mac_loop: + movups (INP), IN # load input + pxor IN, STATE + call _aesni_enc1 + add $16, INP + sub $1, LEN + jge .Lcbc_mac_loop + movups STATE, (OUTP) # store output digest +.Lcbc_mac_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN +#endif + FRAME_END + ret +SYM_FUNC_END(aesni_cbc_mac) + #ifdef __x86_64__ .pushsection .rodata .align 16
arch/x86/crypto/aesni-intel_glue.c +9 -15
@@ -19,8 +19,6 @@#include <linux/types.h> #include <linux/module.h> #include <linux/err.h> -#include <crypto/algapi.h> -#include <crypto/aes.h> #include <crypto/ctr.h> #include <crypto/b128ops.h> #include <crypto/gcm.h>
@@ -37,13 +35,11 @@#include <asm/crypto/glue_helper.h> #endif +#include "aesni-intel.h" -#define AESNI_ALIGN 16#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))#define RFC4106_HASH_SUBKEY_SIZE 16 -#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) -#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)/* This data is stored at the end of the crypto_tfm struct.
@@ -296,16 +292,6 @@ generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) }#endif -static inline struct crypto_aes_ctx *aes_ctx (void *raw_ctx) -{ -unsigned long addr = (unsigned long )raw_ctx; -unsigned long align =AESNI_ALIGN ; - -if (align <=crypto_tfm_ctx_alignment ()) - align =1 ; -return (struct crypto_aes_ctx *)ALIGN (addr, align); -} -static int aes_set_key_common (struct crypto_tfm *tfm,void *raw_ctx,const u8 *in_key,unsigned int key_len) {
@@ -1098,8 +1084,15 @@ static int __init aesni_init(void)if (err) goto unregister_skciphers; + err =crypto_register_ahash (&aesni_cbcmac_alg); +if (err) + goto unregister_aeads; +return 0 ; +unregister_aeads: +simd_unregister_aeads (aesni_aeads,ARRAY_SIZE (aesni_aeads), + aesni_simd_aeads); unregister_skciphers:simd_unregister_skciphers (aesni_skciphers,ARRAY_SIZE (aesni_skciphers), aesni_simd_skciphers);
@@ -1110,6 +1103,7 @@ static int __init aesni_init(void) static void __exit aesni_exit(void) { +crypto_unregister_ahash (&aesni_cbcmac_alg);simd_unregister_aeads (aesni_aeads,ARRAY_SIZE (aesni_aeads), aesni_simd_aeads); simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
arch/x86/crypto/aesni-intel_mac.c new file +206 -0
206 lines not on this page · raw message
Signature · 1 line
2.17.1