ARM64: NEON/CE based crypto in atomic contexts
Subsystems (4)
cover ARM64: NEON/CE based crypto in atomic contexts
I am probably going to be flamed for bringing this up, but here it goes ...
This is more of a request for discussion rather than a request for comments on these patches.
After floating point and SIMD we now have a third class of instructions that use the NEON register file, the AES and SHA instructions that are present in the v8 Crypto Extensions.
This series uses CCMP as an example to make the case for having limited support for the use of the NEON register file in atomic context. CCMP is the encryption standard used in WPA2, and it is based on AES in CCM mode, which is basically both encryption and authentication by passing all the data through AES twice.
The mac80211 layer, which performs this encryption and decryption, does so in a context which does not allow the use of asynchronous ciphers, which in practice means that it uses the C implementation (on ARM64), which I expect to be around an order of magnitude slower than the dedicated instructions(*).
I have included two ways of working around this: patch #3 implements the core AES cipher using only registers q0 and q1. Patch #4 implements the CCM chaining mode using registers q0 - q3. (The significance of the latter is that I expect a certain degree of interleaving to be required to run the AES instructions at full speed, and CCM -while difficult to parallelize- can easily be implemented with a 2-way interleave of the encryption and authentication parts.)
Patch #1 implements the stacking of 4 NEON registers (but note that patch #3 only needs 2 registers). Patch #2 implements emulation of the AES instructions (considering how few of us have access to the Fast Model plugin). Patch #5 modifies the mac80211 code so it relies on the crypto api to supply a CCM implementation rather than cooking up its own (latter is compile tested only and included for reference)
* On ARM, we have the C implementation which runs in ~64 cycles per round and an accelerated synchronous implementation which runs in ~32 cycles per round (on Cortex-A15), but the latter relies heavily on the barrel shifter so its performance is difficult to extrapolate to ARMv8. It should also be noted that the table based C implementation uses 16kB in lookup tables (8 kB each way).
Ard Biesheuvel (5):
ARM64: allow limited use of some NEON registers in exceptions
ARM64: add quick-n-dirty emulation for AES instructions
ARM64: add Crypto Extensions based synchronous core AES cipher
ARM64: add Crypto Extensions based synchronous AES in CCM mode
mac80211: Use CCM crypto driver for CCMP
arch/arm64/Kconfig | 14 ++ arch/arm64/Makefile | 1 + arch/arm64/crypto/Makefile | 16 ++ arch/arm64/crypto/aes-sync.c | 410 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/crypto/aesce-ccm.S | 159 ++++++++++++++++ arch/arm64/crypto/aesce-emu.c | 221 ++++++++++++++++++++++ arch/arm64/include/asm/ptrace.h | 3 + arch/arm64/include/asm/traps.h | 10 + arch/arm64/kernel/asm-offsets.c | 3 + arch/arm64/kernel/entry.S | 12 +- arch/arm64/kernel/traps.c | 49 +++++ net/mac80211/Kconfig | 1 + net/mac80211/aes_ccm.c | 159 +++++----------- net/mac80211/aes_ccm.h | 8 +- net/mac80211/key.h | 2 +- net/mac80211/wpa.c | 21 +- 16 files changed, 961 insertions(+), 128 deletions(-) create mode 100644 arch/arm64/crypto/Makefile create mode 100644 arch/arm64/crypto/aes-sync.c create mode 100644 arch/arm64/crypto/aesce-ccm.S create mode 100644 arch/arm64/crypto/aesce-emu.c
Signature · 1 line
1.8.1.2
RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions
Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.
--- arch/arm64/Kconfig | 14 ++++++++++++++ arch/arm64/include/asm/ptrace.h | 3 +++ arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 8 ++++++++ 4 files changed, 28 insertions(+)
arch/arm64/Kconfig +14 -0
@@ -98,6 +98,9 @@ config IOMMU_HELPER config KERNEL_MODE_NEON def_bool y +config STACK_NEON_REGS_ON_EXCEPTION + def_bool n + source "init/Kconfig" source "kernel/Kconfig.freezer"
@@ -219,6 +222,17 @@ config FORCE_MAX_ZONEORDER default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE) default "11" +config KERNEL_MODE_SYNC_CE_CRYPTO + bool "Support for synchronous crypto ciphers using Crypto Extensions" + depends on KERNEL_MODE_NEON + select STACK_NEON_REGS_ON_EXCEPTION + help + This enables support for using ARMv8 Crypto Extensions instructions + in places where sleeping is not allowed. The synchronous ciphers are + only allowed to use the bottom 4 NEON register q0 - q3, as stacking + the entire NEON register file at every exception is too costly. + + endmenu menu "Boot options"
arch/arm64/include/asm/ptrace.h +3 -0
@@ -103,6 +103,9 @@ struct pt_regs { };u64 orig_x0;u64 syscallno; +#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION +struct {u64 l, h; } qregs[4 ]; +#endif };#define arch_has_single_step () (1)
arch/arm64/kernel/asm-offsets.c +3 -0
@@ -58,6 +58,9 @@ int main(void)DEFINE (S_PC , offsetof(struct pt_regs , pc));DEFINE (S_ORIG_X0 , offsetof(struct pt_regs , orig_x0));DEFINE (S_SYSCALLNO , offsetof(struct pt_regs , syscallno)); +#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION +DEFINE (S_QREGS , offsetof(struct pt_regs , qregs)); +#endif DEFINE (S_FRAME_SIZE ,sizeof (struct pt_regs ));BLANK (); DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id));
arch/arm64/kernel/entry.S +8 -0
@@ -58,6 +58,10 @@ push x4, x5 push x2, x3 push x0, x1 +#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION + add x21, sp, #S_QREGS + st1 {v0.16b-v3.16b}, [x21] +#endif .if \el == 0 mrs x21, sp_el0 .else
@@ -86,6 +90,10 @@ .endm .macro kernel_exit, el, ret = 0 +#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION + add x21, sp, #S_QREGS + ld1 {v0.16b-v3.16b}, [x21] +#endif ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ldr x23, [sp, #S_SP] // load return stack pointer
Signature · 1 line
1.8.1.2
RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions
Quoted 2 lines · Ard Biesheuvel
Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.
If you want to use Neon in places where you're not allowed to sleep, this also means you are not going to be scheduled away unexpectedly. Why not simply save/restore those Neon regs locally instead? Same goes for interrupt context.
Nicolas
RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions
Quoted 10 lines · Nicolas Pitre
Quoted 2 lines · Ard Biesheuvel
Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.
If you want to use Neon in places where you're not allowed to sleep, this also means you are not going to be scheduled away unexpectedly. Why not simply save/restore those Neon regs locally instead? Same goes for interrupt context.
Hmm, I guess it is that simple: if you are certain you will not be interrupted between stacking and unstacking, you can basically do whatever you like at any time.
That is quite good news, actually, if that means there is no penalty in general for allowing NEON in atomic context in some cases.
Signature · 1 line
Ard.
RFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions
In reply to Ard Biesheuvel--- arch/arm64/Makefile | 1 + arch/arm64/crypto/Makefile | 11 ++ arch/arm64/crypto/aesce-emu.c | 221 +++++++++++++++++++++++++++++++++++++++++ arch/arm64/include/asm/traps.h | 10 ++ arch/arm64/kernel/entry.S | 4 +- arch/arm64/kernel/traps.c | 49 +++++++++ 6 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/crypto/Makefile create mode 100644 arch/arm64/crypto/aesce-emu.c
arch/arm64/Makefile +1 -0
@@ -39,6 +39,7 @@ export TEXT_OFFSET GZFLAGS core-y += arch/arm64/kernel/ arch/arm64/mm/ core-$(CONFIG_KVM) += arch/arm64/kvm/ core-$(CONFIG_XEN) += arch/arm64/xen/ +core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ libs-y := arch/arm64/lib/ $(libs-y) libs-y += $(LIBGCC)
arch/arm64/crypto/Makefile new file +11 -0
@@ -0,0 +1,11 @@ +# +# linux/arch/arm64/crypto/Makefile +# +# Copyright (C) 2013 Linaro Ltd <[email protected]> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# + +obj-y += aesce-emu.o
arch/arm64/crypto/aesce-emu.c new file +221 -0
221 lines not on this page · raw message
arch/arm64/include/asm/traps.h +10 -0
@@ -27,4 +27,14 @@ static inline int in_exception_text(unsigned long ptr) ptr < (unsigned long)&__exception_text_end; } +struct undef_hook { +struct list_head node; +u32 instr_mask; +u32 instr_val; +int (*fn)(struct pt_regs *regs,unsigned int instr); +}; + +void register_undef_hook (struct undef_hook *hook); +void unregister_undef_hook (struct undef_hook *hook); +#endif
arch/arm64/kernel/entry.S +3 -1
@@ -291,7 +291,9 @@ el1_undef: * Undefined instruction */ mov x0, sp - b do_undefinstr + bl do_undefinstr + + kernel_exit 1 el1_dbg: /* * Debug exception handling
arch/arm64/kernel/traps.c +49 -0
60 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Catalin Marinas
maintainerRFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions
Quoted 1 line · Ard Biesheuvel
Signed-off-by Ard Biesheuvel <[email protected]>
It's missing a commit message with an explanation. In the meantime, NAK ;)
Signature · 1 line
Catalin
RFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions
Quoted 6 lines · Catalin Marinas
Quoted 1 line · Ard Biesheuvel
Signed-off-by Ard Biesheuvel <[email protected]>It's missing a commit message with an explanation. In the meantime, NAK ;)
Well, I just included it for completeness sake, not for upstreaming, but if all it takes to get you to accept it is a commit message, than maybe I will reconsider :-)
RFC PATCH 3/5 ARM64: add Crypto Extensions based synchronous core AES cipher
In reply to Ard BiesheuvelThis implements the core AES cipher using the Crypto Extensions, using only NEON register q0 and q1.
--- arch/arm64/crypto/Makefile | 5 +++ arch/arm64/crypto/aes-sync.c | 95 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 arch/arm64/crypto/aes-sync.c
arch/arm64/crypto/Makefile +5 -0
@@ -9,3 +9,8 @@ # obj-y += aesce-emu.o + +ifeq ($(CONFIG_KERNEL_MODE_SYNC_CE_CRYPTO),y) +aesce-sync-y := aes-sync.o +obj-m += aesce-sync.o +endif
arch/arm64/crypto/aes-sync.c new file +95 -0
95 lines not on this page · raw message
Signature · 1 line
1.8.1.2
RFC PATCH 4/5 ARM64: add Crypto Extensions based synchronous AES in CCM mode
In reply to Ard BiesheuvelThis implements the CCM AEAD chaining mode for AES using Crypto Extensions instructions.
--- arch/arm64/crypto/Makefile | 2 +- arch/arm64/crypto/aes-sync.c | 323 +++++++++++++++++++++++++++++++++++++++++- arch/arm64/crypto/aesce-ccm.S | 159 +++++++++++++++++++++ 3 files changed, 479 insertions(+), 5 deletions(-) create mode 100644 arch/arm64/crypto/aesce-ccm.S
arch/arm64/crypto/Makefile +1 -1
@@ -11,6 +11,6 @@ obj-y += aesce-emu.o ifeq ($(CONFIG_KERNEL_MODE_SYNC_CE_CRYPTO),y) -aesce-sync-y := aes-sync.o +aesce-sync-y := aes-sync.o aesce-ccm.o obj-m += aesce-sync.o endif
arch/arm64/crypto/aes-sync.c +319 -4
351 lines not on this page · raw message
arch/arm64/crypto/aesce-ccm.S new file +159 -0
159 lines not on this page · raw message
Signature · 1 line
1.8.1.2
RFC PATCH 5/5 mac80211: Use CCM crypto driver for CCMP
In reply to Ard BiesheuvelLet the crypto layer supply the CCM implementation rather than coding it directly on top of the core AES cipher.
--- net/mac80211/Kconfig | 1 + net/mac80211/aes_ccm.c | 159 +++++++++++++++---------------------------------- net/mac80211/aes_ccm.h | 8 +-- net/mac80211/key.h | 2 +- net/mac80211/wpa.c | 21 ++++--- 5 files changed, 64 insertions(+), 127 deletions(-)
net/mac80211/Kconfig +1 -0
@@ -4,6 +4,7 @@ config MAC80211 select CRYPTO select CRYPTO_ARC4 select CRYPTO_AES + select CRYPTO_CCM select CRC32 select AVERAGE ---help---
net/mac80211/aes_ccm.c +48 -111
182 lines not on this page · raw message
net/mac80211/aes_ccm.h +4 -4
@@ -12,13 +12,13 @@#include <linux/crypto.h> -struct crypto_cipher *ieee80211_aes_key_setup_encrypt (const u8 key[]); -void ieee80211_aes_ccm_encrypt (struct crypto_cipher *tfm,u8 *scratch, +struct crypto_aead *ieee80211_aes_key_setup_encrypt (const u8 key[]); +void ieee80211_aes_ccm_encrypt (struct crypto_aead *tfm,u8 *scratch,u8 *data,size_t data_len,u8 *cdata,u8 *mic); -int ieee80211_aes_ccm_decrypt (struct crypto_cipher *tfm,u8 *scratch, +int ieee80211_aes_ccm_decrypt (struct crypto_aead *tfm,u8 *scratch,u8 *cdata,size_t data_len,u8 *mic,u8 *data); -void ieee80211_aes_key_free (struct crypto_cipher *tfm); +void ieee80211_aes_key_free (struct crypto_aead *tfm);#endif /* AES_CCM_H */
net/mac80211/key.h +1 -1
@@ -83,7 +83,7 @@ struct ieee80211_key {* Management frames. */ u8 rx_pn[IEEE80211_NUM_TIDS +1 ][IEEE80211_CCMP_PN_LEN ]; -struct crypto_cipher *tfm; +struct crypto_aead *tfm;u32 replays;/* dot11RSNAStatsCCMPReplays */ } ccmp;struct {
net/mac80211/wpa.c +10 -11
@@ -343,7 +343,7 @@ static void ccmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *scratch, data_len -=IEEE80211_CCMP_MIC_LEN ;/* First block, b_0 */ - b_0[0 ] =0x59 ;/* flags: Adata: 1, M: 011, L: 001 */ + b_0[0 ] =0x1 ;/* L == 1 */ /* Nonce: Nonce Flags | A2 | PN * Nonce Flags: Priority (b0..b3) | Management (b4) | Reserved (b5..b7) */
@@ -355,21 +355,20 @@ static void ccmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *scratch,/* AAD (extra authenticate-only data) / masked 802.11 header * FC | A1 | A2 | A3 | SC | [A4] | [QC] */ -put_unaligned_be16 (len_a, &aad[0 ]); -put_unaligned (mask_fc, (__le16 *)&aad[2 ]); -memcpy (&aad[4 ], &hdr->addr1,3 *ETH_ALEN ); +put_unaligned (mask_fc, (__le16 *)&aad[0 ]); +memcpy (&aad[2 ], &hdr->addr1,3 *ETH_ALEN );/* Mask Seq#, leave Frag# */ - aad[22 ] = *((u8 *) &hdr->seq_ctrl) &0x0f ; - aad[23 ] =0 ; + aad[20 ] = *((u8 *) &hdr->seq_ctrl) &0x0f ; + aad[21 ] =0 ;if (a4_included) { -memcpy (&aad[24 ], hdr->addr4,ETH_ALEN ); - aad[30 ] = qos_tid; - aad[31 ] =0 ; +memcpy (&aad[22 ], hdr->addr4,ETH_ALEN ); + aad[28 ] = qos_tid; + aad[29 ] =0 ; }else { -memset (&aad[24 ],0 ,ETH_ALEN +IEEE80211_QOS_CTL_LEN ); - aad[24 ] = qos_tid; +memset (&aad[22 ],0 ,ETH_ALEN +IEEE80211_QOS_CTL_LEN ); + aad[22 ] = qos_tid; } }
Signature · 1 line
1.8.1.2