Skip to contentSkip to conversation

ARM64: NEON/CE based crypto in atomic contexts

RFC PATCH New Ard Biesheuvel 3 participants 10 messages Oct 7, 2013

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

cover ARM64: NEON/CE based crypto in atomic contexts

I am probably going to be flamed for bringing this up, but here it goes ...

This is more of a request for discussion rather than a request for comments on these patches.

After floating point and SIMD we now have a third class of instructions that use the NEON register file, the AES and SHA instructions that are present in the v8 Crypto Extensions.

This series uses CCMP as an example to make the case for having limited support for the use of the NEON register file in atomic context. CCMP is the encryption standard used in WPA2, and it is based on AES in CCM mode, which is basically both encryption and authentication by passing all the data through AES twice.

The mac80211 layer, which performs this encryption and decryption, does so in a context which does not allow the use of asynchronous ciphers, which in practice means that it uses the C implementation (on ARM64), which I expect to be around an order of magnitude slower than the dedicated instructions(*).

I have included two ways of working around this: patch #3 implements the core AES cipher using only registers q0 and q1. Patch #4 implements the CCM chaining mode using registers q0 - q3. (The significance of the latter is that I expect a certain degree of interleaving to be required to run the AES instructions at full speed, and CCM -while difficult to parallelize- can easily be implemented with a 2-way interleave of the encryption and authentication parts.)

Patch #1 implements the stacking of 4 NEON registers (but note that patch #3 only needs 2 registers). Patch #2 implements emulation of the AES instructions (considering how few of us have access to the Fast Model plugin). Patch #5 modifies the mac80211 code so it relies on the crypto api to supply a CCM implementation rather than cooking up its own (latter is compile tested only and included for reference)

* On ARM, we have the C implementation which runs in ~64 cycles per round and an accelerated synchronous implementation which runs in ~32 cycles per round (on Cortex-A15), but the latter relies heavily on the barrel shifter so its performance is difficult to extrapolate to ARMv8. It should also be noted that the table based C implementation uses 16kB in lookup tables (8 kB each way).

Ard Biesheuvel (5):
ARM64: allow limited use of some NEON registers in exceptions
ARM64: add quick-n-dirty emulation for AES instructions
ARM64: add Crypto Extensions based synchronous core AES cipher
ARM64: add Crypto Extensions based synchronous AES in CCM mode
mac80211: Use CCM crypto driver for CCMP

 arch/arm64/Kconfig              |  14 ++
 arch/arm64/Makefile             |   1 +
 arch/arm64/crypto/Makefile      |  16 ++
 arch/arm64/crypto/aes-sync.c    | 410 ++++++++++++++++++++++++++++++++++++++++
 arch/arm64/crypto/aesce-ccm.S   | 159 ++++++++++++++++
 arch/arm64/crypto/aesce-emu.c   | 221 ++++++++++++++++++++++
 arch/arm64/include/asm/ptrace.h |   3 +
 arch/arm64/include/asm/traps.h  |  10 +
 arch/arm64/kernel/asm-offsets.c |   3 +
 arch/arm64/kernel/entry.S       |  12 +-
 arch/arm64/kernel/traps.c       |  49 +++++
 net/mac80211/Kconfig            |   1 +
 net/mac80211/aes_ccm.c          | 159 +++++-----------
 net/mac80211/aes_ccm.h          |   8 +-
 net/mac80211/key.h              |   2 +-
 net/mac80211/wpa.c              |  21 +-
 16 files changed, 961 insertions(+), 128 deletions(-)
 create mode 100644 arch/arm64/crypto/Makefile
 create mode 100644 arch/arm64/crypto/aes-sync.c
 create mode 100644 arch/arm64/crypto/aesce-ccm.S
 create mode 100644 arch/arm64/crypto/aesce-emu.c
Signature · 1 line
1.8.1.2
#1 lore raw

RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions

Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.

---
 arch/arm64/Kconfig              | 14 ++++++++++++++
 arch/arm64/include/asm/ptrace.h |  3 +++
 arch/arm64/kernel/asm-offsets.c |  3 +++
 arch/arm64/kernel/entry.S       |  8 ++++++++
 4 files changed, 28 insertions(+)
arch/arm64/Kconfig +14 -0
index c044548..b97a458 100644
@@ -98,6 +98,9 @@ config IOMMU_HELPER
 config KERNEL_MODE_NEON
 	def_bool y
 
+config STACK_NEON_REGS_ON_EXCEPTION
+	def_bool n
+
 source "init/Kconfig"
 
 source "kernel/Kconfig.freezer"
@@ -219,6 +222,17 @@ config FORCE_MAX_ZONEORDER
 	default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE)
 	default "11"
 
+config KERNEL_MODE_SYNC_CE_CRYPTO
+	bool "Support for synchronous crypto ciphers using Crypto Extensions"
+	depends on KERNEL_MODE_NEON
+	select STACK_NEON_REGS_ON_EXCEPTION
+	help
+	  This enables support for using ARMv8 Crypto Extensions instructions
+	  in places where sleeping is not allowed. The synchronous ciphers are
+	  only allowed to use the bottom 4 NEON register q0 - q3, as stacking
+	  the entire NEON register file at every exception is too costly.
+
+
 endmenu
 
 menu "Boot options"
arch/arm64/include/asm/ptrace.h +3 -0
index 0dacbbf..17ea483 100644
@@ -103,6 +103,9 @@ struct pt_regs {
 	};
 	u64 orig_x0;
 	u64 syscallno;
+#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION
+	struct { u64 l, h; } qregs[4];
+#endif
 };
 
 #define arch_has_single_step()	(1)
arch/arm64/kernel/asm-offsets.c +3 -0
index 666e231..73c944a 100644
@@ -58,6 +58,9 @@ int main(void)
   DEFINE(S_PC,			offsetof(struct pt_regs, pc));
   DEFINE(S_ORIG_X0,		offsetof(struct pt_regs, orig_x0));
   DEFINE(S_SYSCALLNO,		offsetof(struct pt_regs, syscallno));
+#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION
+  DEFINE(S_QREGS,		offsetof(struct pt_regs, qregs));
+#endif
   DEFINE(S_FRAME_SIZE,		sizeof(struct pt_regs));
   BLANK();
   DEFINE(MM_CONTEXT_ID,		offsetof(struct mm_struct, context.id));
arch/arm64/kernel/entry.S +8 -0
index 3881fd1..c74dcca 100644
@@ -58,6 +58,10 @@
 	push	x4, x5
 	push	x2, x3
 	push	x0, x1
+#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION
+	add	x21, sp, #S_QREGS
+	st1	{v0.16b-v3.16b}, [x21]
+#endif
 	.if	\el == 0
 	mrs	x21, sp_el0
 	.else
@@ -86,6 +90,10 @@
 	.endm
 
 	.macro	kernel_exit, el, ret = 0
+#ifdef CONFIG_STACK_NEON_REGS_ON_EXCEPTION
+	add	x21, sp, #S_QREGS
+	ld1	{v0.16b-v3.16b}, [x21]
+#endif
 	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
 	.if	\el == 0
 	ldr	x23, [sp, #S_SP]		// load return stack pointer
Signature · 1 line
1.8.1.2
#2 lore raw

RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions

Quoted 2 lines · Ard Biesheuvel

Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.

If you want to use Neon in places where you're not allowed to sleep, this also means you are not going to be scheduled away unexpectedly. Why not simply save/restore those Neon regs locally instead? Same goes for interrupt context.

Nicolas

#3 lore raw

RFC PATCH 1/5 ARM64: allow limited use of some NEON registers in exceptions

Quoted 10 lines · Nicolas Pitre
Quoted 2 lines · Ard Biesheuvel

Stack/unstack the bottom 4 NEON registers on exception entry/exit so we can use them in places where we are not allowed to sleep.

If you want to use Neon in places where you're not allowed to sleep, this also means you are not going to be scheduled away unexpectedly. Why not simply save/restore those Neon regs locally instead? Same goes for interrupt context.

Hmm, I guess it is that simple: if you are certain you will not be interrupted between stacking and unstacking, you can basically do whatever you like at any time.
That is quite good news, actually, if that means there is no penalty in general for allowing NEON in atomic context in some cases.

Signature · 1 line
Ard.
#4 lore raw

RFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions

In reply to Ard Biesheuvel
---
 arch/arm64/Makefile            |   1 +
 arch/arm64/crypto/Makefile     |  11 ++
 arch/arm64/crypto/aesce-emu.c  | 221 +++++++++++++++++++++++++++++++++++++++++
 arch/arm64/include/asm/traps.h |  10 ++
 arch/arm64/kernel/entry.S      |   4 +-
 arch/arm64/kernel/traps.c      |  49 +++++++++
 6 files changed, 295 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/crypto/Makefile
 create mode 100644 arch/arm64/crypto/aesce-emu.c
arch/arm64/Makefile +1 -0
index d90cf79..c864bb5 100644
@@ -39,6 +39,7 @@ export	TEXT_OFFSET GZFLAGS
 core-y		+= arch/arm64/kernel/ arch/arm64/mm/
 core-$(CONFIG_KVM) += arch/arm64/kvm/
 core-$(CONFIG_XEN) += arch/arm64/xen/
+core-$(CONFIG_CRYPTO) += arch/arm64/crypto/
 libs-y		:= arch/arm64/lib/ $(libs-y)
 libs-y		+= $(LIBGCC)
 
arch/arm64/crypto/Makefile new file +11 -0
new file mode 100644
index 0000000..f87ec80
@@ -0,0 +1,11 @@
+#
+# linux/arch/arm64/crypto/Makefile
+#
+# Copyright (C) 2013 Linaro Ltd <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+
+obj-y += aesce-emu.o
arch/arm64/crypto/aesce-emu.c new file +221 -0

221 lines not on this page · raw message

arch/arm64/include/asm/traps.h +10 -0
index 10ca8ff..781e50cb2 100644
@@ -27,4 +27,14 @@ static inline int in_exception_text(unsigned long ptr)
 	       ptr < (unsigned long)&__exception_text_end;
 }
 
+struct undef_hook {
+	struct list_head node;
+	u32 instr_mask;
+	u32 instr_val;
+	int (*fn)(struct pt_regs *regs, unsigned int instr);
+};
+
+void register_undef_hook(struct undef_hook *hook);
+void unregister_undef_hook(struct undef_hook *hook);
+
 #endif
arch/arm64/kernel/entry.S +3 -1
index c74dcca..e4d89df 100644
@@ -291,7 +291,9 @@ el1_undef:
 	 * Undefined instruction
 	 */
 	mov	x0, sp
-	b	do_undefinstr
+	bl	do_undefinstr
+
+	kernel_exit 1
 el1_dbg:
 	/*
 	 * Debug exception handling
arch/arm64/kernel/traps.c +49 -0

60 lines not on this page · raw message

Signature · 1 line
1.8.1.2
maintainer
#5 lore raw

RFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions

Quoted 1 line · Ard Biesheuvel

It's missing a commit message with an explanation. In the meantime, NAK ;)

Signature · 1 line
Catalin
#6 lore raw

RFC PATCH 2/5 ARM64: add quick-n-dirty emulation for AES instructions

Quoted 6 lines · Catalin Marinas
Quoted 1 line · Ard Biesheuvel

It's missing a commit message with an explanation. In the meantime, NAK ;)

Well, I just included it for completeness sake, not for upstreaming, but if all it takes to get you to accept it is a commit message, than maybe I will reconsider :-)

#7 lore raw

RFC PATCH 3/5 ARM64: add Crypto Extensions based synchronous core AES cipher

In reply to Ard Biesheuvel

This implements the core AES cipher using the Crypto Extensions, using only NEON register q0 and q1.

---
 arch/arm64/crypto/Makefile   |  5 +++
 arch/arm64/crypto/aes-sync.c | 95 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 arch/arm64/crypto/aes-sync.c
arch/arm64/crypto/Makefile +5 -0
index f87ec80..e598c0a 100644
@@ -9,3 +9,8 @@
 #
 
 obj-y += aesce-emu.o
+
+ifeq ($(CONFIG_KERNEL_MODE_SYNC_CE_CRYPTO),y)
+aesce-sync-y	:= aes-sync.o
+obj-m		+= aesce-sync.o
+endif
arch/arm64/crypto/aes-sync.c new file +95 -0

95 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#8 lore raw

RFC PATCH 4/5 ARM64: add Crypto Extensions based synchronous AES in CCM mode

In reply to Ard Biesheuvel

This implements the CCM AEAD chaining mode for AES using Crypto Extensions instructions.

---
 arch/arm64/crypto/Makefile    |   2 +-
 arch/arm64/crypto/aes-sync.c  | 323 +++++++++++++++++++++++++++++++++++++++++-
 arch/arm64/crypto/aesce-ccm.S | 159 +++++++++++++++++++++
 3 files changed, 479 insertions(+), 5 deletions(-)
 create mode 100644 arch/arm64/crypto/aesce-ccm.S
arch/arm64/crypto/Makefile +1 -1
index e598c0a..dfd1886 100644
@@ -11,6 +11,6 @@
 obj-y += aesce-emu.o
 
 ifeq ($(CONFIG_KERNEL_MODE_SYNC_CE_CRYPTO),y)
-aesce-sync-y	:= aes-sync.o
+aesce-sync-y	:= aes-sync.o aesce-ccm.o
 obj-m		+= aesce-sync.o
 endif
arch/arm64/crypto/aes-sync.c +319 -4

351 lines not on this page · raw message

arch/arm64/crypto/aesce-ccm.S new file +159 -0

159 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#9 lore raw

RFC PATCH 5/5 mac80211: Use CCM crypto driver for CCMP

In reply to Ard Biesheuvel

Let the crypto layer supply the CCM implementation rather than coding it directly on top of the core AES cipher.

---
 net/mac80211/Kconfig   |   1 +
 net/mac80211/aes_ccm.c | 159 +++++++++++++++----------------------------------
 net/mac80211/aes_ccm.h |   8 +--
 net/mac80211/key.h     |   2 +-
 net/mac80211/wpa.c     |  21 ++++---
 5 files changed, 64 insertions(+), 127 deletions(-)
net/mac80211/Kconfig +1 -0
index 62535fe..dc31ec3 100644
@@ -4,6 +4,7 @@ config MAC80211
 	select CRYPTO
 	select CRYPTO_ARC4
 	select CRYPTO_AES
+	select CRYPTO_CCM
 	select CRC32
 	select AVERAGE
 	---help---
net/mac80211/aes_ccm.c +48 -111

182 lines not on this page · raw message

net/mac80211/aes_ccm.h +4 -4
index 5b7d744..797f3f0 100644
@@ -12,13 +12,13 @@
 
 #include <linux/crypto.h>
 
-struct crypto_cipher *ieee80211_aes_key_setup_encrypt(const u8 key[]);
-void ieee80211_aes_ccm_encrypt(struct crypto_cipher *tfm, u8 *scratch,
+struct crypto_aead *ieee80211_aes_key_setup_encrypt(const u8 key[]);
+void ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *scratch,
 			       u8 *data, size_t data_len,
 			       u8 *cdata, u8 *mic);
-int ieee80211_aes_ccm_decrypt(struct crypto_cipher *tfm, u8 *scratch,
+int ieee80211_aes_ccm_decrypt(struct crypto_aead *tfm, u8 *scratch,
 			      u8 *cdata, size_t data_len,
 			      u8 *mic, u8 *data);
-void ieee80211_aes_key_free(struct crypto_cipher *tfm);
+void ieee80211_aes_key_free(struct crypto_aead *tfm);
 
 #endif /* AES_CCM_H */
net/mac80211/key.h +1 -1
index 036d57e..aaae0ed 100644
@@ -83,7 +83,7 @@ struct ieee80211_key {
 			 * Management frames.
 			 */
 			u8 rx_pn[IEEE80211_NUM_TIDS + 1][IEEE80211_CCMP_PN_LEN];
-			struct crypto_cipher *tfm;
+			struct crypto_aead *tfm;
 			u32 replays; /* dot11RSNAStatsCCMPReplays */
 		} ccmp;
 		struct {
net/mac80211/wpa.c +10 -11
index c9edfcb..c3df1d7 100644
@@ -343,7 +343,7 @@ static void ccmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *scratch,
 		data_len -= IEEE80211_CCMP_MIC_LEN;
 
 	/* First block, b_0 */
-	b_0[0] = 0x59; /* flags: Adata: 1, M: 011, L: 001 */
+	b_0[0] = 0x1; /* L == 1 */
 	/* Nonce: Nonce Flags | A2 | PN
 	 * Nonce Flags: Priority (b0..b3) | Management (b4) | Reserved (b5..b7)
 	 */
@@ -355,21 +355,20 @@ static void ccmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *scratch,
 
 	/* AAD (extra authenticate-only data) / masked 802.11 header
 	 * FC | A1 | A2 | A3 | SC | [A4] | [QC] */
-	put_unaligned_be16(len_a, &aad[0]);
-	put_unaligned(mask_fc, (__le16 *)&aad[2]);
-	memcpy(&aad[4], &hdr->addr1, 3 * ETH_ALEN);
+	put_unaligned(mask_fc, (__le16 *)&aad[0]);
+	memcpy(&aad[2], &hdr->addr1, 3 * ETH_ALEN);
 
 	/* Mask Seq#, leave Frag# */
-	aad[22] = *((u8 *) &hdr->seq_ctrl) & 0x0f;
-	aad[23] = 0;
+	aad[20] = *((u8 *) &hdr->seq_ctrl) & 0x0f;
+	aad[21] = 0;
 
 	if (a4_included) {
-		memcpy(&aad[24], hdr->addr4, ETH_ALEN);
-		aad[30] = qos_tid;
-		aad[31] = 0;
+		memcpy(&aad[22], hdr->addr4, ETH_ALEN);
+		aad[28] = qos_tid;
+		aad[29] = 0;
 	} else {
-		memset(&aad[24], 0, ETH_ALEN + IEEE80211_QOS_CTL_LEN);
-		aad[24] = qos_tid;
+		memset(&aad[22], 0, ETH_ALEN + IEEE80211_QOS_CTL_LEN);
+		aad[22] = qos_tid;
 	}
 }
 
Signature · 1 line
1.8.1.2