crypto: arm/aes-neonbs - some polish

PATCH In review Ard Biesheuvel 2 participants 5 messages Sep 16, 2020 – Sep 25, 2020

Subsystems (2)

Arm (32-bit) Crypto API

lore

Share & tools

lore

Ard Biesheuvel

author

2020-09-16 17:46 UTC #0 lore raw

cover crypto: arm/aes-neonbs - some polish

Some polish for the ARM bit-sliced NEON implementation. No functional or performance changes anticipated.

Ard Biesheuvel (3):
crypto: arm/aes-neonbs - avoid hacks to prevent Thumb2 mode switches
crypto: arm/aes-neonbs - avoid loading reorder argument on encryption
crypto: arm/aes-neonbs - use typed init/exit routines for XTS

 arch/arm/crypto/aes-neonbs-core.S | 54 +++++++++-----------
 arch/arm/crypto/aes-neonbs-glue.c | 12 ++---
 2 files changed, 31 insertions(+), 35 deletions(-)

Signature · 1 line

2.17.1

Ard Biesheuvel

author

2020-09-16 19:05 UTC #1 lore raw

PATCH 1/3 crypto: arm/aes-neonbs - avoid hacks to prevent Thumb2 mode switches

Instead of using a homegrown macrofied version of the adr instruction that sets the Thumb bit in the output value, only to ensure that any bx instructions consuming that value will not switch out of Thumb mode when branching, use non-interworking mov (to PC) instructions, which achieve the same thing.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm/crypto/aes-neonbs-core.S | 49 +++++++++-----------
 1 file changed, 22 insertions(+), 27 deletions(-)

arch/arm/crypto/aes-neonbs-core.S +22 ~~-27~~

index cfaed4e67535..07cde1374bb0 100644
@@ -77,11 +77,6 @@
 	vldr		\out\()h, \sym + 8
 	.endm
 
-	.macro		__adr, reg, lbl
-	adr		\reg, \lbl
-THUMB(	orr		\reg, \reg, #1		)
-	.endm
-
 	.macro		in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
 	veor		\b2, \b2, \b1
 	veor		\b5, \b5, \b6

@@ -629,11 +624,11 @@ ENDPROC(aesbs_decrypt8)
 	push		{r4-r6, lr}
 	ldr		r5, [sp, #16]		// number of blocks
 
-99:	__adr		ip, 0f
+99:	adr		ip, 0f
 	and		lr, r5, #7
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #2
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q0}, [r1]!
 	vld1.8		{q1}, [r1]!

@@ -648,11 +643,11 @@ ENDPROC(aesbs_decrypt8)
 	mov		rounds, r3
 	bl		\do8
 
-	__adr		ip, 1f
+	adr		ip, 1f
 	and		lr, r5, #7
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #2
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vst1.8		{\o0}, [r0]!
 	vst1.8		{\o1}, [r0]!

@@ -689,12 +684,12 @@ ENTRY(aesbs_cbc_decrypt)
 	push		{r4-r6, lr}
 	ldm		ip, {r5-r6}		// load args 4-5
 
-99:	__adr		ip, 0f
+99:	adr		ip, 0f
 	and		lr, r5, #7
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #2
 	mov		lr, r1
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q0}, [lr]!
 	vld1.8		{q1}, [lr]!

@@ -718,11 +713,11 @@ ENTRY(aesbs_cbc_decrypt)
 	vmov		q14, q8
 	vmov		q15, q8
 
-	__adr		ip, 1f
+	adr		ip, 1f
 	and		lr, r5, #7
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #2
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q9}, [r1]!
 	vld1.8		{q10}, [r1]!

@@ -733,9 +728,9 @@ ENTRY(aesbs_cbc_decrypt)
 	vld1.8		{q15}, [r1]!
 	W(nop)
 
-1:	__adr		ip, 2f
+1:	adr		ip, 2f
 	sub		ip, ip, lr, lsl #3
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	veor		q0, q0, q8
 	vst1.8		{q0}, [r0]!

@@ -804,13 +799,13 @@ ENTRY(aesbs_ctr_encrypt)
 	vmov		q6, q0
 	vmov		q7, q0
 
-	__adr		ip, 0f
+	adr		ip, 0f
 	sub		lr, r5, #1
 	and		lr, lr, #7
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #5
 	sub		ip, ip, lr, lsl #2
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	next_ctr	q1
 	next_ctr	q2

@@ -824,13 +819,13 @@ ENTRY(aesbs_ctr_encrypt)
 	mov		rounds, r3
 	bl		aesbs_encrypt8
 
-	__adr		ip, 1f
+	adr		ip, 1f
 	and		lr, r5, #7
 	cmp		r5, #8
 	movgt		r4, #0
 	ldrle		r4, [sp, #40]		// load final in the last round
 	sub		ip, ip, lr, lsl #2
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q8}, [r1]!
 	vld1.8		{q9}, [r1]!

@@ -843,10 +838,10 @@ ENTRY(aesbs_ctr_encrypt)
 1:	bne		2f
 	vld1.8		{q15}, [r1]!
 
-2:	__adr		ip, 3f
+2:	adr		ip, 3f
 	cmp		r5, #8
 	sub		ip, ip, lr, lsl #3
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	veor		q0, q0, q8
 	vst1.8		{q0}, [r0]!

@@ -900,12 +895,12 @@ __xts_prepare8:
 	vshr.u64	d30, d31, #7
 	vmov		q12, q14
 
-	__adr		ip, 0f
+	adr		ip, 0f
 	and		r4, r6, #7
 	cmp		r6, #8
 	sub		ip, ip, r4, lsl #5
 	mov		r4, sp
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q0}, [r1]!
 	next_tweak	q12, q14, q15, q13

@@ -973,12 +968,12 @@ ENDPROC(__xts_prepare8)
 	mov		rounds, r3
 	bl		\do8
 
-	__adr		ip, 0f
+	adr		ip, 0f
 	and		lr, r6, #7
 	cmp		r6, #8
 	sub		ip, ip, lr, lsl #2
 	mov		r4, sp
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	vld1.8		{q8}, [r4, :128]!
 	vld1.8		{q9}, [r4, :128]!

@@ -989,9 +984,9 @@ ENDPROC(__xts_prepare8)
 	vld1.8		{q14}, [r4, :128]!
 	vld1.8		{q15}, [r4, :128]
 
-0:	__adr		ip, 1f
+0:	adr		ip, 1f
 	sub		ip, ip, lr, lsl #3
-	bxlt		ip			// computed goto if blocks < 8
+	movlt		pc, ip			// computed goto if blocks < 8
 
 	veor		\o0, \o0, q8
 	vst1.8		{\o0}, [r0]!

Signature · 1 line

2.17.1

Ard Biesheuvel

author

2020-09-16 19:12 UTC #2 lore raw

PATCH 2/3 crypto: arm/aes-neonbs - avoid loading reorder argument on encryption

In reply to Ard Biesheuvel

Reordering the tweak is never necessary for encryption, so avoid the argument load on the encryption path.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm/crypto/aes-neonbs-core.S | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

arch/arm/crypto/aes-neonbs-core.S +3 -2

index 07cde1374bb0..7d0cc7f226a5 100644
@@ -956,8 +956,7 @@ ENDPROC(__xts_prepare8)
 	push		{r4-r8, lr}
 	mov		r5, sp			// preserve sp
 	ldrd		r6, r7, [sp, #24]	// get blocks and iv args
-	ldr		r8, [sp, #32]		// reorder final tweak?
-	rsb		r8, r8, #1
+	rsb		r8, ip, #1
 	sub		ip, sp, #128		// make room for 8x tweak
 	bic		ip, ip, #0xf		// align sp to 16 bytes
 	mov		sp, ip

@@ -1013,9 +1012,11 @@ ENDPROC(__xts_prepare8)
 	.endm
 
 ENTRY(aesbs_xts_encrypt)
+	mov		ip, #0			// never reorder final tweak
 	__xts_crypt	aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
 ENDPROC(aesbs_xts_encrypt)
 
 ENTRY(aesbs_xts_decrypt)
+	ldr		ip, [sp, #8]		// reorder final tweak?
 	__xts_crypt	aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
 ENDPROC(aesbs_xts_decrypt)

Signature · 1 line

2.17.1

Ard Biesheuvel

author

2020-09-16 16:51 UTC #3 lore raw

PATCH 3/3 crypto: arm/aes-neonbs - use typed init/exit routines for XTS

In reply to Ard Biesheuvel

Use the typed skcipher init/exit routines instead of the generic cra_init/_exit routines when instantiating/releasing the XTS skciphers.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm/crypto/aes-neonbs-glue.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

arch/arm/crypto/aes-neonbs-glue.c +6 -6

index e1603ec7e815..bda8bf17631e 100644
@@ -314,9 +314,9 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 	return aesbs_setkey(tfm, in_key, key_len);
 }
 
-static int xts_init(struct crypto_tfm *tfm)
+static int xts_init(struct crypto_skcipher *tfm)
 {
-	struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 
 	ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0);
 	if (IS_ERR(ctx->cts_tfm))

@@ -329,9 +329,9 @@ static int xts_init(struct crypto_tfm *tfm)
 	return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
 }
 
-static void xts_exit(struct crypto_tfm *tfm)
+static void xts_exit(struct crypto_skcipher *tfm)
 {
-	struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 
 	crypto_free_cipher(ctx->tweak_tfm);
 	crypto_free_cipher(ctx->cts_tfm);

@@ -493,8 +493,6 @@ static struct skcipher_alg aes_algs[] = { {
 	.base.cra_ctxsize	= sizeof(struct aesbs_xts_ctx),
 	.base.cra_module	= THIS_MODULE,
 	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
-	.base.cra_init		= xts_init,
-	.base.cra_exit		= xts_exit,
 
 	.min_keysize		= 2 * AES_MIN_KEY_SIZE,
 	.max_keysize		= 2 * AES_MAX_KEY_SIZE,

@@ -503,6 +501,8 @@ static struct skcipher_alg aes_algs[] = { {
 	.setkey			= aesbs_xts_setkey,
 	.encrypt		= xts_encrypt,
 	.decrypt		= xts_decrypt,
+	.init			= xts_init,
+	.exit			= xts_exit,
 } };
 
 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];

Signature · 1 line

2.17.1

Herbert Xu

maintainer

2020-09-25 08:15 UTC #4 lore raw

In reply to Ard Biesheuvel

Quoted 11 lines · Ard Biesheuvel

Some polish for the ARM bit-sliced NEON implementation. No functional or performance changes anticipated.
Ard Biesheuvel (3):
crypto: arm/aes-neonbs - avoid hacks to prevent Thumb2 mode switches
crypto: arm/aes-neonbs - avoid loading reorder argument on encryption
crypto: arm/aes-neonbs - use typed init/exit routines for XTS
 arch/arm/crypto/aes-neonbs-core.S | 54 +++++++++-----------
 arch/arm/crypto/aes-neonbs-glue.c | 12 ++---
 2 files changed, 31 insertions(+), 35 deletions(-)

All applied. Thanks.

Signature · 3 lines

Email: Herbert Xu <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt