arm64: pull in <asm/simd.h> from asm-generic
Subsystems (2)
PATCH RESEND 10/15 arm64: pull in <asm/simd.h> from asm-generic
--- arch/arm64/include/asm/Kbuild | 1 + 1 file changed, 1 insertion(+)
arch/arm64/include/asm/Kbuild +1 -0
@@ -40,6 +40,7 @@ generic-y += segment.h generic-y += sembuf.h generic-y += serial.h generic-y += shmbuf.h +generic-y += simd.h generic-y += sizes.h generic-y += socket.h generic-y += sockios.h
Signature · 1 line
1.8.3.2
PATCH RESEND 11/15 arm64/crypto: AES-ECB/CBC/CTR/XTS using ARMv8 NEON and Crypto Extensions
This adds ARMv8 implementations of AES in ECB, CBC, CTR and XTS modes, both for ARMv8 with Crypto Extensions and for plain ARMv8 NEON.
The Crypto Extensions version can only run on ARMv8 implementations that have support for these optional extensions.
The plain NEON version is a table based yet time invariant implementation. All S-box substitutions are performed in parallel, leveraging the wide range of ARMv8's tbl/tbx instructions, and the huge NEON register file, which can comfortably hold the entire S-box and still have room to spare for doing the actual computations.
The key expansion routines were borrowed from aes_generic.
--- arch/arm64/crypto/Kconfig | 14 ++ arch/arm64/crypto/Makefile | 14 ++ arch/arm64/crypto/aes-ce.S | 147 +++++++++++ arch/arm64/crypto/aes-glue.c | 446 ++++++++++++++++++++++++++++++++++ arch/arm64/crypto/aes-modes.S | 548 ++++++++++++++++++++++++++++++++++++++++++ arch/arm64/crypto/aes-neon.S | 382 +++++++++++++++++++++++++++++ 6 files changed, 1551 insertions(+) create mode 100644 arch/arm64/crypto/aes-ce.S create mode 100644 arch/arm64/crypto/aes-glue.c create mode 100644 arch/arm64/crypto/aes-modes.S create mode 100644 arch/arm64/crypto/aes-neon.S
arch/arm64/crypto/Kconfig +14 -0
@@ -36,4 +36,18 @@ config CRYPTO_AES_ARM64_CE_CCM select CRYPTO_AES select CRYPTO_AEAD +config CRYPTO_AES_ARM64_CE_BLK + tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_ABLK_HELPER + +config CRYPTO_AES_ARM64_NEON_BLK + tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_ABLK_HELPER + endif
arch/arm64/crypto/Makefile +14 -0
@@ -22,3 +22,17 @@ CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o + +obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o +aes-ce-blk-y := aes-glue-ce.o aes-ce.o + +obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o +aes-neon-blk-y := aes-glue-neon.o aes-neon.o + +AFLAGS_aes-ce.o := -DINTERLEAVE=8 # -DINTERLEAVE_INLINE +AFLAGS_aes-neon.o := -DINTERLEAVE=4 + +CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS + +$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE + $(call if_changed_dep,cc_o_c)
arch/arm64/crypto/aes-ce.S new file +147 -0
147 lines not on this page · raw message
arch/arm64/crypto/aes-glue.c new file +446 -0
446 lines not on this page · raw message
arch/arm64/crypto/aes-modes.S new file +548 -0
548 lines not on this page · raw message
arch/arm64/crypto/aes-neon.S new file +382 -0
382 lines not on this page · raw message
Signature · 1 line
1.8.3.2
PATCH RESEND 12/15 arm64/crypto: add shared macro to test for NEED_RESCHED
In reply to Ard BiesheuvelThis adds the asm macro definition 'b_if_no_resched' that performs a conditional branch depending on the preempt need_resched state.
--- arch/arm64/include/asm/assembler.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)
arch/arm64/include/asm/assembler.h +21 -0
@@ -22,6 +22,11 @@#include <asm/ptrace.h> +#if defined(CONFIG_PREEMPT ) || defined(CONFIG_PREEMPT_VOLUNTARY ) +#include <asm/asm-offsets.h> +#include <asm/thread_info.h> +#endif +/* * Stack pushing/popping (register pairs only). Equivalent to store decrement * before, load increment after.
@@ -146,3 +151,19 @@ lr .req x30 // link register#endif orr \rd, \lbits, \hbits, lsl #32 .endm + +/* + * Branch to 'lb' but only if we have not been tagged for preemption. + * + * Expects current->thread_info in ti, or NULL if running in interrupt + * context. reg is a scratch x register. + */ + .macro b_if_no_resched, ti, reg, lb +#if defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY) + cbz \ti, \lb /* have thread_info? */ + ldr \reg, [\ti, #TI_FLAGS] /* get flags */ + tbz \reg, #TIF_NEED_RESCHED, \lb /* need rescheduling? */ +#else + b \lb +#endif + .endm
Signature · 1 line
1.8.3.2
PATCH RESEND 13/15 arm64/crypto: add voluntary preemption to Crypto Extensions SHA1
In reply to Ard BiesheuvelThe Crypto Extensions based SHA1 implementation uses the NEON register file, and hence runs with preemption disabled. This patch adds a TIF_NEED_RESCHED check to its inner loop so we at least give up the CPU voluntarily when we are running in process context and have been tagged for preemption by the scheduler.
--- arch/arm64/crypto/sha1-ce-core.S | 19 ++++++++-------- arch/arm64/crypto/sha1-ce-glue.c | 49 +++++++++++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 20 deletions(-)
arch/arm64/crypto/sha1-ce-core.S +10 -9
@@ -66,8 +66,8 @@ .word 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 /* - * void sha1_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * int sha1_ce_transform(int blocks, u8 const *src, u32 *state, + * u8 *head, long bytes, struct thread_info *ti) */ ENTRY(sha1_ce_transform) /* load round constants */
@@ -127,7 +127,13 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbnz w0, 0b + cbz w0, 4f + b_if_no_resched x5, x8, 0b + + /* store new state */ +3: str dga, [x2] + str dgb, [x2, #16] + ret /* * Final block: add padding and total bit count.
@@ -135,7 +141,7 @@ CPU_LE( rev32 v11.16b, v11.16b ) * size was not a round multiple of the block size, and the padding is * handled by the C code. */ - cbz x4, 3f +4: cbz x4, 3b movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0
@@ -145,9 +151,4 @@ CPU_LE( rev32 v11.16b, v11.16b ) mov v11.d[0], xzr mov v11.d[1], x7 b 2b - - /* store new state */ -3: str dga, [x2] - str dgb, [x2, #16] - ret ENDPROC(sha1_ce_transform)
arch/arm64/crypto/sha1-ce-glue.c +38 -11
83 lines not on this page · raw message
Signature · 1 line
1.8.3.2
PATCH RESEND 13/15 arm64/crypto: add voluntary preemption to Crypto Extensions SHA1
Quoted 8 lines · Ard Biesheuvel
The Crypto Extensions based SHA1 implementation uses the NEON register file, and hence runs with preemption disabled. This patch adds a
TIF_NEED_RESCHEDcheck to its inner loop so we at least give up the CPU voluntarily when we are running in process context and have been tagged for preemption by the scheduler.Signed-off-by Ard Biesheuvel <[email protected]>---
<snip>
Quoted 65 lines
arch/arm64/crypto/sha1-ce-glue.c +27
-9@@ -42,6 +42,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, sctx->count += len;if ((partial + len) >=SHA1_BLOCK_SIZE ) { +struct thread_info *ti =NULL ;int blocks;if (partial) {@@ -52,16 +53,30 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, len -= p; } +/* +* Pass current's thread info pointer to sha1_ce_transform() +* below if we want it to play nice under preemption. +*/ +if ((IS_ENABLED (CONFIG_PREEMPT_VOLUNTARY ) || +IS_ENABLED (CONFIG_PREEMPT )) && !in_interrupt ()) + ti =current_thread_info (); + blocks = len /SHA1_BLOCK_SIZE ; len %=SHA1_BLOCK_SIZE ; -kernel_neon_begin_partial (16 ); -sha1_ce_transform (blocks, data, sctx->state, - partial ? sctx->buffer :NULL ,0 ); -kernel_neon_end (); +do { +int rem; + +kernel_neon_begin_partial (16 ); + rem =sha1_ce_transform (blocks, data, sctx->state, + partial ? sctx->buffer :NULL , +0 , ti); +kernel_neon_end (); - data += blocks *SHA1_BLOCK_SIZE ; - partial =0 ; + data += (blocks - rem) *SHA1_BLOCK_SIZE ; + blocks = rem; + partial =0 ; + }while (unlikely (ti && blocks >0 )); }if (len)memcpy (sctx->buffer + partial, data, len);@@ -94,6 +109,7 @@ static int sha1_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha1_state *sctx = shash_desc_ctx(desc); +struct thread_info *ti =NULL ;__be32 *dst = (__be32 *)out;int blocks;int i;@@ -111,9 +127,20 @@ static int sha1_finup(struct shash_desc *desc, const u8 *data,*/ blocks = len /SHA1_BLOCK_SIZE ; -kernel_neon_begin_partial (16 ); -sha1_ce_transform (blocks, data, sctx->state,NULL , len); -kernel_neon_end (); +if ((IS_ENABLED (CONFIG_PREEMPT_VOLUNTARY ) || +IS_ENABLED (CONFIG_PREEMPT )) && !in_interrupt ()) + ti =current_thread_info (); + + do {
arch/arm64/crypto/sha1-ce-glue.c static int sha1_finup(struct shash_desc *desc, const u8 *data, · line 144
Quoted 10 lines
arch/arm64/crypto/sha1-ce-glue.c +9
-0+int rem; + +kernel_neon_begin_partial (16 ); + rem =sha1_ce_transform (blocks, data, sctx->state, +NULL , len, ti); +kernel_neon_end (); + data += (blocks - rem) *SHA1_BLOCK_SIZE ; + blocks = rem; + }while (unlikely (ti && blocks >0 ));
These seem to be similar, how about renaming assembly function to __sha1_ce_transform and moving this loop to new sha1_ce_transform.
Otherwise, patches looks good.
-Jussi
Herbert Xu
maintainerPATCH RESEND 13/15 arm64/crypto: add voluntary preemption to Crypto Extensions SHA1
In reply to Ard Biesheuvelarch/arm64/crypto/sha1-ce-glue.c static int sha1_update(struct shash_desc *desc, const u8 *data, · line 62
Quoted 8 lines · Ard Biesheuvel
arch/arm64/crypto/sha1-ce-glue.c +7
-0+/* +* Pass current's thread info pointer to sha1_ce_transform() +* below if we want it to play nice under preemption. +*/ +if ((IS_ENABLED (CONFIG_PREEMPT_VOLUNTARY ) || +IS_ENABLED (CONFIG_PREEMPT )) && !in_interrupt ()) + ti =current_thread_info ();
Instead of in_interrupt you should test CRYPTO_TFM_REQ_MAY_SLEEP. It's not a matter of correctness but it may avoid unnecessary breaks in the loop.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
PATCH RESEND 14/15 arm64/crypto: add voluntary preemption to Crypto Extensions SHA2
In reply to Ard BiesheuvelThe Crypto Extensions based SHA2 implementation uses the NEON register file, and hence runs with preemption disabled. This patch adds a TIF_NEED_RESCHED check to its inner loop so we at least give up the CPU voluntarily when we are running in process context and have been tagged for preemption by the scheduler.
--- arch/arm64/crypto/sha2-ce-core.S | 19 ++++++++------- arch/arm64/crypto/sha2-ce-glue.c | 51 ++++++++++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 20 deletions(-)
arch/arm64/crypto/sha2-ce-core.S +11 -8
@@ -73,8 +73,8 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * void sha2_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * int sha2_ce_transform(int blocks, u8 const *src, u32 *state, + * u8 *head, long bytes, struct thread_info *ti) */ ENTRY(sha2_ce_transform) /* load round constants */
@@ -131,7 +131,14 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbnz w0, 0b + cbz w0, 4f + + /* should we exit early? */ + b_if_no_resched x5, x8, 0b + + /* store new state */ +3: stp dga, dgb, [x2] + ret /* * Final block: add padding and total bit count.
@@ -139,7 +146,7 @@ CPU_LE( rev32 v19.16b, v19.16b ) * size was not a round multiple of the block size, and the padding is * handled by the C code. */ - cbz x4, 3f +4: cbz x4, 3b movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0
@@ -149,8 +156,4 @@ CPU_LE( rev32 v19.16b, v19.16b ) mov v19.d[0], xzr mov v19.d[1], x7 b 2b - - /* store new state */ -3: stp dga, dgb, [x2] - ret ENDPROC(sha2_ce_transform)
arch/arm64/crypto/sha2-ce-glue.c +39 -12
88 lines not on this page · raw message
Signature · 1 line
1.8.3.2
PATCH RESEND 15/15 arm64/crypto: add voluntary preemption to Crypto Extensions GHASH
In reply to Ard BiesheuvelThe Crypto Extensions based GHASH implementation uses the NEON register file, and hence runs with preemption disabled. This patch adds a TIF_NEED_RESCHED check to its inner loop so we at least give up the CPU voluntarily when we are running in process context and have been tagged for preemption by the scheduler.
--- arch/arm64/crypto/ghash-ce-core.S | 10 ++++++---- arch/arm64/crypto/ghash-ce-glue.c | 33 +++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 12 deletions(-)
arch/arm64/crypto/ghash-ce-core.S +6 -4
@@ -31,8 +31,9 @@ .arch armv8-a+crypto /* - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, - * struct ghash_key const *k, const char *head) + * int pmull_ghash_update(int blocks, u64 dg[], const char *src, + * struct ghash_key const *k, const char *head, + * struct thread_info *ti) */ ENTRY(pmull_ghash_update) ld1 {DATA.16b}, [x1]
@@ -88,8 +89,9 @@ CPU_LE( rev64 IN1.16b, IN1.16b ) eor T1.16b, T1.16b, T2.16b eor DATA.16b, DATA.16b, T1.16b - cbnz w0, 0b + cbz w0, 2f + b_if_no_resched x5, x7, 0b - st1 {DATA.16b}, [x1] +2: st1 {DATA.16b}, [x1] ret ENDPROC(pmull_ghash_update)
arch/arm64/crypto/ghash-ce-glue.c +25 -8
@@ -33,8 +33,9 @@ struct ghash_desc_ctx {u32 count; }; -asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src, -struct ghash_key const *k,const char *head); +asmlinkage int pmull_ghash_update(int blocks, u64 dg[], const char *src, +struct ghash_key const *k,const char *head, +struct thread_info *ti);static int ghash_init (struct shash_desc *desc) {
@@ -54,6 +55,7 @@ static int ghash_update(struct shash_desc *desc, const u8 *src,if ((partial + len) >=GHASH_BLOCK_SIZE ) {struct ghash_key *key =crypto_shash_ctx (desc->tfm); +struct thread_info *ti =NULL ;int blocks;if (partial) {
@@ -64,14 +66,29 @@ static int ghash_update(struct shash_desc *desc, const u8 *src, len -= p; } +/* +* Pass current's thread info pointer to pmull_ghash_update() +* below if we want it to play nice under preemption. +*/ +if ((IS_ENABLED (CONFIG_PREEMPT_VOLUNTARY ) || +IS_ENABLED (CONFIG_PREEMPT )) && !in_interrupt ()) + ti =current_thread_info (); + blocks = len /GHASH_BLOCK_SIZE ; len %=GHASH_BLOCK_SIZE ; -kernel_neon_begin_partial (6 ); -pmull_ghash_update (blocks, ctx->digest, src, key, - partial ? ctx->buf :NULL ); -kernel_neon_end (); - src += blocks *GHASH_BLOCK_SIZE ; +do { +int rem; + +kernel_neon_begin_partial (6 ); + rem =pmull_ghash_update (blocks, ctx->digest, src, key, + partial ? ctx->buf :NULL , ti); +kernel_neon_end (); + + src += (blocks - rem) *GHASH_BLOCK_SIZE ; + blocks = rem; + partial =0 ; + }while (unlikely (ti && blocks >0 )); }if (len)memcpy (ctx->buf + partial, src, len);
@@ -89,7 +106,7 @@ static int ghash_final(struct shash_desc *desc, u8 *dst)memset (ctx->buf + partial,0 ,GHASH_BLOCK_SIZE - partial);kernel_neon_begin_partial (6 ); -pmull_ghash_update (1 , ctx->digest, ctx->buf, key,NULL ); +pmull_ghash_update (1 , ctx->digest, ctx->buf, key,NULL ,NULL );kernel_neon_end (); }put_unaligned_be64 (ctx->digest[1 ], dst);
Signature · 1 line
1.8.3.2