kernel mode NEON optimizations
Subsystems (3)
v2 cover kernel mode NEON optimizations
The main purpose of this series is to allow Crypto Extensions instructions to be used in interrupt context. For instance, the WPA2 CCMP handling (i.e., AES-128 in CCM mode) in the mac80211 layer runs entirely in softirq context.
Patch #1 does some preparatory work so all code external to kernel/fpsimd.c uses accessor functions to manipulate both the on-CPU and preserved FPSIMD states rather than poking them directly.
Patch #2 implements an optimization that reduces the number of times the userland FPSIMD state is needlessly preserved and restored.
Patch #3 modifies kernel_neon_begin() and kernel_neon_end() so they may be called from interrupt context.
Patch #4 contains the actual AES implementation, updated to reflect some recent feedback I received when I posted it separately from this series.
Changes since previous version:
- added preparatory patch #1
- based on feedback from Will Deacon, make sure ptrace() and return from signal handler cases are covered.
Mildly but not yet thoroughly tested.
All feedback highly appreciated.
Ard Biesheuvel (4):
arm64: add abstractions for FPSIMD state manipulation
arm64: defer reloading a task's FPSIMD state to userland resume
arm64: add support for kernel mode NEON in interrupt context
arm64: add Crypto Extensions based synchronous core AES cipher
arch/arm64/Makefile | 1 + arch/arm64/crypto/Makefile | 13 +++ arch/arm64/crypto/aes-ce-cipher.c | 134 +++++++++++++++++++++++++++++ arch/arm64/include/asm/fpsimd.h | 30 ++++++- arch/arm64/include/asm/fpsimdmacros.h | 37 ++++++++ arch/arm64/include/asm/neon.h | 6 +- arch/arm64/include/asm/thread_info.h | 4 +- arch/arm64/kernel/entry-fpsimd.S | 24 ++++++ arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/fpsimd.c | 156 +++++++++++++++++++++++++++++----- arch/arm64/kernel/process.c | 2 +- arch/arm64/kernel/ptrace.c | 22 +++-- arch/arm64/kernel/signal.c | 10 ++- arch/arm64/kernel/signal32.c | 6 +- crypto/Kconfig | 6 ++ 15 files changed, 411 insertions(+), 42 deletions(-) create mode 100644 arch/arm64/crypto/Makefile create mode 100644 arch/arm64/crypto/aes-ce-cipher.c
Signature · 1 line
1.8.3.2
PATCH v2 1/4 arm64: add abstractions for FPSIMD state manipulation
This is a preparatory patch that replaces code that saves or restores the on-CPU or preserved FPSIMD state directly with wrapper functions, resulting in all direct manipulation of the FPSIMD state to be concentrated in fpsimd.c
--- arch/arm64/include/asm/fpsimd.h | 9 ++++++--- arch/arm64/kernel/fpsimd.c | 31 +++++++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 2 +- arch/arm64/kernel/ptrace.c | 22 +++++++++++++--------- arch/arm64/kernel/signal.c | 6 +++--- arch/arm64/kernel/signal32.c | 6 +++--- 6 files changed, 57 insertions(+), 19 deletions(-)
arch/arm64/include/asm/fpsimd.h +6 -3
@@ -52,12 +52,15 @@ struct fpsimd_state {struct task_struct ; -extern void fpsimd_save_state (struct fpsimd_state *state); -extern void fpsimd_load_state (struct fpsimd_state *state); -extern void fpsimd_thread_switch (struct task_struct *next);extern void fpsimd_flush_thread (void ); +struct fpsimd_state *fpsimd_get_task_state (void ); +void fpsimd_set_task_state (struct fpsimd_state *state); + +struct user_fpsimd_state *fpsimd_get_user_state (struct task_struct *t); +void fpsimd_set_user_state (struct task_struct *t,struct user_fpsimd_state *st); +#endif #endif
arch/arm64/kernel/fpsimd.c +31 -0
@@ -34,6 +34,10 @@#define FPEXC_IXF (1 << 4)#define FPEXC_IDF (1 << 7) +/* defined in entry-fpsimd.S but only used in this unit */ +void fpsimd_save_state (struct fpsimd_state *state); +void fpsimd_load_state (struct fpsimd_state *state); +/* * Trapped FP/ASIMD access. */
@@ -87,6 +91,33 @@ void fpsimd_flush_thread(void)preempt_enable (); } +/* +* Sync the saved FPSIMD context with the FPSIMD register file +*/ +struct fpsimd_state *fpsimd_get_task_state (void ) +{ +fpsimd_save_state (¤t->thread.fpsimd_state); +return ¤t->thread.fpsimd_state; +} + +/* +* Load a new FPSIMD state into the FPSIMD register file. +*/ +void fpsimd_set_task_state (struct fpsimd_state *state) +{ +fpsimd_load_state (state); +} + +struct user_fpsimd_state *fpsimd_get_user_state (struct task_struct *t) +{ +return &t->thread.fpsimd_state.user_fpsimd; +} + +void fpsimd_set_user_state (struct task_struct *t,struct user_fpsimd_state *st) +{ + t->thread.fpsimd_state.user_fpsimd = *st; +} +#ifdef CONFIG_KERNEL_MODE_NEON /*
arch/arm64/kernel/process.c +1 -1
@@ -199,7 +199,7 @@ void release_thread(struct task_struct *dead_task)int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src) { -fpsimd_save_state (¤t->thread.fpsimd_state); +fpsimd_get_task_state (); *dst = *src;return 0 ; }
arch/arm64/kernel/ptrace.c +13 -9
@@ -500,8 +500,7 @@ static int fpr_get(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - struct user_fpsimd_state *uregs; - uregs = &target->thread.fpsimd_state.user_fpsimd; + struct user_fpsimd_state *uregs = fpsimd_get_user_state(target);return user_regset_copyout (&pos, &count, &kbuf, &ubuf, uregs,0 ,-1 ); }
@@ -516,7 +515,7 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset,if (ret)return ret; - target->thread.fpsimd_state.user_fpsimd = newstate; +fpsimd_set_user_state (target, &newstate);return ret; }
@@ -723,7 +722,7 @@ static int compat_vfp_get(struct task_struct *target,compat_ulong_t fpscr;int ret; - uregs = &target->thread.fpsimd_state.user_fpsimd; + uregs =fpsimd_get_user_state (target);/* * The VFP registers are packed into the fpsimd_state, so they all sit
@@ -746,24 +745,29 @@ static int compat_vfp_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct user_fpsimd_state *uregs; + struct user_fpsimd_state newstate;compat_ulong_t fpscr;int ret;if (pos + count >VFP_STATE_SIZE )return -EIO ; - uregs = &target->thread.fpsimd_state.user_fpsimd; +/* +* We will not overwrite the entire FPSIMD state, so we need to +* initialize 'newstate' with sane values. +*/ + newstate = *fpsimd_get_user_state (target); - ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, uregs,0 , + ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, &newstate,0 ,VFP_STATE_SIZE -sizeof (compat_ulong_t));if (count && !ret) { ret =get_user (fpscr, (compat_ulong_t *)ubuf); - uregs->fpsr = fpscr &VFP_FPSCR_STAT_MASK ; - uregs->fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; + newstate.fpsr = fpscr &VFP_FPSCR_STAT_MASK ; + newstate.fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; } +fpsimd_set_user_state (target, &newstate);return ret; }
arch/arm64/kernel/signal.c +3 -3
@@ -47,11 +47,11 @@ struct rt_sigframe {static int preserve_fpsimd_context (struct fpsimd_context __user *ctx) { -struct fpsimd_state *fpsimd = ¤t->thread.fpsimd_state; +struct fpsimd_state *fpsimd;int err;/* dump the hardware registers to the fpsimd_state structure */ -fpsimd_save_state (fpsimd); + fpsimd =fpsimd_get_task_state ();/* copy the FP and status/control registers */ err =__copy_to_user (ctx->vregs, fpsimd->vregs,sizeof (fpsimd->vregs));
@@ -88,7 +88,7 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)/* load the hardware registers from the fpsimd_state structure */ if (!err) {preempt_disable (); -fpsimd_load_state (&fpsimd); +fpsimd_set_task_state (&fpsimd);preempt_enable (); }
arch/arm64/kernel/signal32.c +3 -3
@@ -208,7 +208,7 @@ int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)*/ static int compat_preserve_vfp_context (struct compat_vfp_sigframe __user *frame) { -struct fpsimd_state *fpsimd = ¤t->thread.fpsimd_state; +struct fpsimd_state *fpsimd;compat_ulong_t magic =VFP_MAGIC ;compat_ulong_t size =VFP_STORAGE_SIZE ;compat_ulong_t fpscr, fpexc;
@@ -219,7 +219,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)* Note that this also saves V16-31, which aren't visible * in AArch32. */ -fpsimd_save_state (fpsimd); + fpsimd =fpsimd_get_task_state ();/* Place structure header on the stack */ __put_user_error (magic, &frame->magic, err);
@@ -284,7 +284,7 @@ static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame)*/ if (!err) {preempt_disable (); -fpsimd_load_state (&fpsimd); +fpsimd_set_task_state (&fpsimd);preempt_enable (); }
Signature · 1 line
1.8.3.2
Catalin Marinas
maintainerPATCH v2 1/4 arm64: add abstractions for FPSIMD state manipulation
arch/arm64/kernel/fpsimd.c line 39
Quoted 11 lines · Ard Biesheuvel
arch/arm64/kernel/fpsimd.c +3
-0@@ -34,6 +34,10 @@#define FPEXC_IXF (1 << 4)#define FPEXC_IDF (1 << 7) +/* defined in entry-fpsimd.S but only used in this unit */ +void fpsimd_save_state (struct fpsimd_state *state); +void fpsimd_load_state (struct fpsimd_state *state);
This functions still make sense for some situations where fpsimd_get_task_state() is confusing.
Quoted 14 lines
arch/arm64/kernel/fpsimd.c +7
-0+/* * Trapped FP/ASIMD access. */ @@ -87,6 +91,33 @@ void fpsimd_flush_thread(void)preempt_enable (); } +/* +* Sync the saved FPSIMD context with the FPSIMD register file +*/ +struct fpsimd_state *fpsimd_get_task_state (void ) +{ +fpsimd_save_state (¤t->thread.fpsimd_state);
arch/arm64/kernel/fpsimd.c void fpsimd_flush_thread(void) · line 109
Quoted 10 lines
arch/arm64/kernel/fpsimd.c +10
-0+return ¤t->thread.fpsimd_state; +} + +/* +* Load a new FPSIMD state into the FPSIMD register file. +*/ +void fpsimd_set_task_state (struct fpsimd_state *state) +{ +fpsimd_load_state (state); +}
Maybe s/task/current/. But in general I see a "get" function as not having side-effects while here it populates the fpsimd_state. I don't think the code gets clearer.
arch/arm64/kernel/process.c void release_thread(struct task_struct *dead_task) · line 202
Quoted 10 lines
arch/arm64/kernel/process.c +1
-1@@ -199,7 +199,7 @@ void release_thread(struct task_struct *dead_task)int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src) { -fpsimd_save_state (¤t->thread.fpsimd_state); +fpsimd_get_task_state ();
That's where get vs save looks strange to me.
Quoted 24 lines
arch/arm64/kernel/process.c +7
-3@@ -746,24 +745,29 @@ static int compat_vfp_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct user_fpsimd_state *uregs; + struct user_fpsimd_state newstate;compat_ulong_t fpscr;int ret;if (pos + count >VFP_STATE_SIZE )return -EIO ; - uregs = &target->thread.fpsimd_state.user_fpsimd; +/* +* We will not overwrite the entire FPSIMD state, so we need to +* initialize 'newstate' with sane values. +*/ + newstate = *fpsimd_get_user_state (target); - ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, uregs,0 , + ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, &newstate,0 ,VFP_STATE_SIZE -sizeof (compat_ulong_t));if (count && !ret) {
arch/arm64/kernel/process.c static int compat_vfp_set(struct task_struct *target, · line 772
Quoted 10 lines
arch/arm64/kernel/process.c +3
-2ret =get_user (fpscr, (compat_ulong_t *)ubuf); - uregs->fpsr = fpscr &VFP_FPSCR_STAT_MASK ; - uregs->fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; + newstate.fpsr = fpscr &VFP_FPSCR_STAT_MASK ; + newstate.fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; } +fpsimd_set_user_state (target, &newstate);return ret; }
What's the reason for this change? The patch is aimed at adding abstractions but it does some more. Now you save half KB on the stack as user_fpsimd_state. What if at some point we grow this structure?
Signature · 1 line
Catalin
PATCH v2 1/4 arm64: add abstractions for FPSIMD state manipulation
Quoted 9 lines · Catalin Marinas
Quoted 8 lines · Ard Biesheuvel
arch/arm64/kernel/fpsimd.c +0
-0@@ -34,6 +34,10 @@#define FPEXC_IXF (1 << 4)#define FPEXC_IDF (1 << 7)
Quoted 7 lines
arch/arm64/kernel/fpsimd.c line 39
Quoted 3 lines
arch/arm64/kernel/fpsimd.c +3
-0+/* defined in entry-fpsimd.S but only used in this unit */ +void fpsimd_save_state (struct fpsimd_state *state); +void fpsimd_load_state (struct fpsimd_state *state);This functions still make sense for some situations where
fpsimd_get_task_state()is confusing.
This is an attempt to distinguish between, on the one hand, functions that operate on the contents of the register, regardless of who owns the context they belong to, and on the other hand, having a level of abstraction where 'store my task's FPSIMD context to memory' does not need to handle the situation where 'my task's FPSIMD state' is somewhere else than in the registers.
So after the next patch, it is up to the caller of fpsimd_{load|save}_state to understand that what he loads/saves depends on TIF_FOREIGN_FPSTATE etc.
Quoted 20 lines
Quoted 20 lines
arch/arm64/kernel/fpsimd.c +13
-0+/* * Trapped FP/ASIMD access. */ @@ -87,6 +91,33 @@ void fpsimd_flush_thread(void)preempt_enable (); } +/* +* Sync the saved FPSIMD context with the FPSIMD register file +*/ +struct fpsimd_state *fpsimd_get_task_state (void ) +{ +fpsimd_save_state (¤t->thread.fpsimd_state); +return ¤t->thread.fpsimd_state; +} + +/* +* Load a new FPSIMD state into the FPSIMD register file. +*/
Quoted 9 lines
arch/arm64/kernel/fpsimd.c void fpsimd_flush_thread(void) · line 109
Quoted 4 lines
arch/arm64/kernel/fpsimd.c +4
-0+void fpsimd_set_task_state (struct fpsimd_state *state) +{ +fpsimd_load_state (state); +}Maybe s/task/current/. But in general I see a "get" function as not having side-effects while here it populates the
fpsimd_state. I don't think the code gets clearer.
I am happy to improve the names, but I think this is a meaningful abstraction to have, i.e., get a pointer to the task's saved FPSIMD state and make sure the saved state is up to date.
In the next patch, the call to fpsimd_save_state() becomes conditional, because the registers may contain something unrelated.
Quoted 6 lines
Quoted 6 lines
arch/arm64/kernel/process.c +0
-0@@ -199,7 +199,7 @@ void release_thread(struct task_struct *dead_task)
Quoted 7 lines
arch/arm64/kernel/process.c void release_thread(struct task_struct *dead_task) · line 202
Quoted 4 lines
arch/arm64/kernel/process.c +1
-1int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src) { -fpsimd_save_state (¤t->thread.fpsimd_state); +fpsimd_get_task_state ();That's where get vs save looks strange to me.
The existing function already relies on the assumption that src == current. Perhaps it would be better to explicitly copy the fpsimd_state of src in dst instead, but it would result in the FPSIMD state being copied twice.
But the bottom line is that there needs to be some abstraction to distinguish 'the FPSIMD state of current' from 'the FPSIMD in this CPU's registers', as they will not be the same after the next patch.
Quoted 31 lines
Quoted 31 lines
arch/arm64/kernel/process.c +9
-5@@ -746,24 +745,29 @@ static int compat_vfp_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct user_fpsimd_state *uregs; + struct user_fpsimd_state newstate;compat_ulong_t fpscr;int ret;if (pos + count >VFP_STATE_SIZE )return -EIO ; - uregs = &target->thread.fpsimd_state.user_fpsimd; +/* +* We will not overwrite the entire FPSIMD state, so we need to +* initialize 'newstate' with sane values. +*/ + newstate = *fpsimd_get_user_state (target); - ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, uregs,0 , + ret =user_regset_copyin (&pos, &count, &kbuf, &ubuf, &newstate,0 ,VFP_STATE_SIZE -sizeof (compat_ulong_t));if (count && !ret) { ret =get_user (fpscr, (compat_ulong_t *)ubuf); - uregs->fpsr = fpscr &VFP_FPSCR_STAT_MASK ; - uregs->fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; + newstate.fpsr = fpscr &VFP_FPSCR_STAT_MASK ; + newstate.fpcr = fpscr &VFP_FPSCR_CTRL_MASK ; }
Quoted 8 lines
arch/arm64/kernel/process.c static int compat_vfp_set(struct task_struct *target, · line 772
Quoted 3 lines
arch/arm64/kernel/process.c +1
-0+fpsimd_set_user_state (target, &newstate);return ret; }What's the reason for this change? The patch is aimed at adding abstractions but it does some more. Now you save half KB on the stack as
user_fpsimd_state. What if at some point we grow this structure?
Perhaps it would be better to have a compat_vfp_state type of the correct size and dedicated getters/setters?
But the reason is the same: in order not to clutter this code with knowledge about the fact that current userland's FPSIMD state may not be present in the registers all the time, I introduced an abstraction. If you look at the next patch, set_user_state() needs to invalidate live copies of the FPSIMD register state that may be present elsewhere, and I think it is better to do that in only a single place, not in all code that deals with FPSIMD state but on a less detailed level.
Regards,
Ard.
PATCH v2 2/4 arm64: defer reloading a task's FPSIMD state to userland resume
In reply to Ard BiesheuvelIf a task gets scheduled out and back in again and nothing has touched its FPSIMD state in the mean time, there is really no reason to reload it from memory. Similarly, repeated calls to kernel_neon_begin() and kernel_neon_end() will preserve and restore the FPSIMD state every time.
This patch defers the FPSIMD state restore to the last possible moment, i.e., right before the task re-enters userland. If a task does not enter userland at all (for any reason), the existing FPSIMD state is preserved and may be reused by the owning task if it gets scheduled in again on the same CPU.
--- arch/arm64/include/asm/fpsimd.h | 4 ++ arch/arm64/include/asm/thread_info.h | 4 +- arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/fpsimd.c | 102 +++++++++++++++++++++++++++++------ arch/arm64/kernel/signal.c | 4 ++ 5 files changed, 98 insertions(+), 18 deletions(-)
arch/arm64/include/asm/fpsimd.h +4 -0
@@ -37,6 +37,8 @@ struct fpsimd_state {u32 fpcr; }; }; +/* the id of the last cpu to have restored this state */ +unsigned int last_cpu; };#if defined(__KERNEL__) && defined(CONFIG_COMPAT )
@@ -61,6 +63,8 @@ void fpsimd_set_task_state(struct fpsimd_state *state);struct user_fpsimd_state *fpsimd_get_user_state (struct task_struct *t);void fpsimd_set_user_state (struct task_struct *t,struct user_fpsimd_state *st); +void fpsimd_reload_fpstate (void ); +#endif #endif
arch/arm64/include/asm/thread_info.h +3 -1
@@ -100,6 +100,7 @@ static inline struct thread_info *current_thread_info(void)#define TIF_SIGPENDING 0#define TIF_NEED_RESCHED 1#define TIF_NOTIFY_RESUME 2/* callback before returning to user */ +#define TIF_FOREIGN_FPSTATE 3/* CPU's FP state is not current's */ #define TIF_SYSCALL_TRACE 8#define TIF_POLLING_NRFLAG 16#define TIF_MEMDIE 18/* is terminating due to OOM killer */
@@ -112,10 +113,11 @@ static inline struct thread_info *current_thread_info(void)#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)#define _TIF_32BIT (1 << TIF_32BIT)#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ - _TIF_NOTIFY_RESUME) + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)#endif /* __KERNEL__ */ #endif /* __ASM_THREAD_INFO_H */
arch/arm64/kernel/entry.S +1 -1
@@ -576,7 +576,7 @@ fast_work_pending: str x0, [sp, #S_X0] // returned x0 work_pending: tbnz x1, #TIF_NEED_RESCHED, work_resched - /* TIF_SIGPENDING or TIF_NOTIFY_RESUME case */ + /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ ldr x2, [sp, #S_PSTATE] mov x0, sp // 'regs' tst x2, #PSR_MODE_MASK // user mode regs?
arch/arm64/kernel/fpsimd.c +86 -16
162 lines not on this page · raw message
arch/arm64/kernel/signal.c +4 -0
@@ -416,4 +416,8 @@ asmlinkage void do_notify_resume(struct pt_regs *regs,clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs); } + +if (thread_flags & _TIF_FOREIGN_FPSTATE) +fpsimd_reload_fpstate (); + }
Signature · 1 line
1.8.3.2
Catalin Marinas
maintainerPATCH v2 2/4 arm64: defer reloading a task's FPSIMD state to userland resume
arch/arm64/include/asm/fpsimd.h struct fpsimd_state { · line 41
Quoted 10 lines · Ard Biesheuvel
arch/arm64/include/asm/fpsimd.h +2
-0@@ -37,6 +37,8 @@ struct fpsimd_state {u32 fpcr; }; }; +/* the id of the last cpu to have restored this state */ +unsigned int last_cpu;
Just "cpu" is enough.
Quoted 63 lines
arch/arm64/include/asm/fpsimd.h +2
-0};#if defined(__KERNEL__) && defined(CONFIG_COMPAT )@@ -61,6 +63,8 @@ void fpsimd_set_task_state(struct fpsimd_state *state);struct user_fpsimd_state *fpsimd_get_user_state (struct task_struct *t);void fpsimd_set_user_state (struct task_struct *t,struct user_fpsimd_state *st); +void fpsimd_reload_fpstate (void ); +#endif #endif arch/arm64/include/asm/thread_info.h +3
-1@@ -100,6 +100,7 @@ static inline struct thread_info *current_thread_info(void)#define TIF_SIGPENDING 0#define TIF_NEED_RESCHED 1#define TIF_NOTIFY_RESUME 2/* callback before returning to user */ +#define TIF_FOREIGN_FPSTATE 3/* CPU's FP state is not current's */ #define TIF_SYSCALL_TRACE 8#define TIF_POLLING_NRFLAG 16#define TIF_MEMDIE 18/* is terminating due to OOM killer */ @@ -112,10 +113,11 @@ static inline struct thread_info *current_thread_info(void)#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)#define _TIF_32BIT (1 << TIF_32BIT)#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ - _TIF_NOTIFY_RESUME) + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)#endif /* __KERNEL__ */ #endif /* __ASM_THREAD_INFO_H */ arch/arm64/kernel/entry.S +1
-1@@ -576,7 +576,7 @@ fast_work_pending: str x0, [sp, #S_X0] // returned x0 work_pending: tbnz x1, #TIF_NEED_RESCHED, work_resched - /* TIF_SIGPENDING or TIF_NOTIFY_RESUME case */ + /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ ldr x2, [sp, #S_PSTATE] mov x0, sp // 'regs' tst x2, #PSR_MODE_MASK // user mode regs?arch/arm64/kernel/fpsimd.c +5
-0@@ -39,6 +39,23 @@ void fpsimd_save_state(struct fpsimd_state *state);void fpsimd_load_state (struct fpsimd_state *state);/* +* In order to reduce the number of times the fpsimd state is needlessly saved +* and restored, keep track here of which task's userland owns the current state +* of the FPSIMD register file. +* +* This percpu variable points to the fpsimd_state.last_cpu field of the task
arch/arm64/kernel/fpsimd.c void fpsimd_save_state(struct fpsimd_state *state); · line 56
Quoted 10 lines
arch/arm64/kernel/fpsimd.c +10
-0+* whose FPSIMD state was most recently loaded onto this cpu. The last_cpu field +* itself contains the id of the cpu onto which the task's FPSIMD state was +* loaded most recently. So, to decide whether we can skip reloading the FPSIMD +* state, we need to check +* (a) whether this task was the last one to have its FPSIMD state loaded onto +* this cpu +* (b) whether this task may have manipulated its FPSIMD state on another cpu in +* the meantime +*/ +static DEFINE_PER_CPU(unsigned int *, fpsimd_last_task);
This can simply be struct fpsimd_state * to avoid &st->last_cpu. Also rename to fpsimd_state_last.
Quoted 20 lines
arch/arm64/kernel/fpsimd.c +10
-2+ +/* * Trapped FP/ASIMD access. */ void do_fpsimd_acc (unsigned int esr,struct pt_regs *regs)@@ -76,36 +93,84 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)void fpsimd_thread_switch (struct task_struct *next) { -/* check if not kernel threads */ -if (current->mm) +/* +* The thread flag TIF_FOREIGN_FPSTATE conveys that the userland FPSIMD +* state belonging to the current task is not present in the registers +* but has (already) been saved to memory in order for the kernel to be +* able to go off and use the registers for something else. Therefore, +* we must not (re)save the register contents if this flag is set. +*/ +if (current->mm && !test_thread_flag (TIF_FOREIGN_FPSTATE ))fpsimd_save_state (¤t->thread.fpsimd_state);
arch/arm64/kernel/fpsimd.c void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) · line 112
Quoted 10 lines
arch/arm64/kernel/fpsimd.c +8
-2-if (next->mm) -fpsimd_load_state (&next->thread.fpsimd_state); + +if (next->mm) { +/* +* If we are switching to a task whose most recent userland +* FPSIMD contents are already in the registers of *this* cpu, +* we can skip loading the state from memory. Otherwise, set +* the TIF_FOREIGN_FPSTATE flag so the state will be loaded +* upon the next entry of userland.
I would say "return" instead of "entry" (we enter the kernel and return to user space ;)).
arch/arm64/kernel/fpsimd.c void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) · line 124
Quoted 12 lines
arch/arm64/kernel/fpsimd.c +11
-0+*/ +struct fpsimd_state *st = &next->thread.fpsimd_state; + +if (__get_cpu_var (fpsimd_last_task) == &st->last_cpu + && st->last_cpu ==smp_processor_id ()) +clear_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); +else +set_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); + } }
I'm still trying to get my head around why we have 3 different type of checks for this (fpsimd_last_task, last_cpu and TIF). The code seems correct but I wonder whether we can reduce this to 2 checks?
Signature · 1 line
Catalin
PATCH v2 2/4 arm64: defer reloading a task's FPSIMD state to userland resume
Quoted 14 lines · Catalin Marinas
arch/arm64/include/asm/fpsimd.h struct fpsimd_state { · line 41
Quoted 10 lines · Ard Biesheuvel
arch/arm64/include/asm/fpsimd.h +2
-0@@ -37,6 +37,8 @@ struct fpsimd_state {u32 fpcr; }; }; +/* the id of the last cpu to have restored this state */ +unsigned int last_cpu;Just "cpu" is enough.
OK
[...]
Quoted 17 lines
Quoted 6 lines
arch/arm64/include/asm/fpsimd.h +5
-0/* +* In order to reduce the number of times the fpsimd state is needlessly saved +* and restored, keep track here of which task's userland owns the current state +* of the FPSIMD register file. +* +* This percpu variable points to the fpsimd_state.last_cpu field of the task arch/arm64/include/asm/fpsimd.h struct fpsimd_state {
Quoted 10 lines
arch/arm64/include/asm/fpsimd.h +10
-0+* whose FPSIMD state was most recently loaded onto this cpu. The last_cpu field +* itself contains the id of the cpu onto which the task's FPSIMD state was +* loaded most recently. So, to decide whether we can skip reloading the FPSIMD +* state, we need to check +* (a) whether this task was the last one to have its FPSIMD state loaded onto +* this cpu +* (b) whether this task may have manipulated its FPSIMD state on another cpu in +* the meantime +*/ +static DEFINE_PER_CPU(unsigned int *, fpsimd_last_task);
Quoted 3 lines
This can simply be struct
fpsimd_state* to avoid&st->last_cpu. Also rename tofpsimd_state_last.
OK
Quoted 25 lines
Quoted 25 lines
arch/arm64/include/asm/fpsimd.h +13
-4+ +/* * Trapped FP/ASIMD access. */ void do_fpsimd_acc (unsigned int esr,struct pt_regs *regs)@@ -76,36 +93,84 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)void fpsimd_thread_switch (struct task_struct *next) { -/* check if not kernel threads */ -if (current->mm) +/* +* The thread flag TIF_FOREIGN_FPSTATE conveys that the userland FPSIMD +* state belonging to the current task is not present in the registers +* but has (already) been saved to memory in order for the kernel to be +* able to go off and use the registers for something else. Therefore, +* we must not (re)save the register contents if this flag is set. +*/ +if (current->mm && !test_thread_flag (TIF_FOREIGN_FPSTATE ))fpsimd_save_state (¤t->thread.fpsimd_state); -if (next->mm) -fpsimd_load_state (&next->thread.fpsimd_state); + +if (next->mm) { +/*
Quoted 9 lines
arch/arm64/include/asm/fpsimd.h void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) · line 112
Quoted 5 lines
arch/arm64/include/asm/fpsimd.h +5
-0+* If we are switching to a task whose most recent userland +* FPSIMD contents are already in the registers of *this* cpu, +* we can skip loading the state from memory. Otherwise, set +* the TIF_FOREIGN_FPSTATE flag so the state will be loaded +* upon the next entry of userland. I would say "return" instead of "entry" (we enter the kernel and return to user space ;)).
OK
Quoted 13 lines
arch/arm64/include/asm/fpsimd.h void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) · line 124
Quoted 12 lines
arch/arm64/include/asm/fpsimd.h +11
-0+*/ +struct fpsimd_state *st = &next->thread.fpsimd_state; + +if (__get_cpu_var (fpsimd_last_task) == &st->last_cpu + && st->last_cpu ==smp_processor_id ()) +clear_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); +else +set_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); + } }
Quoted 4 lines
I'm still trying to get my head around why we have 3 different type of checks for this (
fpsimd_last_task,last_cpuand TIF). The code seems correct but I wonder whether we can reduce this to 2 checks?
Well, I suppose using the TIF flag is somewhat redundant, it is basically a shorthand for expressing that the following does /not/ hold
__get_cpu_var(fpsimd_last_state) == ¤t->thread.fpsimd_state && current->thread.fpsimd_state.cpu == smp_processor_id()
I suppose that the test at resume can tolerate the overhead, so I can rework the code to get rid of it.
Regards,
Ard.
Catalin Marinas
maintainerPATCH v2 2/4 arm64: defer reloading a task's FPSIMD state to userland resume
Quoted 19 lines · Ard Biesheuvel
Quoted 14 lines · Catalin Marinas
Quoted 11 lines · Ard Biesheuvel
arch/arm64/kernel/fpsimd.c +11
-0+*/ +struct fpsimd_state *st = &next->thread.fpsimd_state; + +if (__get_cpu_var (fpsimd_last_task) == &st->last_cpu + && st->last_cpu ==smp_processor_id ()) +clear_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); +else +set_ti_thread_flag (task_thread_info (next), +TIF_FOREIGN_FPSTATE ); + }Quoted 1 line
}
Quoted 3 lines
I'm still trying to get my head around why we have 3 different type of checks for this (
fpsimd_last_task,last_cpuand TIF). The code seems correct but I wonder whether we can reduce this to 2 checks?
Quoted 6 lines
Well, I suppose using the TIF flag is somewhat redundant, it is basically a shorthand for expressing that the following does /not/ hold
__get_cpu_var(fpsimd_last_state)==¤t->thread.fpsimd_state&¤t->thread.fpsimd_state.cpu==smp_processor_id()
OK, it starts to make more sense now ;).
Basically, if we only cared about context switching (rather than Neon in the kernel), we would have to always save the state of the scheduled out task but restore it only if the current hw state is different. A way to check this is fpsimd_last_state && cpu (I can?t really think of a better way).
With the addition of kernel_neon_begin/end(), we want to optimise this further by (a) only saving the state at context switch if it hasn?t been saved already (by kernel_neon_begin) and (b) defer the restoring to user space to avoid re-saving/restoring of the state.
Case (a) is when Neon is used between the syscall entry and switch_to() for a given thread. Case (b) is for scenarios where Neon is used between switch_to() and return to user. Are both of these likely? I think they are (e.g. sending->waiting->receiving).
Quoted 2 lines
I suppose that the test at resume can tolerate the overhead, so I can rework the code to get rid of it.
It may not be that simple since we need per-CPU variables retrieved in assembly. So we end up with a function call plus per-CPU variable checking and this must be done on the return from interrupt path as well. In which case the TIF flag is quicker as an optimisation. If I have any better idea I?ll let you know.
In the meantime, I think it?s ok to keep all three checks for different scenarios but please add some more explanation in the fpsimd.c file so that in a year time we still remember the logic (documenting the scenarios and when we check which TIF flag, per-CPU variable etc.).
Thanks,
Catalin
PATCH v2 3/4 arm64: add support for kernel mode NEON in interrupt context
In reply to Ard BiesheuvelThis patch modifies kernel_neon_begin() and kernel_neon_end(), so they may be called from any context. To address the case where only a couple of registers are needed, kernel_neon_begin_partial(u32) is introduced which takes as a parameter the number of bottom 'n' NEON q-registers required. To mark the end of such a partial section, the regular kernel_neon_end() should be used.
--- arch/arm64/include/asm/fpsimd.h | 17 ++++++++++++++ arch/arm64/include/asm/fpsimdmacros.h | 37 ++++++++++++++++++++++++++++++ arch/arm64/include/asm/neon.h | 6 ++++- arch/arm64/kernel/entry-fpsimd.S | 24 +++++++++++++++++++ arch/arm64/kernel/fpsimd.c | 43 +++++++++++++++++++++++------------ 5 files changed, 112 insertions(+), 15 deletions(-)
arch/arm64/include/asm/fpsimd.h +17 -0
@@ -41,6 +41,19 @@ struct fpsimd_state {unsigned int last_cpu; }; +/* +* Struct for stacking the bottom 'n' FP/SIMD registers. +* Mainly intended for kernel use of v8 Crypto Extensions which only +* needs a few registers and may need to execute in atomic context. +*/ +struct fpsimd_partial_state { +u32 num_regs; +u32 fpsr; +u32 fpcr; +__uint128_t vregs[32 ]__aligned(16) ; +}__aligned(16) ; + +#if defined(__KERNEL__) && defined(CONFIG_COMPAT )/* Masks for extracting the FPSR and FPCR from the FPSCR */ #define VFP_FPSCR_STAT_MASK 0xf800009f
@@ -65,6 +78,10 @@ void fpsimd_set_user_state(struct task_struct *t, struct user_fpsimd_state *st);void fpsimd_reload_fpstate (void ); +extern void fpsimd_save_partial_state (struct fpsimd_partial_state *state, +u32 num_regs); +extern void fpsimd_load_partial_state (struct fpsimd_partial_state *state); +#endif #endif
arch/arm64/include/asm/fpsimdmacros.h +37 -0
@@ -62,3 +62,40 @@ ldr w\tmpnr, [\state, #16 * 2 + 4] msr fpcr, x\tmpnr .endm + +.altmacro +.macro q2op, op, q1, q2, state + \op q\q1, q\q2, [\state, # -16 * \q1 - 16] +.endm + +.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 + mrs x\tmpnr1, fpsr + str w\numnr, [\state] + mrs x\tmpnr2, fpcr + stp w\tmpnr1, w\tmpnr2, [\state, #4] + adr x\tmpnr1, 0f + add \state, \state, x\numnr, lsl #4 + sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 + br x\tmpnr1 + .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 + qb = \qa + 1 + q2op stp, \qa, %qb, \state + .endr +0: +.endm + +.macro fpsimd_restore_partial state, tmpnr1, tmpnr2 + ldp w\tmpnr1, w\tmpnr2, [\state, #4] + msr fpsr, x\tmpnr1 + msr fpcr, x\tmpnr2 + adr x\tmpnr1, 0f + ldr w\tmpnr2, [\state] + add \state, \state, x\tmpnr2, lsl #4 + sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 + br x\tmpnr1 + .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 + qb = \qa + 1 + q2op ldp, \qa, %qb, \state + .endr +0: +.endm
arch/arm64/include/asm/neon.h +5 -1
@@ -8,7 +8,11 @@* published by the Free Software Foundation. */ +#include <linux/types.h> +#define cpu_has_neon () (1) -void kernel_neon_begin (void ); +#define kernel_neon_begin () kernel_neon_begin_partial(32) + +void kernel_neon_begin_partial (u32 num_regs);void kernel_neon_end (void );
arch/arm64/kernel/entry-fpsimd.S +24 -0
@@ -41,3 +41,27 @@ ENTRY(fpsimd_load_state) fpsimd_restore x0, 8 ret ENDPROC(fpsimd_load_state) + +#ifdef CONFIG_KERNEL_MODE_NEON + +/* + * Save the bottom n FP registers. + * + * x0 - pointer to struct fpsimd_partial_state + */ +ENTRY(fpsimd_save_partial_state) + fpsimd_save_partial x0, 1, 8, 9 + ret +ENDPROC(fpsimd_load_partial_state) + +/* + * Load the bottom n FP registers. + * + * x0 - pointer to struct fpsimd_partial_state + */ +ENTRY(fpsimd_load_partial_state) + fpsimd_restore_partial x0, 8, 9 + ret +ENDPROC(fpsimd_load_partial_state) + +#endif
arch/arm64/kernel/fpsimd.c +29 -14
58 lines not on this page · raw message
Signature · 1 line
1.8.3.2
PATCH v2 4/4 arm64: add Crypto Extensions based synchronous core AES cipher
In reply to Ard BiesheuvelThis implements the core AES cipher using the Crypto Extensions, using only NEON registers q0 - q3.
--- arch/arm64/Makefile | 1 + arch/arm64/crypto/Makefile | 13 ++++ arch/arm64/crypto/aes-ce-cipher.c | 134 ++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 6 ++ 4 files changed, 154 insertions(+) create mode 100644 arch/arm64/crypto/Makefile create mode 100644 arch/arm64/crypto/aes-ce-cipher.c
arch/arm64/Makefile +1 -0
@@ -45,6 +45,7 @@ export TEXT_OFFSET GZFLAGS core-y += arch/arm64/kernel/ arch/arm64/mm/ core-$(CONFIG_KVM) += arch/arm64/kvm/ core-$(CONFIG_XEN) += arch/arm64/xen/ +core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ libs-y := arch/arm64/lib/ $(libs-y) libs-y += $(LIBGCC)
arch/arm64/crypto/Makefile new file +13 -0
@@ -0,0 +1,13 @@ +# +# linux/arch/arm64/crypto/Makefile +# +# Copyright (C) 2013 Linaro Ltd <[email protected]> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# + +obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o + +CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
arch/arm64/crypto/aes-ce-cipher.c new file +134 -0
134 lines not on this page · raw message
crypto/Kconfig +6 -0
@@ -791,6 +791,12 @@ config CRYPTO_AES_ARM_BS This implementation does not rely on any lookup tables so it is believed to be invulnerable to cache timing attacks. +config CRYPTO_AES_ARM64_CE + tristate "Synchronous AES cipher using ARMv8 Crypto Extensions" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI
Signature · 1 line
1.8.3.2