Initial Spectre variant 1 patches
Russell King - ARM Linux
maintainercover Initial Spectre variant 1 patches
Hi,
This series adds initial support for the Spectre variant 1 workarounds to the kernel. This does _not_ cover the user accessors, which are still in development.
The series adds support for the array_index_mask_nospec() macro, used by generic kernel code to mask out of bounds pointers, and also adds the syscall table hardening.
arch/arm/include/asm/assembler.h | 8 ++++++++ arch/arm/include/asm/barrier.h | 31 +++++++++++++++++++++++++++++++ arch/arm/kernel/entry-common.S | 14 +++----------- arch/arm/kernel/entry-header.S | 25 +++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 11 deletions(-)
Signature · 3 lines
RMK's Patch system: http://www.armlinux.org.uk/developer/patches/ FTTC broadband for 0.8mile line in suburbia: sync at 8.8Mbps down 630kbps up According to speedtest.net: 8.21Mbps down 510kbps up
PATCH 1/3 ARM: spectre-v1: add speculation barrier (csdb) macros
Add assembly and C macros for the new CSDB instruction.
Signed-off-by Russell King <[email protected]>
--- arch/arm/include/asm/assembler.h | 8 ++++++++ arch/arm/include/asm/barrier.h | 13 +++++++++++++ 2 files changed, 21 insertions(+)
arch/arm/include/asm/assembler.h +8 -0
@@ -447,6 +447,14 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .size \name , . - \name .endm + .macro csdb +#ifdef CONFIG_THUMB2_KERNEL + .inst.w 0xf3af8014 +#else + .inst 0xe320f014 +#endif + .endm + .macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req#ifndef CONFIG_CPU_USE_DOMAINS adds \tmp, \addr, #\size - 1
arch/arm/include/asm/barrier.h +13 -0
@@ -17,6 +17,12 @@#define isb (option) __asm__ __volatile__ ("isb " #option : : : "memory")#define dsb (option) __asm__ __volatile__ ("dsb " #option : : : "memory")#define dmb (option) __asm__ __volatile__ ("dmb " #option : : : "memory") +#ifdef CONFIG_THUMB2_KERNEL +#define CSDB ".inst.w 0xf3af8014" +#else +#define CSDB ".inst 0xe320f014" +#endif +#define csdb () __asm__ __volatile__(CSDB : : : "memory")#elif defined(CONFIG_CPU_XSC3 ) || __LINUX_ARM_ARCH__ ==6 #define isb (x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \ : : "r" (0) : "memory")
@@ -37,6 +43,13 @@#define dmb (x) __asm__ __volatile__ ("" : : : "memory")#endif +#ifndef CSDB +#define CSDB +#endif +#ifndef csdb +#define csdb () +#endif +#ifdef CONFIG_ARM_HEAVY_MB extern void (*soc_mb)(void );extern void arm_heavy_mb (void );
Signature · 1 line
2.7.4
PATCH 1/3 ARM: spectre-v1: add speculation barrier (csdb) macros
Quoted 3 lines · Russell King
Add assembly and C macros for the new CSDB instruction.
Signed-off-by Russell King <[email protected]>
Looks correct to me per the Cache Speculation Side-chaneels 2.1 document.
Acked-by Mark Rutland <[email protected]>
Mark.
Quoted 63 lines
--- arch/arm/include/asm/assembler.h | 8 ++++++++ arch/arm/include/asm/barrier.h | 13 +++++++++++++ 2 files changed, 21 insertions(+)arch/arm/include/asm/assembler.h +8
-0@@ -447,6 +447,14 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .size \name , . - \name .endm + .macro csdb +#ifdef CONFIG_THUMB2_KERNEL + .inst.w 0xf3af8014 +#else + .inst 0xe320f014 +#endif + .endm + .macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req#ifndef CONFIG_CPU_USE_DOMAINS adds \tmp, \addr, #\size - 1arch/arm/include/asm/barrier.h +13
-0@@ -17,6 +17,12 @@#define isb (option) __asm__ __volatile__ ("isb " #option : : : "memory")#define dsb (option) __asm__ __volatile__ ("dsb " #option : : : "memory")#define dmb (option) __asm__ __volatile__ ("dmb " #option : : : "memory") +#ifdef CONFIG_THUMB2_KERNEL +#define CSDB ".inst.w 0xf3af8014" +#else +#define CSDB ".inst 0xe320f014" +#endif +#define csdb () __asm__ __volatile__(CSDB : : : "memory")#elif defined(CONFIG_CPU_XSC3 ) || __LINUX_ARM_ARCH__ ==6 #define isb (x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \ : : "r" (0) : "memory")@@ -37,6 +43,13 @@#define dmb (x) __asm__ __volatile__ ("" : : : "memory")#endif +#ifndef CSDB +#define CSDB +#endif +#ifndef csdb +#define csdb () +#endif +#ifdef CONFIG_ARM_HEAVY_MB extern void (*soc_mb)(void );extern void arm_heavy_mb (void );Signature · 7 lines
2.7.4 _______________________________________________ linux-arm-kernel mailing list linux-arm-kernel at lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH 2/3 ARM: spectre-v1: add array_index_mask_nospec() implementation
In reply to Russell King - ARM LinuxAdd an implementation of the array_index_mask_nospec() function for mitigating Spectre variant 1 throughout the kernel.
Signed-off-by Russell King <[email protected]>
--- arch/arm/include/asm/barrier.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
arch/arm/include/asm/barrier.h +18 -0
@@ -76,6 +76,24 @@ extern void arm_heavy_mb(void);#define __smp_rmb () __smp_mb()#define __smp_wmb () dmb(ishst) +#ifdef CONFIG_CPU_SPECTRE +static inline unsigned long array_index_mask_nospec (unsigned long idx, +unsigned long sz) +{ +unsigned long mask; + + asm("cmp %1, %2\n" +" sbc %0, %1, %1\n" +CSDB + :"=r" (mask) + :"r" (idx),"Ir" (sz) + :"cc" ); + +return mask; +} +#define array_index_mask_nospec array_index_mask_nospec +#endif +#include <asm-generic/barrier.h> #endif /* !__ASSEMBLY__ */
Signature · 1 line
2.7.4
PATCH 2/3 ARM: spectre-v1: add array_index_mask_nospec() implementation
Quoted 23 lines · Russell King
Add an implementation of the
array_index_mask_nospec()function for mitigating Spectre variant 1 throughout the kernel.Signed-off-by Russell King <[email protected]>--- arch/arm/include/asm/barrier.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)arch/arm/include/asm/barrier.h +7
-0@@ -76,6 +76,24 @@ extern void arm_heavy_mb(void);#define __smp_rmb () __smp_mb()#define __smp_wmb () dmb(ishst) +#ifdef CONFIG_CPU_SPECTRE +static inline unsigned long array_index_mask_nospec (unsigned long idx, +unsigned long sz) +{ +unsigned long mask; + + asm( "cmp %1, %2\n"
arch/arm/include/asm/barrier.h extern void arm_heavy_mb(void); · line 95
Quoted 10 lines
arch/arm/include/asm/barrier.h +10
-0+ " sbc %0, %1, %1\n" + CSDB + : "=r" (mask) + : "r" (idx), "Ir" (sz) + : "cc"); + + return mask; +} +#define array_index_mask_nospec array_index_mask_nospec +#endif
I believe that the asm should be volatile, otherwise there are cases where the compiler could remove an invocation that's redundant under architectural execution, but required under speculation [1].
With that made volatile:
Acked-by Mark Rutland <[email protected]>
Thanks,
Mark.
[1] https://lkml.kernel.org/r/20180530060546.jnjoltsturoduohh at salmiak
PATCH 3/3 ARM: spectre-v1: fix syscall entry
In reply to Russell King - ARM LinuxPrevent speculation at the syscall table decoding by clamping the index used to zero on invalid system call numbers, and using the csdb speculative barrier.
Signed-off-by Russell King <[email protected]>
--- arch/arm/kernel/entry-common.S | 14 +++----------- arch/arm/kernel/entry-header.S | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 11 deletions(-)
arch/arm/kernel/entry-common.S +3 -11
@@ -242,9 +242,7 @@ ENTRY(vector_swi) tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? bne __sys_trace - cmp scno, #NR_syscalls @ check upper syscall limit - badr lr, ret_fast_syscall @ return address - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + invoke_syscall tbl, scno, r10, ret_fast_syscall add r1, sp, #S_OFF 2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
@@ -278,14 +276,8 @@ ENDPROC(vector_swi) mov r1, scno add r0, sp, #S_OFF bl syscall_trace_enter - - badr lr, __sys_trace_return @ return address - mov scno, r0 @ syscall number (possibly new) - add r1, sp, #S_R0 + S_OFF @ pointer to regs - cmp scno, #NR_syscalls @ check upper syscall limit - ldmccia r1, {r0 - r6} @ have to reload r0 - r6 - stmccia sp, {r4, r5} @ and update the stack args - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + mov scno, r0 + invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1 cmp scno, #-1 @ skip the syscall? bne 2b add sp, sp, #S_OFF @ restore stack
arch/arm/kernel/entry-header.S +25 -0
@@ -378,6 +378,31 @@ #endif .endm + .macro invoke_syscall, table, nr, tmp, ret, reload=0 +#ifdef CONFIG_CPU_SPECTRE + mov \tmp, \nr + cmp \tmp, #NR_syscalls @ check upper syscall limit + movcs \tmp, #0 + csdb + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine +#else + cmp \nr, #NR_syscalls @ check upper syscall limit + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine +#endif + .endm + /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - r0 to r6.
Signature · 1 line
2.7.4
PATCH 3/3 ARM: spectre-v1: fix syscall entry
Quoted 32 lines · Russell King
Prevent speculation at the syscall table decoding by clamping the index used to zero on invalid system call numbers, and using the csdb speculative barrier.
Signed-off-by Russell King <[email protected]>--- arch/arm/kernel/entry-common.S | 14 +++----------- arch/arm/kernel/entry-header.S | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 11 deletions(-)arch/arm/kernel/entry-common.S +1
-6@@ -242,9 +242,7 @@ ENTRY(vector_swi) tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? bne __sys_trace - cmp scno, #NR_syscalls @ check upper syscall limit - badr lr, ret_fast_syscall @ return address - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + invoke_syscall tbl, scno, r10, ret_fast_syscall add r1, sp, #S_OFF 2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)@@ -278,14 +276,8 @@ ENDPROC(vector_swi) mov r1, scno add r0, sp, #S_OFF bl syscall_trace_enter - - badr lr, __sys_trace_return @ return address - mov scno, r0 @ syscall number (possibly new)
arch/arm/kernel/entry-common.S ENDPROC(vector_swi) · line 283
Quoted 10 lines
arch/arm/kernel/entry-common.S +2
-5- add r1, sp, #S_R0 + S_OFF @ pointer to regs - cmp scno, #NR_syscalls @ check upper syscall limit - ldmccia r1, {r0 - r6} @ have to reload r0 - r6 - stmccia sp, {r4, r5} @ and update the stack args - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + mov scno, r0 + invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1 cmp scno, #-1 @ skip the syscall? bne 2b add sp, sp, #S_OFF @ restore stack
I *think* you might also need to adjust sys_syscall.
Quoted 5 lines
arch/arm/kernel/entry-header.S +0
-0
arch/arm/kernel/entry-header.S line 386
Quoted 9 lines
arch/arm/kernel/entry-header.S +6
-0@@ -378,6 +378,31 @@ #endif .endm + .macro invoke_syscall, table, nr, tmp, ret, reload=0 +#ifdef CONFIG_CPU_SPECTRE + mov \tmp, \nr + cmp \tmp, #NR_syscalls @ check upper syscall limit + movcs \tmp, #0 + csdb
To the best of my understanding, this sequence is sufficient.
Quoted 8 lines
arch/arm/kernel/entry-header.S +8
-0+ badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine +#else
arch/arm/kernel/entry-header.S line 404
Quoted 10 lines
arch/arm/kernel/entry-header.S +10
-0+ cmp \nr, #NR_syscalls @ check upper syscall limit + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine +#endif + .endm
I couldn't find a nice way to avoid dupliacting the body without more ifdeffery, given the nr parameter can't be clobbered.
So modulo sys_syscall, this looks good to me.
Mark.
PATCH v2 3/3 ARM: spectre-v1: fix syscall entry
Prevent speculation at the syscall table decoding by clamping the index used to zero on invalid system call numbers, and using the csdb speculative barrier.
Signed-off-by Russell King <[email protected]>
--- arch/arm/kernel/entry-common.S | 18 +++++++----------- arch/arm/kernel/entry-header.S | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-)
arch/arm/kernel/entry-common.S +7 -11
@@ -242,9 +242,7 @@ ENTRY(vector_swi) tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? bne __sys_trace - cmp scno, #NR_syscalls @ check upper syscall limit - badr lr, ret_fast_syscall @ return address - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + invoke_syscall tbl, scno, r10, ret_fast_syscall add r1, sp, #S_OFF 2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
@@ -278,14 +276,8 @@ ENDPROC(vector_swi) mov r1, scno add r0, sp, #S_OFF bl syscall_trace_enter - - badr lr, __sys_trace_return @ return address - mov scno, r0 @ syscall number (possibly new) - add r1, sp, #S_R0 + S_OFF @ pointer to regs - cmp scno, #NR_syscalls @ check upper syscall limit - ldmccia r1, {r0 - r6} @ have to reload r0 - r6 - stmccia sp, {r4, r5} @ and update the stack args - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + mov scno, r0 + invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1 cmp scno, #-1 @ skip the syscall? bne 2b add sp, sp, #S_OFF @ restore stack
@@ -363,6 +355,10 @@ ENTRY(\sym) bic scno, r0, #__NR_OABI_SYSCALL_BASE cmp scno, #__NR_syscall - __NR_SYSCALL_BASE cmpne scno, #NR_syscalls @ check range +#ifdef CONFIG_CPU_SPECTRE + movhs scno, #0 + csdb +#endif stmloia sp, {r5, r6} @ shuffle args movlo r0, r1 movlo r1, r2
arch/arm/kernel/entry-header.S +25 -0
@@ -378,6 +378,31 @@ #endif .endm + .macro invoke_syscall, table, nr, tmp, ret, reload=0 +#ifdef CONFIG_CPU_SPECTRE + mov \tmp, \nr + cmp \tmp, #NR_syscalls @ check upper syscall limit + movcs \tmp, #0 + csdb + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine +#else + cmp \nr, #NR_syscalls @ check upper syscall limit + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine +#endif + .endm + /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - r0 to r6.
Signature · 1 line
2.7.4
PATCH v2 3/3 ARM: spectre-v1: fix syscall entry
Quoted 5 lines · Russell King
Prevent speculation at the syscall table decoding by clamping the index used to zero on invalid system call numbers, and using the csdb speculative barrier.
Signed-off-by Russell King <[email protected]>
Acked-by Mark Rutland <[email protected]>
Mark.
Quoted 87 lines
--- arch/arm/kernel/entry-common.S | 18 +++++++----------- arch/arm/kernel/entry-header.S | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-)arch/arm/kernel/entry-common.S +7
-11@@ -242,9 +242,7 @@ ENTRY(vector_swi) tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? bne __sys_trace - cmp scno, #NR_syscalls @ check upper syscall limit - badr lr, ret_fast_syscall @ return address - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + invoke_syscall tbl, scno, r10, ret_fast_syscall add r1, sp, #S_OFF 2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)@@ -278,14 +276,8 @@ ENDPROC(vector_swi) mov r1, scno add r0, sp, #S_OFF bl syscall_trace_enter - - badr lr, __sys_trace_return @ return address - mov scno, r0 @ syscall number (possibly new) - add r1, sp, #S_R0 + S_OFF @ pointer to regs - cmp scno, #NR_syscalls @ check upper syscall limit - ldmccia r1, {r0 - r6} @ have to reload r0 - r6 - stmccia sp, {r4, r5} @ and update the stack args - ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + mov scno, r0 + invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1 cmp scno, #-1 @ skip the syscall? bne 2b add sp, sp, #S_OFF @ restore stack@@ -363,6 +355,10 @@ ENTRY(\sym) bic scno, r0, #__NR_OABI_SYSCALL_BASE cmp scno, #__NR_syscall - __NR_SYSCALL_BASE cmpne scno, #NR_syscalls @ check range +#ifdef CONFIG_CPU_SPECTRE + movhs scno, #0 + csdb +#endif stmloia sp, {r5, r6} @ shuffle args movlo r0, r1 movlo r1, r2arch/arm/kernel/entry-header.S +25
-0@@ -378,6 +378,31 @@ #endif .endm + .macro invoke_syscall, table, nr, tmp, ret, reload=0 +#ifdef CONFIG_CPU_SPECTRE + mov \tmp, \nr + cmp \tmp, #NR_syscalls @ check upper syscall limit + movcs \tmp, #0 + csdb + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine +#else + cmp \nr, #NR_syscalls @ check upper syscall limit + badr lr, \ret @ return address + .if \reload + add r1, sp, #S_R0 + S_OFF @ pointer to regs + ldmccia r1, {r0 - r6} @ reload r0-r6 + stmccia sp, {r4, r5} @ update stack arguments + .endif + ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine +#endif + .endm + /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - r0 to r6.Signature · 1 line
2.7.4
cover Initial Spectre variant 1 patches
In reply to Russell King - ARM LinuxQuoted 9 lines · Russell King - ARM Linux
Hi,
This series adds initial support for the Spectre variant 1 workarounds to the kernel. This does _not_ cover the user accessors, which are still in development.
The series adds support for the
array_index_mask_nospec()macro, used by generic kernel code to mask out of bounds pointers, and also adds the syscall table hardening.
For the series including v2 patch of 3/3:
Boot-tested-by Tony Lindgren <[email protected]>
Reviewed-by Tony Lindgren <[email protected]>