Mitigations against spectre-v1 in the arm64 Linux kernel
cover Mitigations against spectre-v1 in the arm64 Linux kernel
Hi all,
This series implements some mitigations against spectre-v1 for arm64. It broadly follows the work that has been done for x86, by implementing:
* A back-end version of array_index_mask_nospec() to suppress any compiler optimisations that could introduce unwanted speculative paths
* Masking of the syscall number to restrict speculation through the syscall table
* Masking of __user pointers prior to deference in uaccess routines
The latter introduces complications in access_ok and set_fs which are also handled in this series.
Cheers,
Will
--->8
Robin Murphy (3):
arm64: Implement array_index_mask_nospec()
arm64: Make USER_DS an inclusive limit
arm64: Use pointer masking to limit uaccess speculation
Will Deacon (6):
arm64: barrier: Add CSDB macros to control data-value prediction
arm64: entry: Ensure branch through syscall table is bounded under
speculation
arm64: uaccess: Prevent speculative use of the current addr_limit
arm64: uaccess: Don't bother eliding access_ok checks in
__{get,put}_user arm64: uaccess: Mask __user pointers for __arch_{clear,copy_*}_user
arm64: futex: Mask __user pointers prior to dereference
arch/arm64/include/asm/assembler.h | 18 +++++ arch/arm64/include/asm/barrier.h | 22 ++++++ arch/arm64/include/asm/futex.h | 9 ++- arch/arm64/include/asm/processor.h | 3 + arch/arm64/include/asm/uaccess.h | 155 +++++++++++++++++++++++++------------ arch/arm64/kernel/arm64ksyms.c | 4 +- arch/arm64/kernel/entry.S | 6 +- arch/arm64/lib/clear_user.S | 6 +- arch/arm64/lib/copy_in_user.S | 5 +- arch/arm64/mm/fault.c | 4 +- 10 files changed, 170 insertions(+), 62 deletions(-)
Signature · 1 line
2.1.4
PATCH 1/9 arm64: barrier: Add CSDB macros to control data-value prediction
For CPUs capable of data value prediction, CSDB waits for any outstanding predictions to architecturally resolve before allowing speculative execution to continue. Provide macros to expose it to the arch code.
--- arch/arm64/include/asm/assembler.h | 7 +++++++ arch/arm64/include/asm/barrier.h | 1 + 2 files changed, 8 insertions(+)
arch/arm64/include/asm/assembler.h +7 -0
@@ -116,6 +116,13 @@ .endm /* + * Value prediction barrier + */ + .macro csdb + hint #20 + .endm + +/* * NOP sequence */ .macro nops, num
arch/arm64/include/asm/barrier.h +1 -0
@@ -32,6 +32,7 @@#define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define psb_csync () asm volatile("hint #17" : : : "memory") +#define csdb () asm volatile("hint #20" : : : "memory")#define mb () dsb(sy)#define rmb () dsb(ld)
Signature · 1 line
2.1.4
PATCH 2/9 arm64: Implement array_index_mask_nospec()
In reply to Will DeaconFrom: Robin Murphy <[email protected]>
Provide an optimised, assembly implementation of array_index_mask_nospec() for arm64 so that the compiler is not in a position to transform the code in ways which affect its ability to inhibit speculation (e.g. by introducing conditional branches).
This is similar to the sequence used by x86, modulo architectural differences in the carry/borrow flags.
--- arch/arm64/include/asm/barrier.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)
arch/arm64/include/asm/barrier.h +21 -0
@@ -41,6 +41,27 @@#define dma_rmb () dmb(oshld)#define dma_wmb () dmb(oshst) +/* +* Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz +* and 0 otherwise. +*/ +#define array_index_mask_nospec array_index_mask_nospec +static inline unsigned long array_index_mask_nospec (unsigned long idx, +unsigned long sz) +{ +unsigned long mask; + + asmvolatile ( +" cmp %1, %2\n" +" sbc %0, xzr, xzr\n" + :"=r" (mask) + :"r" (idx),"Ir" (sz) + :"cc" ); + +csdb (); +return mask; +} +#define __smp_mb () dmb(ish)#define __smp_rmb () dmb(ishld)#define __smp_wmb () dmb(ishst)
Signature · 1 line
2.1.4
PATCH 3/9 arm64: Make USER_DS an inclusive limit
In reply to Will DeaconFrom: Robin Murphy <[email protected]>
Currently, USER_DS represents an exclusive limit while KERNEL_DS is inclusive. In order to do some clever trickery for speculation-safe masking, we need them both to behave equivalently - there aren't enough bits to make KERNEL_DS exclusive, so we have precisely one option. This also happens to correct a longstanding false negative for a range ending on the very top byte of kernel memory.
Mark Rutland points out that we've actually got the semantics of addresses vs. segments muddled up in most of the places we need to amend, so shuffle the {USER,KERNEL}_DS definitions around such that we can correct those properly instead of just pasting "-1"s everywhere.
--- arch/arm64/include/asm/processor.h | 3 +++ arch/arm64/include/asm/uaccess.h | 45 ++++++++++++++++++++++---------------- arch/arm64/kernel/entry.S | 4 ++-- arch/arm64/mm/fault.c | 4 ++-- 4 files changed, 33 insertions(+), 23 deletions(-)
arch/arm64/include/asm/processor.h +3 -0
@@ -21,6 +21,9 @@#define TASK_SIZE_64 (UL(1) << VA_BITS) +#define KERNEL_DS UL(-1) +#define USER_DS (TASK_SIZE_64 - 1) +#ifndef __ASSEMBLY__/*
arch/arm64/include/asm/uaccess.h +26 -19
65 lines not on this page · raw message
arch/arm64/kernel/entry.S +2 -2
@@ -167,10 +167,10 @@ alternative_else_nop_endif .else add x21, sp, #S_FRAME_SIZE get_thread_info tsk - /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ + /* Save the task's original addr_limit and set USER_DS */ ldr x20, [tsk, #TSK_TI_ADDR_LIMIT] str x20, [sp, #S_ORIG_ADDR_LIMIT] - mov x20, #TASK_SIZE_64 + mov x20, #USER_DS str x20, [tsk, #TSK_TI_ADDR_LIMIT] /* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */ .endif /* \el == 0 */
arch/arm64/mm/fault.c +2 -2
@@ -240,7 +240,7 @@ static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs,if (fsc_type == ESR_ELx_FSC_PERM)return true; -if (addr <USER_DS &&system_uses_ttbr0_pan ()) +if (addr <TASK_SIZE &&system_uses_ttbr0_pan ())return fsc_type == ESR_ELx_FSC_FAULT && (regs->pstate &PSR_PAN_BIT );
@@ -414,7 +414,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, mm_flags |=FAULT_FLAG_WRITE ; } -if (addr <USER_DS &&is_permission_fault (esr, regs, addr)) { +if (addr <TASK_SIZE &&is_permission_fault (esr, regs, addr)) {/* regs->orig_addr_limit may be 0 if we entered from EL0 */ if (regs->orig_addr_limit ==KERNEL_DS )die ("Accessing user space memory with fs=KERNEL_DS" , regs, esr);
Signature · 1 line
2.1.4
PATCH 4/9 arm64: Use pointer masking to limit uaccess speculation
In reply to Will DeaconFrom: Robin Murphy <[email protected]>
Similarly to x86, mitigate speculation past an access_ok() check by masking the pointer against the address limit before use.
Even if we don't expect speculative writes per se, it is plausible that a CPU may still speculate at least as far as fetching a cache line for writing, hence we also harden put_user() and clear_user() for peace of mind.
--- arch/arm64/include/asm/uaccess.h | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-)
arch/arm64/include/asm/uaccess.h +23 -3
@@ -228,6 +228,26 @@ static inline void uaccess_enable_not_uao(void) }/* +* Sanitise a uaccess pointer such that it becomes NULL if above the +* current addr_limit. +*/ +#define uaccess_mask_ptr (ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr) +static inline void __user *__uaccess_mask_ptr (const void __user *ptr) +{ +void __user *safe_ptr; + + asmvolatile ( +" bics xzr, %1, %2\n" +" csel %0, %1, xzr, eq\n" + :"=&r" (safe_ptr) + :"r" (ptr),"r" (current_thread_info ()->addr_limit) + :"cc" ); + +csdb (); +return safe_ptr; +} + +/* * The "__xxx" versions of the user access functions do not verify the address * space - it must have been done previously with a separate "access_ok()" * call.
@@ -297,7 +317,7 @@ do { \ __typeof__(*(ptr)) __user *__p = (ptr); \might_fault (); \access_ok (VERIFY_READ , __p,sizeof (*__p)) ? \ -__get_user ((x), __p) : \ + __p =uaccess_mask_ptr (__p),__get_user ((x), __p) : \ ((x) =0 , -EFAULT ); \ })
@@ -361,7 +381,7 @@ do { \ __typeof__(*(ptr)) __user *__p = (ptr); \might_fault (); \access_ok (VERIFY_WRITE , __p,sizeof (*__p)) ? \ -__put_user ((x), __p) : \ + __p =uaccess_mask_ptr (__p),__put_user ((x), __p) : \ -EFAULT ; \ })
@@ -377,7 +397,7 @@ extern unsigned long __must_check __clear_user(void __user *addr, unsigned longstatic inline unsigned long __must_check clear_user (void __user *to,unsigned long n) {if (access_ok (VERIFY_WRITE , to, n)) - n =__clear_user (to, n); + n =__clear_user (__uaccess_mask_ptr (to), n);return n; }
Signature · 1 line
2.1.4
PATCH 5/9 arm64: entry: Ensure branch through syscall table is bounded under speculation
In reply to Will DeaconIn a similar manner to array_index_mask_nospec, this patch introduces an assembly macro (mask_nospec64) which can be used to bound a value under speculation. This macro is then used to ensure that the indirect branch through the syscall table is bounded under speculation, with out-of-range addresses speculating as calls to sys_io_setup (0).
--- arch/arm64/include/asm/assembler.h | 11 +++++++++++ arch/arm64/kernel/entry.S | 2 ++ 2 files changed, 13 insertions(+)
arch/arm64/include/asm/assembler.h +11 -0
@@ -123,6 +123,17 @@ .endm /* + * Sanitise a 64-bit bounded index wrt speculation, returning zero if out + * of bounds. + */ + .macro mask_nospec64, idx, limit, tmp + sub \tmp, \idx, \limit + bic \tmp, \tmp, \idx + and \idx, \idx, \tmp, asr #63 + csdb + .endm + +/* * NOP sequence */ .macro nops, num
arch/arm64/kernel/entry.S +2 -0
@@ -378,6 +378,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 * x7 is reserved for the system call number in 32-bit mode. */ wsc_nr .req w25 // number of system calls +xsc_nr .req x25 // number of system calls (zero-extended) wscno .req w26 // syscall number xscno .req x26 // syscall number (zero-extended) stbl .req x27 // syscall table pointer
@@ -935,6 +936,7 @@ el0_svc_naked: // compat entry point b.ne __sys_trace cmp wscno, wsc_nr // check upper syscall limit b.hs ni_sys + mask_nospec64 xscno, xsc_nr, x19 // enforce bounds for syscall number ldr x16, [stbl, xscno, lsl #3] // address in the syscall table blr x16 // call sys_* routine b ret_fast_syscall
Signature · 1 line
2.1.4
PATCH 6/9 arm64: uaccess: Prevent speculative use of the current addr_limit
In reply to Will DeaconA mispredicted conditional call to set_fs could result in the wrong addr_limit being forwarded under speculation to a subsequent access_ok check, potentially forming part of a spectre-v1 attack using uaccess routines.
This patch prevents this forwarding from taking place, but putting heavy barriers in set_fs after writing the addr_limit.
--- arch/arm64/include/asm/uaccess.h | 7 +++++++ 1 file changed, 7 insertions(+)
arch/arm64/include/asm/uaccess.h +7 -0
@@ -42,6 +42,13 @@ static inline void set_fs(mm_segment_t fs) {current_thread_info ()->addr_limit = fs; +/* +* Prevent a mispredicted conditional call to set_fs from forwarding +* the wrong address limit to access_ok under speculation. +*/ +dsb (nsh); +isb (); +/* On user-mode return, check fs is correct */ set_thread_flag (TIF_FSCHECK );
Signature · 1 line
2.1.4
PATCH 7/9 arm64: uaccess: Don't bother eliding access_ok checks in __{get, put}_user
In reply to Will Deaconaccess_ok isn't an expensive operation once the addr_limit for the current thread has been loaded into the cache. Given that the initial access_ok check preceding a sequence of __{get,put}_user operations will take the brunt of the miss, we can make the __* variants identical to the full-fat versions, which brings with it the benefits of address masking.
The likely cost in these sequences will be from toggling PAN/UAO, which we can address later by implementing the *_unsafe versions.
--- arch/arm64/include/asm/uaccess.h | 54 ++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 22 deletions(-)
arch/arm64/include/asm/uaccess.h +32 -22
@@ -306,28 +306,33 @@ do { \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ } while (0) -#define __get_user (x, ptr) \ +#define __get_user_check (x, ptr, err) \ ({ \ - int __gu_err = 0; \ - __get_user_err((x), (ptr), __gu_err); \ - __gu_err; \ + __typeof__(*(ptr)) __user *__p = (ptr); \ + might_fault(); \ + if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \ + __p = uaccess_mask_ptr(__p); \ + __get_user_err((x), __p, (err)); \ + } else { \ + (x) = 0; (err) = -EFAULT; \ + } \ })#define __get_user_error (x, ptr, err) \ ({ \ - __get_user_err((x), (ptr), (err)); \ + __get_user_check((x), (ptr), (err)); \ (void)0; \ }) -#define get_user (x, ptr) \ +#define __get_user (x, ptr) \ ({ \ - __typeof__(*(ptr)) __user *__p = (ptr); \ - might_fault(); \ - access_ok(VERIFY_READ, __p, sizeof(*__p)) ? \ - __p = uaccess_mask_ptr(__p), __get_user((x), __p) : \ - ((x) = 0, -EFAULT); \ + int __gu_err = 0; \ + __get_user_check((x), (ptr), __gu_err); \ + __gu_err; \ }) +#define get_user __get_user +#define __put_user_asm (instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
@@ -370,28 +375,33 @@ do { \uaccess_disable_not_uao (); \ } while (0) -#define __put_user (x, ptr) \ +#define __put_user_check (x, ptr, err) \ ({ \ - int __pu_err = 0; \ - __put_user_err((x), (ptr), __pu_err); \ - __pu_err; \ + __typeof__(*(ptr)) __user *__p = (ptr); \ + might_fault(); \ + if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \ + __p = uaccess_mask_ptr(__p); \ + __put_user_err((x), __p, (err)); \ + } else { \ + (err) = -EFAULT; \ + } \ })#define __put_user_error (x, ptr, err) \ ({ \ - __put_user_err((x), (ptr), (err)); \ + __put_user_check((x), (ptr), (err)); \ (void)0; \ }) -#define put_user (x, ptr) \ +#define __put_user (x, ptr) \ ({ \ - __typeof__(*(ptr)) __user *__p = (ptr); \ - might_fault(); \ - access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ? \ - __p = uaccess_mask_ptr(__p), __put_user((x), __p) : \ - -EFAULT; \ + int __pu_err = 0; \ + __put_user_check((x), (ptr), __pu_err); \ + __pu_err; \ }) +#define put_user __put_user +extern unsigned long __must_check __arch_copy_from_user (void *to,const void __user *from,unsigned long n);#define raw_copy_from_user __arch_copy_from_userextern unsigned long __must_check __arch_copy_to_user (void __user *to,const void *from,unsigned long n);
Signature · 1 line
2.1.4
PATCH 8/9 arm64: uaccess: Mask __user pointers for __arch_{clear, copy_*}_user
In reply to Will DeaconLike we've done for get_user and put_user, ensure that user pointers are masked before invoking the underlying __arch_{clear,copy_*}_user operations.
--- arch/arm64/include/asm/uaccess.h | 29 ++++++++++++++++++++++------- arch/arm64/kernel/arm64ksyms.c | 4 ++-- arch/arm64/lib/clear_user.S | 6 +++--- arch/arm64/lib/copy_in_user.S | 5 +++-- 4 files changed, 30 insertions(+), 14 deletions(-)
arch/arm64/include/asm/uaccess.h +22 -7
@@ -403,20 +403,35 @@ do { \#define put_user __put_userextern unsigned long __must_check __arch_copy_from_user (void *to,const void __user *from,unsigned long n); -#define raw_copy_from_user __arch_copy_from_user +#define raw_copy_from_user (to, from, n) \ +({ \ + __arch_copy_from_user((to), __uaccess_mask_ptr(from), (n)); \ +}) +extern unsigned long __must_check __arch_copy_to_user (void __user *to,const void *from,unsigned long n); -#define raw_copy_to_user __arch_copy_to_user -extern unsigned long __must_check raw_copy_in_user (void __user *to,const void __user *from,unsigned long n); -extern unsigned long __must_check __clear_user (void __user *addr,unsigned long n); +#define raw_copy_to_user (to, from, n) \ +({ \ + __arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n)); \ +}) + +extern unsigned long __must_check __arch_copy_in_user (void __user *to,const void __user *from,unsigned long n); +#define raw_copy_in_user (to, from, n) \ +({ \ + __arch_copy_in_user(__uaccess_mask_ptr(to), \ + __uaccess_mask_ptr(from), (n)); \ +}) +#define INLINE_COPY_TO_USER #define INLINE_COPY_FROM_USER -static inline unsigned long __must_check clear_user (void __user *to,unsigned long n) +extern unsigned long __must_check __arch_clear_user (void __user *to,unsigned long n); +static inline unsigned long __must_check __clear_user (void __user *to,unsigned long n) {if (access_ok (VERIFY_WRITE , to, n)) - n =__clear_user (__uaccess_mask_ptr (to), n); + n =__arch_clear_user (__uaccess_mask_ptr (to), n);return n; } +#define clear_user __clear_userextern long strncpy_from_user (char *dest,const char __user *src,long count);
@@ -430,7 +445,7 @@ extern unsigned long __must_check __copy_user_flushcache(void *to, const void __static inline int __copy_from_user_flushcache (void *dst,const void __user *src,unsigned size) {kasan_check_write (dst, size); -return __copy_user_flushcache (dst, src, size); +return __copy_user_flushcache (dst,__uaccess_mask_ptr (src), size); }#endif
arch/arm64/kernel/arm64ksyms.c +2 -2
@@ -37,8 +37,8 @@ EXPORT_SYMBOL(clear_page);/* user mem (segment) */ EXPORT_SYMBOL (__arch_copy_from_user);EXPORT_SYMBOL (__arch_copy_to_user); -EXPORT_SYMBOL (__clear_user); -EXPORT_SYMBOL (raw_copy_in_user); +EXPORT_SYMBOL (__arch_clear_user); +EXPORT_SYMBOL (__arch_copy_in_user);/* physical memory */ EXPORT_SYMBOL (memstart_addr);
arch/arm64/lib/clear_user.S +3 -3
@@ -21,7 +21,7 @@ .text -/* Prototype: int __clear_user(void *addr, size_t sz) +/* Prototype: int __arch_clear_user(void *addr, size_t sz) * Purpose : clear some user memory * Params : addr - user memory address to clear * : sz - number of bytes to clear
@@ -29,7 +29,7 @@ * * Alignment fixed up by hardware. */ -ENTRY(__clear_user) +ENTRY(__arch_clear_user) uaccess_enable_not_uao x2, x3, x4 mov x2, x1 // save the size for fixup return subs x1, x1, #8
@@ -52,7 +52,7 @@ uao_user_alternative 9f, strb, sttrb, wzr, x0, 0 5: mov x0, #0 uaccess_disable_not_uao x2, x3 ret -ENDPROC(__clear_user) +ENDPROC(__arch_clear_user) .section .fixup,"ax" .align 2
arch/arm64/lib/copy_in_user.S +3 -2
@@ -64,14 +64,15 @@ .endm end .req x5 -ENTRY(raw_copy_in_user) + +ENTRY(__arch_copy_in_user) uaccess_enable_not_uao x3, x4, x5 add end, x0, x2 #include "copy_template.S" uaccess_disable_not_uao x3, x4 mov x0, #0 ret -ENDPROC(raw_copy_in_user) +ENDPROC(__arch_copy_in_user) .section .fixup,"ax" .align 2
Signature · 1 line
2.1.4
PATCH 9/9 arm64: futex: Mask __user pointers prior to dereference
In reply to Will DeaconThe arm64 futex code has some explicit dereferencing of user pointers where performing atomic operations in response to a futex command. This patch uses masking to limit any speculative futex operations to within the user address space.
--- arch/arm64/include/asm/futex.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
arch/arm64/include/asm/futex.h +6 -3
@@ -48,9 +48,10 @@ do { \ }while (0 ) static inline int -arch_futex_atomic_op_inuser (int op,int oparg,int *oval,u32 __user *uaddr) +arch_futex_atomic_op_inuser (int op,int oparg,int *oval,u32 __user *_uaddr) {int oldval =0 , ret, tmp; +u32 __user *uaddr =__uaccess_mask_ptr (_uaddr);pagefault_disable ();
@@ -88,15 +89,17 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) }static inline int -futex_atomic_cmpxchg_inatomic (u32 *uval,u32 __user *uaddr, +futex_atomic_cmpxchg_inatomic (u32 *uval,u32 __user *_uaddr,u32 oldval,u32 newval) {int ret =0 ;u32 val, tmp; +u32 __user *uaddr; -if (!access_ok (VERIFY_WRITE , uaddr,sizeof (u32))) +if (!access_ok (VERIFY_WRITE , _uaddr,sizeof (u32)))return -EFAULT ; + uaddr =__uaccess_mask_ptr (_uaddr);uaccess_enable (); asmvolatile ("// futex_atomic_cmpxchg_inatomic\n" " prfm pstl1strm, %2\n"
Signature · 1 line
2.1.4
cover Mitigations against spectre-v1 in the arm64 Linux kernel
In reply to Will DeaconQuoted 9 lines · Will Deacon
Hi all,
This series implements some mitigations against spectre-v1 for arm64. It broadly follows the work that has been done for x86, by implementing:
* A back-end version of
array_index_mask_nospec()to suppress any compiler optimisations that could introduce unwanted speculative paths
Quoted 7 lines
* Masking of the syscall number to restrict speculation through the syscall table
* Masking of
__userpointers prior to deference in uaccess routinesThe latter introduces complications in
access_okandset_fswhich are also handled in this series.
Queued for 4.16. Thanks.
Signature · 1 line
Catalin