x86/uaccess: Avoid barrier_nospec()
v2 cover x86/uaccess: Avoid barrier_nospec()
At least for now, continue to assume mask_user_address() is safe on AMD when combined with STAC/CLAC -- as get_user(), put_user() and masked_user_access_begin() already do today.
v2:
- Separate copy_to_user() and clear_user() changes out into separate patches
- Add masking to __get_user() and __put_user()
v1:
https://lore.kernel.org/b626840e55d4aa86b4b9b377a4cc2cda7038d33d.1728706156.git.jpoimboe@kernel.org
Josh Poimboeuf (6):
x86/uaccess: Avoid barrier_nospec() in copy_from_user() x86/uaccess: Avoid barrier_nospec() in __get_user() x86/uaccess: Rearrange putuser.S
x86/uaccess: Add user pointer masking to __put_user() x86/uaccess: Add user pointer masking to copy_to_user() x86/uaccess: Add user pointer masking to clear_user()
arch/powerpc/include/asm/uaccess.h | 2 + arch/x86/include/asm/uaccess_64.h | 10 ++-- arch/x86/lib/getuser.S | 27 +++++++-- arch/x86/lib/putuser.S | 92 ++++++++++++++++++------------ include/linux/uaccess.h | 6 -- 5 files changed, 86 insertions(+), 51 deletions(-)
Signature · 1 line
2.47.0
PATCH v2 1/6 x86/uaccess: Avoid barrier_nospec() in copy_from_user()
For x86-64, the barrier_nospec() in copy_from_user() is overkill and painfully slow. Instead, use pointer masking to force the user pointer to a non-kernel value in speculative paths.
To avoid regressing powerpc, move the barrier_nospec() to the powerpc raw_copy_from_user() implementation so there's no functional change.
--- arch/powerpc/include/asm/uaccess.h | 2 ++ arch/x86/include/asm/uaccess_64.h | 7 ++++--- arch/x86/lib/getuser.S | 2 +- arch/x86/lib/putuser.S | 2 +- include/linux/uaccess.h | 6 ------ 5 files changed, 8 insertions(+), 11 deletions(-)
arch/powerpc/include/asm/uaccess.h +2 -0
@@ -7,6 +7,7 @@#include <asm/extable.h> #include <asm/kup.h> #include <asm/asm-compat.h> +#include <asm/barrier.h> #ifdef __powerpc64__/* We use TASK_SIZE_USER64 as TASK_SIZE is not constant */
@@ -341,6 +342,7 @@ static inline unsigned long raw_copy_from_user(void *to, {unsigned long ret; +barrier_nospec ();allow_read_from_user (from, n); ret =__copy_tofrom_user ((__force void __user *)to, from, n);prevent_read_from_user (from, n);
arch/x86/include/asm/uaccess_64.h +4 -3
@@ -54,11 +54,11 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,#define valid_user_address (x) ((__force long)(x) >= 0)/* -* Masking the user address is an alternative to a conditional -* user_access_begin that can avoid the fencing. This only works -* for dense accesses starting at the address. +* If it's a kernel address, force it to all 1's. This prevents a mispredicted +* access_ok() from speculatively accessing kernel space. */ #define mask_user_address (x) ((typeof(x))((long)(x)|((long)(x)>>63))) +#define masked_user_access_begin (x) ({ \ __auto_type __masked_ptr = (x); \ __masked_ptr = mask_user_address(__masked_ptr); \
@@ -133,6 +133,7 @@ copy_user_generic(void *to, const void *from, unsigned long len)static __always_inline __must_check unsigned long raw_copy_from_user (void *dst,const void __user *src,unsigned long size) { + src =mask_user_address (src);return copy_user_generic (dst, (__force void *)src, size); }
arch/x86/lib/getuser.S +1 -1
@@ -39,7 +39,7 @@ .macro check_range size:req .if IS_ENABLED(CONFIG_X86_64) - mov %rax, %rdx + mov %rax, %rdx /* mask_user_address() */ sar $63, %rdx or %rdx, %rax .else
arch/x86/lib/putuser.S +1 -1
@@ -34,7 +34,7 @@ .macro check_range size:req .if IS_ENABLED(CONFIG_X86_64) - mov %rcx, %rbx + mov %rcx, %rbx /* mask_user_address() */ sar $63, %rbx or %rbx, %rcx .else
include/linux/uaccess.h +0 -6
@@ -160,12 +160,6 @@ _inline_copy_from_user(void *to, const void __user *from, unsigned long n)unsigned long res = n;might_fault ();if (!should_fail_usercopy () &&likely (access_ok (from, n))) { -/* -* Ensure that bad access_ok() speculation will not -* lead to nasty side effects *after* the copy is -* finished: -*/ -barrier_nospec ();instrument_copy_from_user_before (to, from, n); res =raw_copy_from_user (to, from, n);instrument_copy_from_user_after (to, from, n, res);
Signature · 1 line
2.47.0
PATCH v2 2/6 x86/uaccess: Avoid barrier_nospec() in __get_user()
In reply to Josh PoimboeufOn 64-bit, the barrier_nospec() in __get_user() is overkill and painfully slow. Instead, use pointer masking to force the user pointer to a non-kernel value in speculative paths.
Doing so makes get_user() and __get_user() identical in behavior, so converge their implementations.
--- arch/x86/lib/getuser.S | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-)
arch/x86/lib/getuser.S +21 -4
@@ -105,6 +105,26 @@ SYM_FUNC_START(__get_user_8) SYM_FUNC_END(__get_user_8) EXPORT_SYMBOL(__get_user_8) +#ifdef CONFIG_X86_64 + +/* + * On x86-64, get_user() does address masking rather than a conditional + * bounds check so there's no functional difference with __get_user(). + */ +SYM_FUNC_ALIAS(__get_user_nocheck_1, __get_user_1); +EXPORT_SYMBOL(__get_user_nocheck_1); + +SYM_FUNC_ALIAS(__get_user_nocheck_2, __get_user_2); +EXPORT_SYMBOL(__get_user_nocheck_2); + +SYM_FUNC_ALIAS(__get_user_nocheck_4, __get_user_4); +EXPORT_SYMBOL(__get_user_nocheck_4); + +SYM_FUNC_ALIAS(__get_user_nocheck_8, __get_user_8); +EXPORT_SYMBOL(__get_user_nocheck_8); + +#else /* CONFIG_X86_32 */ + /* .. and the same for __get_user, just without the range checks */ SYM_FUNC_START(__get_user_nocheck_1) ASM_STAC
@@ -139,19 +159,16 @@ EXPORT_SYMBOL(__get_user_nocheck_4) SYM_FUNC_START(__get_user_nocheck_8) ASM_STAC ASM_BARRIER_NOSPEC -#ifdef CONFIG_X86_64 - UACCESS movq (%_ASM_AX),%rdx -#else xor %ecx,%ecx UACCESS movl (%_ASM_AX),%edx UACCESS movl 4(%_ASM_AX),%ecx -#endif xor %eax,%eax ASM_CLAC RET SYM_FUNC_END(__get_user_nocheck_8) EXPORT_SYMBOL(__get_user_nocheck_8) +#endif /* CONFIG_X86_32 */ SYM_CODE_START_LOCAL(__get_user_handle_exception) ASM_CLAC
Signature · 1 line
2.47.0
PATCH v2 3/6 x86/uaccess: Rearrange putuser.S
In reply to Josh PoimboeufSeparate __put_user_*() from __put_user_nocheck_*() to make the layout similar to getuser.S. This will also make it easier to do a subsequent change. No functional changes.
--- arch/x86/lib/putuser.S | 67 ++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 32 deletions(-)
arch/x86/lib/putuser.S +35 -32
@@ -54,59 +54,32 @@ SYM_FUNC_START(__put_user_1) SYM_FUNC_END(__put_user_1) EXPORT_SYMBOL(__put_user_1) -SYM_FUNC_START(__put_user_nocheck_1) - ASM_STAC -2: movb %al,(%_ASM_CX) - xor %ecx,%ecx - ASM_CLAC - RET -SYM_FUNC_END(__put_user_nocheck_1) -EXPORT_SYMBOL(__put_user_nocheck_1) - SYM_FUNC_START(__put_user_2) check_range size=2 ASM_STAC -3: movw %ax,(%_ASM_CX) +2: movw %ax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC RET SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2) -SYM_FUNC_START(__put_user_nocheck_2) - ASM_STAC -4: movw %ax,(%_ASM_CX) - xor %ecx,%ecx - ASM_CLAC - RET -SYM_FUNC_END(__put_user_nocheck_2) -EXPORT_SYMBOL(__put_user_nocheck_2) - SYM_FUNC_START(__put_user_4) check_range size=4 ASM_STAC -5: movl %eax,(%_ASM_CX) +3: movl %eax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC RET SYM_FUNC_END(__put_user_4) EXPORT_SYMBOL(__put_user_4) -SYM_FUNC_START(__put_user_nocheck_4) - ASM_STAC -6: movl %eax,(%_ASM_CX) - xor %ecx,%ecx - ASM_CLAC - RET -SYM_FUNC_END(__put_user_nocheck_4) -EXPORT_SYMBOL(__put_user_nocheck_4) - SYM_FUNC_START(__put_user_8) check_range size=8 ASM_STAC -7: mov %_ASM_AX,(%_ASM_CX) +4: mov %_ASM_AX,(%_ASM_CX) #ifdef CONFIG_X86_32 -8: movl %edx,4(%_ASM_CX) +5: movl %edx,4(%_ASM_CX) #endif xor %ecx,%ecx ASM_CLAC
@@ -114,6 +87,34 @@ SYM_FUNC_START(__put_user_8) SYM_FUNC_END(__put_user_8) EXPORT_SYMBOL(__put_user_8) +/* .. and the same for __put_user, just without the range checks */ +SYM_FUNC_START(__put_user_nocheck_1) + ASM_STAC +6: movb %al,(%_ASM_CX) + xor %ecx,%ecx + ASM_CLAC + RET +SYM_FUNC_END(__put_user_nocheck_1) +EXPORT_SYMBOL(__put_user_nocheck_1) + +SYM_FUNC_START(__put_user_nocheck_2) + ASM_STAC +7: movw %ax,(%_ASM_CX) + xor %ecx,%ecx + ASM_CLAC + RET +SYM_FUNC_END(__put_user_nocheck_2) +EXPORT_SYMBOL(__put_user_nocheck_2) + +SYM_FUNC_START(__put_user_nocheck_4) + ASM_STAC +8: movl %eax,(%_ASM_CX) + xor %ecx,%ecx + ASM_CLAC + RET +SYM_FUNC_END(__put_user_nocheck_4) +EXPORT_SYMBOL(__put_user_nocheck_4) + SYM_FUNC_START(__put_user_nocheck_8) ASM_STAC 9: mov %_ASM_AX,(%_ASM_CX)
@@ -137,11 +138,13 @@ SYM_CODE_END(__put_user_handle_exception) _ASM_EXTABLE_UA(2b, __put_user_handle_exception) _ASM_EXTABLE_UA(3b, __put_user_handle_exception) _ASM_EXTABLE_UA(4b, __put_user_handle_exception) +#ifdef CONFIG_X86_32 _ASM_EXTABLE_UA(5b, __put_user_handle_exception) +#endif _ASM_EXTABLE_UA(6b, __put_user_handle_exception) _ASM_EXTABLE_UA(7b, __put_user_handle_exception) + _ASM_EXTABLE_UA(8b, __put_user_handle_exception) _ASM_EXTABLE_UA(9b, __put_user_handle_exception) #ifdef CONFIG_X86_32 - _ASM_EXTABLE_UA(8b, __put_user_handle_exception) _ASM_EXTABLE_UA(10b, __put_user_handle_exception) #endif
Signature · 1 line
2.47.0
Re: [PATCH v2 3/6] x86/uaccess: Rearrange putuser.S
arch/x86/lib/putuser.S SYM_FUNC_START(__put_user_1) · line 65
Quoted 10 lines · Josh Poimboeuf
SYM_FUNC_START(__put_user_2)arch/x86/lib/putuser.S +1
-1check_range size=2 ASM_STAC -3: movw %ax,(%_ASM_CX) +2: movw %ax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC RET SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2)
This patch provides an opportunity to give these labels more meaningful names, so that future rearrangements do not require as much boilerplate.
For example, we can rename this label 2: to .Luser_2 or something similar.
Signature · 1 line
Kiryl Shutsemau / Kirill A. Shutemov
Re: [PATCH v2 3/6] x86/uaccess: Rearrange putuser.S
Quoted 12 lines · Kirill A . Shutemov
arch/x86/lib/putuser.S SYM_FUNC_START(__put_user_1) · line 65
Quoted 10 lines · Josh Poimboeuf
SYM_FUNC_START(__put_user_2)arch/x86/lib/putuser.S +1
-1check_range size=2 ASM_STAC -3: movw %ax,(%_ASM_CX) +2: movw %ax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC RET SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2)
Quoted 2 lines
This patch provides an opportunity to give these labels more meaningful names, so that future rearrangements do not require as much boilerplate.
Yeah, I can add a patch like Linus' patch to getuser.S which encapsulates it all in a macro:
.macro UACCESS op src dst
1: \op \src,\dst
_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
.endm
.textSYM_FUNC_START(__get_user_1)
check_range size=1
ASM_STAC
UACCESS movzbl (%_ASM_AX),%edx
xor %eax,%eax
ASM_CLAC
RETSYM_FUNC_END(__get_user_1)EXPORT_SYMBOL(__get_user_1)
Signature · 1 line
Josh
PATCH v2 4/6 x86/uaccess: Add user pointer masking to __put_user()
In reply to Josh PoimboeufAdd user pointer masking to __put_user() to mitigate Spectre v1.
A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.
This makes its behavior identical to put_user(), so converge their implementations.
--- arch/x86/lib/putuser.S | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-)
arch/x86/lib/putuser.S +22 -5
@@ -87,7 +87,26 @@ SYM_FUNC_START(__put_user_8) SYM_FUNC_END(__put_user_8) EXPORT_SYMBOL(__put_user_8) -/* .. and the same for __put_user, just without the range checks */ +#ifdef CONFIG_X86_64 + +/* + * On x86-64, put_user() does address masking rather than a conditional + * bounds check so there's no functional difference with __put_user(). + */ +SYM_FUNC_ALIAS(__put_user_nocheck_1, __put_user_1); +EXPORT_SYMBOL(__put_user_nocheck_1); + +SYM_FUNC_ALIAS(__put_user_nocheck_2, __put_user_2); +EXPORT_SYMBOL(__put_user_nocheck_2); + +SYM_FUNC_ALIAS(__put_user_nocheck_4, __put_user_4); +EXPORT_SYMBOL(__put_user_nocheck_4); + +SYM_FUNC_ALIAS(__put_user_nocheck_8, __put_user_8); +EXPORT_SYMBOL(__put_user_nocheck_8); + +#else /* CONFIG_X86_32 */ + SYM_FUNC_START(__put_user_nocheck_1) ASM_STAC 6: movb %al,(%_ASM_CX)
@@ -118,15 +137,15 @@ EXPORT_SYMBOL(__put_user_nocheck_4) SYM_FUNC_START(__put_user_nocheck_8) ASM_STAC 9: mov %_ASM_AX,(%_ASM_CX) -#ifdef CONFIG_X86_32 10: movl %edx,4(%_ASM_CX) -#endif xor %ecx,%ecx ASM_CLAC RET SYM_FUNC_END(__put_user_nocheck_8) EXPORT_SYMBOL(__put_user_nocheck_8) +#endif /* CONFIG_X86_32 */ + SYM_CODE_START_LOCAL(__put_user_handle_exception) ASM_CLAC .Lbad_put_user:
@@ -140,11 +159,9 @@ SYM_CODE_END(__put_user_handle_exception) _ASM_EXTABLE_UA(4b, __put_user_handle_exception) #ifdef CONFIG_X86_32 _ASM_EXTABLE_UA(5b, __put_user_handle_exception) -#endif _ASM_EXTABLE_UA(6b, __put_user_handle_exception) _ASM_EXTABLE_UA(7b, __put_user_handle_exception) _ASM_EXTABLE_UA(8b, __put_user_handle_exception) _ASM_EXTABLE_UA(9b, __put_user_handle_exception) -#ifdef CONFIG_X86_32 _ASM_EXTABLE_UA(10b, __put_user_handle_exception) #endif
Signature · 1 line
2.47.0
PATCH v2 5/6 x86/uaccess: Add user pointer masking to copy_to_user()
In reply to Josh PoimboeufAdd user pointer masking to copy_to_user() to mitigate Spectre v1.
A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.
--- arch/x86/include/asm/uaccess_64.h | 1 + 1 file changed, 1 insertion(+)
arch/x86/include/asm/uaccess_64.h +1 -0
@@ -140,6 +140,7 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size)static __always_inline __must_check unsigned long raw_copy_to_user (void __user *dst,const void *src,unsigned long size) { + dst =mask_user_address (dst);return copy_user_generic ((__force void *)dst, src, size); }
Signature · 1 line
2.47.0
PATCH v2 6/6 x86/uaccess: Add user pointer masking to clear_user()
In reply to Josh PoimboeufAdd user pointer masking to clear_user() to mitigate Spectre v1.
A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.
--- arch/x86/include/asm/uaccess_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/include/asm/uaccess_64.h +1 -1
@@ -199,7 +199,7 @@ static __always_inline __must_check unsigned long __clear_user(void __user *addrstatic __always_inline unsigned long clear_user (void __user *to,unsigned long n) {if (__access_ok (to, n)) -return __clear_user (to, n); +return __clear_user (mask_user_address (to), n);return n; }#endif /* _ASM_X86_UACCESS_64_H */
Signature · 1 line
2.47.0
Quoted 3 lines · Josh Poimboeuf
At least for now, continue to assume
mask_user_address()is safe on AMD when combined with STAC/CLAC -- asget_user(),put_user()andmasked_user_access_begin()already do today.
Honestly, I find this a very worrying position to take.
It's one thing not to know there's a speculative security vulnerability with how mask_user_address() is used.
It's totally another to say "lets pretend that it doesn't exist so we can continue to make things faster".
Even if you can get Intel and AMD to agree that STAC/CLAC are really LFENCEs (and I think you'll struggle), they'd only confer the safety you want between a real conditional that excludes the non-canonical range, and the pointer deference.
Any path that genuinely deferences a non-canonical pointer is not safe, whatever serialisation you put in the way. The attacker wins the moment the load uop executes.
The final hunk of patch 1 is safe (iff STAC is given extra guarantees) because it is between the conditional and the deference. Patch 4 is not safe (if the comment is correct) because it removes the conditional.
Or state that you intend to disregard this non-canoncal speculation problem; that's fine(ish) too, as long as it's done transparently.
~Andrew
Quoted 5 lines · Andrew Cooper
Even if you can get Intel and AMD to agree that STAC/CLAC are really LFENCEs (and I think you'll struggle), they'd only confer the safety you want between a real conditional that excludes the non-canonical range, and the pointer deference.
Quoted 7 lines
Any path that genuinely deferences a non-canonical pointer is not safe, whatever serialisation you put in the way. The attacker wins the moment the load uop executes.
The final hunk of patch 1 is safe (iff STAC is given extra guarantees) because it is between the conditional and the deference. Patch 4 is not safe (if the comment is correct) because it removes the conditional.
So the naming is confusing:
- put_user() implementation is __put_user_*()
- __put_user() implementation is __put_user_nocheck_*()
Patch 4 only affects __put_user(), for which the user is expected to call access_ok() beforehand.
The current implementations of get_user(), put_user() and masked_user_access_begin() avoid the conditional. Those are the ones it sounds like you're worried about?
None of my patches remove conditional checks.
Signature · 1 line
Josh