Skip to contentSkip to conversation

x86/uaccess: Avoid barrier_nospec()

PATCH Superseded Josh Poimboeuf 3 participants 11 messages Oct 17, 2024 – Oct 18, 2024

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover x86/uaccess: Avoid barrier_nospec()

At least for now, continue to assume mask_user_address() is safe on AMD when combined with STAC/CLAC -- as get_user(), put_user() and masked_user_access_begin() already do today.

v2:
- Separate copy_to_user() and clear_user() changes out into separate patches
- Add masking to __get_user() and __put_user()

v1:
https://lore.kernel.org/b626840e55d4aa86b4b9b377a4cc2cda7038d33d.1728706156.git.jpoimboe@kernel.org

Josh Poimboeuf (6):
x86/uaccess: Avoid barrier_nospec() in copy_from_user() x86/uaccess: Avoid barrier_nospec() in __get_user() x86/uaccess: Rearrange putuser.S
x86/uaccess: Add user pointer masking to __put_user() x86/uaccess: Add user pointer masking to copy_to_user() x86/uaccess: Add user pointer masking to clear_user()

 arch/powerpc/include/asm/uaccess.h |  2 +
 arch/x86/include/asm/uaccess_64.h  | 10 ++--
 arch/x86/lib/getuser.S             | 27 +++++++--
 arch/x86/lib/putuser.S             | 92 ++++++++++++++++++------------
 include/linux/uaccess.h            |  6 --
 5 files changed, 86 insertions(+), 51 deletions(-)
Signature · 1 line
2.47.0
#1 lore raw

PATCH v2 1/6 x86/uaccess: Avoid barrier_nospec() in copy_from_user()

For x86-64, the barrier_nospec() in copy_from_user() is overkill and painfully slow. Instead, use pointer masking to force the user pointer to a non-kernel value in speculative paths.

To avoid regressing powerpc, move the barrier_nospec() to the powerpc raw_copy_from_user() implementation so there's no functional change.

---
 arch/powerpc/include/asm/uaccess.h | 2 ++
 arch/x86/include/asm/uaccess_64.h  | 7 ++++---
 arch/x86/lib/getuser.S             | 2 +-
 arch/x86/lib/putuser.S             | 2 +-
 include/linux/uaccess.h            | 6 ------
 5 files changed, 8 insertions(+), 11 deletions(-)
arch/powerpc/include/asm/uaccess.h +2 -0
index 4f5a46a77fa2..12abb8bf5eda 100644
@@ -7,6 +7,7 @@
 #include <asm/extable.h>
 #include <asm/kup.h>
 #include <asm/asm-compat.h>
+#include <asm/barrier.h>
 
 #ifdef __powerpc64__
 /* We use TASK_SIZE_USER64 as TASK_SIZE is not constant */
@@ -341,6 +342,7 @@ static inline unsigned long raw_copy_from_user(void *to,
 {
 	unsigned long ret;
 
+	barrier_nospec();
 	allow_read_from_user(from, n);
 	ret = __copy_tofrom_user((__force void __user *)to, from, n);
 	prevent_read_from_user(from, n);
arch/x86/include/asm/uaccess_64.h +4 -3
index afce8ee5d7b7..61693028ea2b 100644
@@ -54,11 +54,11 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
 #define valid_user_address(x) ((__force long)(x) >= 0)
 
 /*
- * Masking the user address is an alternative to a conditional
- * user_access_begin that can avoid the fencing. This only works
- * for dense accesses starting at the address.
+ * If it's a kernel address, force it to all 1's.  This prevents a mispredicted
+ * access_ok() from speculatively accessing kernel space.
  */
 #define mask_user_address(x) ((typeof(x))((long)(x)|((long)(x)>>63)))
+
 #define masked_user_access_begin(x) ({				\
 	__auto_type __masked_ptr = (x);				\
 	__masked_ptr = mask_user_address(__masked_ptr);		\
@@ -133,6 +133,7 @@ copy_user_generic(void *to, const void *from, unsigned long len)
 static __always_inline __must_check unsigned long
 raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 {
+	src = mask_user_address(src);
 	return copy_user_generic(dst, (__force void *)src, size);
 }
 
arch/x86/lib/getuser.S +1 -1
index d066aecf8aeb..094224ec9dca 100644
@@ -39,7 +39,7 @@
 
 .macro check_range size:req
 .if IS_ENABLED(CONFIG_X86_64)
-	mov %rax, %rdx
+	mov %rax, %rdx		/* mask_user_address() */
 	sar $63, %rdx
 	or %rdx, %rax
 .else
arch/x86/lib/putuser.S +1 -1
index 975c9c18263d..09b7e37934ab 100644
@@ -34,7 +34,7 @@
 
 .macro check_range size:req
 .if IS_ENABLED(CONFIG_X86_64)
-	mov %rcx, %rbx
+	mov %rcx, %rbx		/* mask_user_address() */
 	sar $63, %rbx
 	or %rbx, %rcx
 .else
include/linux/uaccess.h +0 -6
index 39c7cf82b0c2..dda9725a9559 100644
@@ -160,12 +160,6 @@ _inline_copy_from_user(void *to, const void __user *from, unsigned long n)
 	unsigned long res = n;
 	might_fault();
 	if (!should_fail_usercopy() && likely(access_ok(from, n))) {
-		/*
-		 * Ensure that bad access_ok() speculation will not
-		 * lead to nasty side effects *after* the copy is
-		 * finished:
-		 */
-		barrier_nospec();
 		instrument_copy_from_user_before(to, from, n);
 		res = raw_copy_from_user(to, from, n);
 		instrument_copy_from_user_after(to, from, n, res);
Signature · 1 line
2.47.0
#2 lore raw

PATCH v2 2/6 x86/uaccess: Avoid barrier_nospec() in __get_user()

In reply to Josh Poimboeuf

On 64-bit, the barrier_nospec() in __get_user() is overkill and painfully slow. Instead, use pointer masking to force the user pointer to a non-kernel value in speculative paths.

Doing so makes get_user() and __get_user() identical in behavior, so converge their implementations.

---
 arch/x86/lib/getuser.S | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)
arch/x86/lib/getuser.S +21 -4
index 094224ec9dca..7c9bf8f0b3ac 100644
@@ -105,6 +105,26 @@ SYM_FUNC_START(__get_user_8)
 SYM_FUNC_END(__get_user_8)
 EXPORT_SYMBOL(__get_user_8)
 
+#ifdef CONFIG_X86_64
+
+/*
+ * On x86-64, get_user() does address masking rather than a conditional
+ * bounds check so there's no functional difference with __get_user().
+ */
+SYM_FUNC_ALIAS(__get_user_nocheck_1, __get_user_1);
+EXPORT_SYMBOL(__get_user_nocheck_1);
+
+SYM_FUNC_ALIAS(__get_user_nocheck_2, __get_user_2);
+EXPORT_SYMBOL(__get_user_nocheck_2);
+
+SYM_FUNC_ALIAS(__get_user_nocheck_4, __get_user_4);
+EXPORT_SYMBOL(__get_user_nocheck_4);
+
+SYM_FUNC_ALIAS(__get_user_nocheck_8, __get_user_8);
+EXPORT_SYMBOL(__get_user_nocheck_8);
+
+#else /* CONFIG_X86_32 */
+
 /* .. and the same for __get_user, just without the range checks */
 SYM_FUNC_START(__get_user_nocheck_1)
 	ASM_STAC
@@ -139,19 +159,16 @@ EXPORT_SYMBOL(__get_user_nocheck_4)
 SYM_FUNC_START(__get_user_nocheck_8)
 	ASM_STAC
 	ASM_BARRIER_NOSPEC
-#ifdef CONFIG_X86_64
-	UACCESS movq (%_ASM_AX),%rdx
-#else
 	xor %ecx,%ecx
 	UACCESS movl (%_ASM_AX),%edx
 	UACCESS movl 4(%_ASM_AX),%ecx
-#endif
 	xor %eax,%eax
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__get_user_nocheck_8)
 EXPORT_SYMBOL(__get_user_nocheck_8)
 
+#endif /* CONFIG_X86_32 */
 
 SYM_CODE_START_LOCAL(__get_user_handle_exception)
 	ASM_CLAC
Signature · 1 line
2.47.0
#3 lore raw

PATCH v2 3/6 x86/uaccess: Rearrange putuser.S

In reply to Josh Poimboeuf

Separate __put_user_*() from __put_user_nocheck_*() to make the layout similar to getuser.S. This will also make it easier to do a subsequent change. No functional changes.

---
 arch/x86/lib/putuser.S | 67 ++++++++++++++++++++++--------------------
 1 file changed, 35 insertions(+), 32 deletions(-)
arch/x86/lib/putuser.S +35 -32
index 09b7e37934ab..cb137e0286be 100644
@@ -54,59 +54,32 @@ SYM_FUNC_START(__put_user_1)
 SYM_FUNC_END(__put_user_1)
 EXPORT_SYMBOL(__put_user_1)
 
-SYM_FUNC_START(__put_user_nocheck_1)
-	ASM_STAC
-2:	movb %al,(%_ASM_CX)
-	xor %ecx,%ecx
-	ASM_CLAC
-	RET
-SYM_FUNC_END(__put_user_nocheck_1)
-EXPORT_SYMBOL(__put_user_nocheck_1)
-
 SYM_FUNC_START(__put_user_2)
 	check_range size=2
 	ASM_STAC
-3:	movw %ax,(%_ASM_CX)
+2:	movw %ax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
 
-SYM_FUNC_START(__put_user_nocheck_2)
-	ASM_STAC
-4:	movw %ax,(%_ASM_CX)
-	xor %ecx,%ecx
-	ASM_CLAC
-	RET
-SYM_FUNC_END(__put_user_nocheck_2)
-EXPORT_SYMBOL(__put_user_nocheck_2)
-
 SYM_FUNC_START(__put_user_4)
 	check_range size=4
 	ASM_STAC
-5:	movl %eax,(%_ASM_CX)
+3:	movl %eax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_4)
 EXPORT_SYMBOL(__put_user_4)
 
-SYM_FUNC_START(__put_user_nocheck_4)
-	ASM_STAC
-6:	movl %eax,(%_ASM_CX)
-	xor %ecx,%ecx
-	ASM_CLAC
-	RET
-SYM_FUNC_END(__put_user_nocheck_4)
-EXPORT_SYMBOL(__put_user_nocheck_4)
-
 SYM_FUNC_START(__put_user_8)
 	check_range size=8
 	ASM_STAC
-7:	mov %_ASM_AX,(%_ASM_CX)
+4:	mov %_ASM_AX,(%_ASM_CX)
 #ifdef CONFIG_X86_32
-8:	movl %edx,4(%_ASM_CX)
+5:	movl %edx,4(%_ASM_CX)
 #endif
 	xor %ecx,%ecx
 	ASM_CLAC
@@ -114,6 +87,34 @@ SYM_FUNC_START(__put_user_8)
 SYM_FUNC_END(__put_user_8)
 EXPORT_SYMBOL(__put_user_8)
 
+/* .. and the same for __put_user, just without the range checks */
+SYM_FUNC_START(__put_user_nocheck_1)
+	ASM_STAC
+6:	movb %al,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_1)
+EXPORT_SYMBOL(__put_user_nocheck_1)
+
+SYM_FUNC_START(__put_user_nocheck_2)
+	ASM_STAC
+7:	movw %ax,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_2)
+EXPORT_SYMBOL(__put_user_nocheck_2)
+
+SYM_FUNC_START(__put_user_nocheck_4)
+	ASM_STAC
+8:	movl %eax,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_4)
+EXPORT_SYMBOL(__put_user_nocheck_4)
+
 SYM_FUNC_START(__put_user_nocheck_8)
 	ASM_STAC
 9:	mov %_ASM_AX,(%_ASM_CX)
@@ -137,11 +138,13 @@ SYM_CODE_END(__put_user_handle_exception)
 	_ASM_EXTABLE_UA(2b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(3b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(4b, __put_user_handle_exception)
+#ifdef CONFIG_X86_32
 	_ASM_EXTABLE_UA(5b, __put_user_handle_exception)
+#endif
 	_ASM_EXTABLE_UA(6b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(7b, __put_user_handle_exception)
+	_ASM_EXTABLE_UA(8b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(9b, __put_user_handle_exception)
 #ifdef CONFIG_X86_32
-	_ASM_EXTABLE_UA(8b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(10b, __put_user_handle_exception)
 #endif
Signature · 1 line
2.47.0
#6 lore raw

PATCH v2 4/6 x86/uaccess: Add user pointer masking to __put_user()

In reply to Josh Poimboeuf

Add user pointer masking to __put_user() to mitigate Spectre v1.

A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.

This makes its behavior identical to put_user(), so converge their implementations.

---
 arch/x86/lib/putuser.S | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)
arch/x86/lib/putuser.S +22 -5
index cb137e0286be..1b122261b7aa 100644
@@ -87,7 +87,26 @@ SYM_FUNC_START(__put_user_8)
 SYM_FUNC_END(__put_user_8)
 EXPORT_SYMBOL(__put_user_8)
 
-/* .. and the same for __put_user, just without the range checks */
+#ifdef CONFIG_X86_64
+
+/*
+ * On x86-64, put_user() does address masking rather than a conditional
+ * bounds check so there's no functional difference with __put_user().
+ */
+SYM_FUNC_ALIAS(__put_user_nocheck_1, __put_user_1);
+EXPORT_SYMBOL(__put_user_nocheck_1);
+
+SYM_FUNC_ALIAS(__put_user_nocheck_2, __put_user_2);
+EXPORT_SYMBOL(__put_user_nocheck_2);
+
+SYM_FUNC_ALIAS(__put_user_nocheck_4, __put_user_4);
+EXPORT_SYMBOL(__put_user_nocheck_4);
+
+SYM_FUNC_ALIAS(__put_user_nocheck_8, __put_user_8);
+EXPORT_SYMBOL(__put_user_nocheck_8);
+
+#else /* CONFIG_X86_32 */
+
 SYM_FUNC_START(__put_user_nocheck_1)
 	ASM_STAC
 6:	movb %al,(%_ASM_CX)
@@ -118,15 +137,15 @@ EXPORT_SYMBOL(__put_user_nocheck_4)
 SYM_FUNC_START(__put_user_nocheck_8)
 	ASM_STAC
 9:	mov %_ASM_AX,(%_ASM_CX)
-#ifdef CONFIG_X86_32
 10:	movl %edx,4(%_ASM_CX)
-#endif
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_nocheck_8)
 EXPORT_SYMBOL(__put_user_nocheck_8)
 
+#endif /* CONFIG_X86_32 */
+
 SYM_CODE_START_LOCAL(__put_user_handle_exception)
 	ASM_CLAC
 .Lbad_put_user:
@@ -140,11 +159,9 @@ SYM_CODE_END(__put_user_handle_exception)
 	_ASM_EXTABLE_UA(4b, __put_user_handle_exception)
 #ifdef CONFIG_X86_32
 	_ASM_EXTABLE_UA(5b, __put_user_handle_exception)
-#endif
 	_ASM_EXTABLE_UA(6b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(7b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(8b, __put_user_handle_exception)
 	_ASM_EXTABLE_UA(9b, __put_user_handle_exception)
-#ifdef CONFIG_X86_32
 	_ASM_EXTABLE_UA(10b, __put_user_handle_exception)
 #endif
Signature · 1 line
2.47.0
#7 lore raw

PATCH v2 5/6 x86/uaccess: Add user pointer masking to copy_to_user()

In reply to Josh Poimboeuf

Add user pointer masking to copy_to_user() to mitigate Spectre v1.

A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.

---
 arch/x86/include/asm/uaccess_64.h | 1 +
 1 file changed, 1 insertion(+)
arch/x86/include/asm/uaccess_64.h +1 -0
index 61693028ea2b..0587830a47e1 100644
@@ -140,6 +140,7 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 static __always_inline __must_check unsigned long
 raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
 {
+	dst = mask_user_address(dst);
 	return copy_user_generic((__force void *)dst, src, size);
 }
 
Signature · 1 line
2.47.0
#8 lore raw

PATCH v2 6/6 x86/uaccess: Add user pointer masking to clear_user()

In reply to Josh Poimboeuf

Add user pointer masking to clear_user() to mitigate Spectre v1.

A write in a mispredicted access_ok() branch to a user-controlled kernel address can populate the rest of the affected cache line with kernel data.

---
 arch/x86/include/asm/uaccess_64.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/include/asm/uaccess_64.h +1 -1
index 0587830a47e1..8027db7f68c2 100644
@@ -199,7 +199,7 @@ static __always_inline __must_check unsigned long __clear_user(void __user *addr
 static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
 {
 	if (__access_ok(to, n))
-		return __clear_user(to, n);
+		return __clear_user(mask_user_address(to), n);
 	return n;
 }
 #endif /* _ASM_X86_UACCESS_64_H */
Signature · 1 line
2.47.0
#9 lore raw
In reply to Josh Poimboeuf
Quoted 3 lines · Josh Poimboeuf

At least for now, continue to assume mask_user_address() is safe on AMD when combined with STAC/CLAC -- as get_user(), put_user() and masked_user_access_begin() already do today.

Honestly, I find this a very worrying position to take.

It's one thing not to know there's a speculative security vulnerability with how mask_user_address() is used.

It's totally another to say "lets pretend that it doesn't exist so we can continue to make things faster".

Even if you can get Intel and AMD to agree that STAC/CLAC are really LFENCEs (and I think you'll struggle), they'd only confer the safety you want between a real conditional that excludes the non-canonical range, and the pointer deference.

Any path that genuinely deferences a non-canonical pointer is not safe, whatever serialisation you put in the way.  The attacker wins the moment the load uop executes.

The final hunk of patch 1 is safe (iff STAC is given extra guarantees) because it is between the conditional and the deference.  Patch 4 is not safe (if the comment is correct) because it removes the conditional.

Or state that you intend to disregard this non-canoncal speculation problem;  that's fine(ish) too, as long as it's done transparently.

~Andrew

#10 lore raw
Quoted 5 lines · Andrew Cooper

Even if you can get Intel and AMD to agree that STAC/CLAC are really LFENCEs (and I think you'll struggle), they'd only confer the safety you want between a real conditional that excludes the non-canonical range, and the pointer deference.

Quoted 7 lines

Any path that genuinely deferences a non-canonical pointer is not safe, whatever serialisation you put in the way.  The attacker wins the moment the load uop executes.

The final hunk of patch 1 is safe (iff STAC is given extra guarantees) because it is between the conditional and the deference.  Patch 4 is not safe (if the comment is correct) because it removes the conditional.

So the naming is confusing:

- put_user() implementation is __put_user_*()
- __put_user() implementation is __put_user_nocheck_*()

Patch 4 only affects __put_user(), for which the user is expected to call access_ok() beforehand.

The current implementations of get_user(), put_user() and masked_user_access_begin() avoid the conditional. Those are the ones it sounds like you're worried about?

None of my patches remove conditional checks.

Signature · 1 line
Josh
#4 lore raw

Re: [PATCH v2 3/6] x86/uaccess: Rearrange putuser.S

In reply to Josh Poimboeuf

arch/x86/lib/putuser.S SYM_FUNC_START(__put_user_1) · line 65

Quoted 10 lines · Josh Poimboeuf

SYM_FUNC_START(__put_user_2)

arch/x86/lib/putuser.S +1 -1
 	check_range size=2
 	ASM_STAC
-3:	movw %ax,(%_ASM_CX)
+2:	movw %ax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)

This patch provides an opportunity to give these labels more meaningful names, so that future rearrangements do not require as much boilerplate.

For example, we can rename this label 2: to .Luser_2 or something similar.

Signature · 1 line
  Kiryl Shutsemau / Kirill A. Shutemov
#5 lore raw

Re: [PATCH v2 3/6] x86/uaccess: Rearrange putuser.S

Quoted 12 lines · Kirill A . Shutemov

arch/x86/lib/putuser.S SYM_FUNC_START(__put_user_1) · line 65

Quoted 10 lines · Josh Poimboeuf

SYM_FUNC_START(__put_user_2)

arch/x86/lib/putuser.S +1 -1
 	check_range size=2
 	ASM_STAC
-3:	movw %ax,(%_ASM_CX)
+2:	movw %ax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
Quoted 2 lines

This patch provides an opportunity to give these labels more meaningful names, so that future rearrangements do not require as much boilerplate.

Yeah, I can add a patch like Linus' patch to getuser.S which encapsulates it all in a macro:

.macro UACCESS op src dst
1: \op \src,\dst
_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
.endm

.text
SYM_FUNC_START(__get_user_1)
check_range size=1
ASM_STAC
UACCESS movzbl (%_ASM_AX),%edx
xor %eax,%eax
ASM_CLAC
RET
SYM_FUNC_END(__get_user_1)
EXPORT_SYMBOL(__get_user_1)

Signature · 1 line
Josh