x86: add user_atomic_cmpxchg_inatomic at uaccess.h
PATCH 1/2 x86: add user_atomic_cmpxchg_inatomic at uaccess.h
This patch adds user_atomic_cmpxchg_inatomic() to use CMPXCHG instruction against a user space address.
This generalizes the already existing futex_atomic_cmpxchg_inatomic() so it can be used in other contexts. This will be used in the upcoming support for Intel MPX (Memory Protection Extensions.)
--- arch/x86/include/asm/uaccess.h | 91 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 91 insertions(+), 0 deletions(-)
arch/x86/include/asm/uaccess.h +91 -0
@@ -525,6 +525,97 @@ extern __must_check long strnlen_user(const char __user *str, long n);unsigned long __must_check clear_user (void __user *mem,unsigned long len);unsigned long __must_check __clear_user (void __user *mem,unsigned long len); +extern void __cmpxchg_wrong_size(void) + __compiletime_error("Bad argument size for cmpxchg"); + +#define __user_atomic_cmpxchg_inatomic (uval, ptr, old, new, size) \ +({ \ + int __ret = 0; \ + __typeof__(ptr) __uval = (uval); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + switch (size) { \ + case 1: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "q" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + case 2: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + case 4: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ +#ifdef CONFIG_X86_64 \ + case 8: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ +#endif \ + default: \ + __cmpxchg_wrong_size(); \ + } \ + *__uval = __old; \ + __ret; \ +}) + +#define user_atomic_cmpxchg_inatomic (uval, ptr, old, new) \ +({ \ + access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ + __user_atomic_cmpxchg_inatomic((uval), (ptr), \ + (old), (new), sizeof(*(ptr))) : \ + -EFAULT; \ +}) +/* * movsl can be slow when source and dest are not both 8-byte aligned */
Signature · 1 line
1.7.1
PATCH 2/2 x86: replace futex_atomic_cmpxchg_inatomic() with user_atomic_cmpxchg_inatomic
futex_atomic_cmpxchg_inatomic() is only the 32bit implementation of user_atomic_cmpxchg_inatomic(). This patch replaces it with user_atomic_cmpxchg_inatomic().
--- arch/x86/include/asm/futex.h | 27 ++------------------------- 1 files changed, 2 insertions(+), 25 deletions(-)
arch/x86/include/asm/futex.h +2 -25
@@ -41,6 +41,8 @@ "+m" (*uaddr), "=&r" (tem) \ : "r" (oparg), "i" (-EFAULT), "1" (0)) +#define futex_atomic_cmpxchg_inatomic user_atomic_cmpxchg_inatomic +static inline int futex_atomic_op_inuser (int encoded_op,u32 __user *uaddr) {int op = (encoded_op >>28 ) &7 ;
@@ -107,30 +109,5 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)return ret; } -static inline int futex_atomic_cmpxchg_inatomic (u32 *uval,u32 __user *uaddr, -u32 oldval,u32 newval) -{ -int ret =0 ; - -if (!access_ok (VERIFY_WRITE , uaddr,sizeof (u32))) -return -EFAULT ; - - asmvolatile ("\t" ASM_STAC "\n" -"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" -"2:\t" ASM_CLAC "\n" -"\t.section .fixup, \"ax\"\n" -"3:\tmov %3, %0\n" -"\tjmp 2b\n" -"\t.previous\n" - _ASM_EXTABLE(1b, 3b) - : "+r" (ret), "=a" (oldval), "+m" (*uaddr) - : "i" (-EFAULT), "r" (newval), "1" (oldval) - : "memory" - ); - - *uval = oldval; -return ret; -} -#endif #endif /* _ASM_X86_FUTEX_H */
Signature · 1 line
1.7.1
tip-bot for Qiaowei Ren
merge bot[tip:x86/mpx] x86: replace futex_atomic_cmpxchg_inatomic() with user_atomic_cmpxchg_inatomic
x86: replace futex_atomic_cmpxchg_inatomic() with user_atomic_cmpxchg_inatomic
Commit-ID: 0ee3b6f87d4d748d5362cb47ff33fa1553805cb4 Gitweb: http://git.kernel.org/tip/0ee3b6f87d4d748d5362cb47ff33fa1553805cb4 Author: Qiaowei Ren <[email protected]> AuthorDate: Sat, 14 Dec 2013 14:25:03 +0800 Committer: H. Peter Anvin <[email protected]> CommitDate: Mon, 16 Dec 2013 09:08:13 -0800
x86: replace futex_atomic_cmpxchg_inatomic() with user_atomic_cmpxchg_inatomic
futex_atomic_cmpxchg_inatomic() is simply the 32-bit implementation of user_atomic_cmpxchg_inatomic(), which in turn is simply a generalization of the original code in futex_atomic_cmpxchg_inatomic().
Use the newly generalized user_atomic_cmpxchg_inatomic() as the futex implementation, too.
[ hpa: retain the inline in futex.h rather than changing it to a macro ]
--- arch/x86/include/asm/futex.h | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-)
arch/x86/include/asm/futex.h +1 -20
@@ -110,26 +110,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)static inline int futex_atomic_cmpxchg_inatomic (u32 *uval,u32 __user *uaddr,u32 oldval,u32 newval) { -int ret =0 ; - -if (!access_ok (VERIFY_WRITE , uaddr,sizeof (u32))) -return -EFAULT ; - - asmvolatile ("\t" ASM_STAC "\n" -"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" -"2:\t" ASM_CLAC "\n" -"\t.section .fixup, \"ax\"\n" -"3:\tmov %3, %0\n" -"\tjmp 2b\n" -"\t.previous\n" - _ASM_EXTABLE(1b, 3b) - : "+r" (ret), "=a" (oldval), "+m" (*uaddr) - : "i" (-EFAULT), "r" (newval), "1" (oldval) - : "memory" - ); - - *uval = oldval; -return ret; +return user_atomic_cmpxchg_inatomic (uval, uaddr, oldval, newval); }#endif
Quoted 7 lines · Qiaowei Ren
This patch adds
user_atomic_cmpxchg_inatomic()to use CMPXCHG instruction against a user space address.This generalizes the already existing
futex_atomic_cmpxchg_inatomic()so it can be used in other contexts. This will be used in the upcoming support for Intel MPX (Memory Protection Extensions.)
Peter, do you have any other comments for this version?
Thanks,
Qiaowei
H. Peter Anvin
reviewerQuoted 8 lines · Qiaowei Ren
arch/x86/include/asm/uaccess.h +8
-0+#ifdef CONFIG_X86_64 \ +case 8 : \ + { \ + asmvolatile ("\t" ASM_STAC "\n" \ +"1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ +"2:\t" ASM_CLAC "\n" \ +"\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \
arch/x86/include/asm/uaccess.h extern __must_check long strnlen_user(const char __user *str, long n); · line 603
Quoted 10 lines
arch/x86/include/asm/uaccess.h +10
-0+ "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ +#endif \
This, of course, doesn't work because this is embedded in a macro. I realize I requested it, but you should still have tested it.
Anyway, I'll fix it up.
-hpa
tip-bot for Qiaowei Ren
merge botx86: add user_atomic_cmpxchg_inatomic at uaccess.h
Commit-ID: f09174c501f8bb259788cc36d5a7aa5b2831fb5e Gitweb: http://git.kernel.org/tip/f09174c501f8bb259788cc36d5a7aa5b2831fb5e Author: Qiaowei Ren <[email protected]> AuthorDate: Sat, 14 Dec 2013 14:25:02 +0800 Committer: H. Peter Anvin <[email protected]> CommitDate: Mon, 16 Dec 2013 09:07:57 -0800
x86: add user_atomic_cmpxchg_inatomic at uaccess.h
This patch adds user_atomic_cmpxchg_inatomic() to use CMPXCHG instruction against a user space address.
This generalizes the already existing futex_atomic_cmpxchg_inatomic() so it can be used in other contexts. This will be used in the upcoming support for Intel MPX (Memory Protection Extensions.)
[ hpa: replaced #ifdef inside a macro with IS_ENABLED() ]
--- arch/x86/include/asm/uaccess.h | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+)
arch/x86/include/asm/uaccess.h +92 -0
@@ -525,6 +525,98 @@ extern __must_check long strnlen_user(const char __user *str, long n);unsigned long __must_check clear_user (void __user *mem,unsigned long len);unsigned long __must_check __clear_user (void __user *mem,unsigned long len); +extern void __cmpxchg_wrong_size(void) + __compiletime_error("Bad argument size for cmpxchg"); + +#define __user_atomic_cmpxchg_inatomic (uval, ptr, old, new, size) \ +({ \ + int __ret = 0; \ + __typeof__(ptr) __uval = (uval); \ + __typeof__(*(ptr)) __old = (old); \ + __typeof__(*(ptr)) __new = (new); \ + switch (size) { \ + case 1: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "q" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + case 2: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + case 4: \ + { \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + case 8: \ + { \ + if (!IS_ENABLED(CONFIG_X86_64)) \ + __cmpxchg_wrong_size(); \ + \ + asm volatile("\t" ASM_STAC "\n" \ + "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ + "2:\t" ASM_CLAC "\n" \ + "\t.section .fixup, \"ax\"\n" \ + "3:\tmov %3, %0\n" \ + "\tjmp 2b\n" \ + "\t.previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ + : "i" (-EFAULT), "r" (__new), "1" (__old) \ + : "memory" \ + ); \ + break; \ + } \ + default: \ + __cmpxchg_wrong_size(); \ + } \ + *__uval = __old; \ + __ret; \ +}) + +#define user_atomic_cmpxchg_inatomic (uval, ptr, old, new) \ +({ \ + access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ + __user_atomic_cmpxchg_inatomic((uval), (ptr), \ + (old), (new), sizeof(*(ptr))) : \ + -EFAULT; \ +}) +/* * movsl can be slow when source and dest are not both 8-byte aligned */