x86/percpu: Introduce pcpu_binary_op() macro
Subsystems (2)
PATCH 1/5 x86/percpu: Introduce pcpu_binary_op() macro
Introduce pcpu_binary_op() macro, a copy of percpu_to_op() macro. Update percpu binary operators to use the new macro, since percpu_to_op() will be re-purposed as a raw percpu write accessor in a follow-up patch.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 47 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 17 deletions(-)
arch/x86/include/asm/percpu.h +30 -17
@@ -163,6 +163,19 @@ do { \ : [var] "+m" (__my_cpu_var(_var))); \ }) +#define percpu_binary_op (size, qual, op, _var, _val) \ +do { \ + __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ + if (0) { \ + typeof(_var) pto_tmp__; \ + pto_tmp__ = (_val); \ + (void)pto_tmp__; \ + } \ + asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + : [var] "+m" (__my_cpu_var(_var)) \ + : [val] __pcpu_reg_imm_##size(pto_val__)); \ +} while (0) +/* * Generate a percpu add to memory instruction and optimize code * if one is added or subtracted.
@@ -182,7 +195,7 @@ do { \else if (pao_ID__ ==-1 ) \percpu_unary_op (size, qual,"dec" , var); \else \ -percpu_to_op (size, qual,"add" , var, val); \ +percpu_binary_op (size, qual,"add" , var, val); \ }while (0 ) #define percpu_from_op(size, qual, op, _var) \
@@ -492,12 +505,12 @@ do { \#define raw_cpu_add_1 (pcp, val) percpu_add_op(1, , (pcp), val)#define raw_cpu_add_2 (pcp, val) percpu_add_op(2, , (pcp), val)#define raw_cpu_add_4 (pcp, val) percpu_add_op(4, , (pcp), val) -#define raw_cpu_and_1 (pcp, val) percpu_to_op(1, , "and", (pcp), val) -#define raw_cpu_and_2 (pcp, val) percpu_to_op(2, , "and", (pcp), val) -#define raw_cpu_and_4 (pcp, val) percpu_to_op(4, , "and", (pcp), val) -#define raw_cpu_or_1 (pcp, val) percpu_to_op(1, , "or", (pcp), val) -#define raw_cpu_or_2 (pcp, val) percpu_to_op(2, , "or", (pcp), val) -#define raw_cpu_or_4 (pcp, val) percpu_to_op(4, , "or", (pcp), val) +#define raw_cpu_and_1 (pcp, val) percpu_binary_op(1, , "and", (pcp), val) +#define raw_cpu_and_2 (pcp, val) percpu_binary_op(2, , "and", (pcp), val) +#define raw_cpu_and_4 (pcp, val) percpu_binary_op(4, , "and", (pcp), val) +#define raw_cpu_or_1 (pcp, val) percpu_binary_op(1, , "or", (pcp), val) +#define raw_cpu_or_2 (pcp, val) percpu_binary_op(2, , "or", (pcp), val) +#define raw_cpu_or_4 (pcp, val) percpu_binary_op(4, , "or", (pcp), val)#define raw_cpu_xchg_1 (pcp, val) raw_percpu_xchg_op(pcp, val)#define raw_cpu_xchg_2 (pcp, val) raw_percpu_xchg_op(pcp, val)#define raw_cpu_xchg_4 (pcp, val) raw_percpu_xchg_op(pcp, val)
@@ -505,12 +518,12 @@ do { \#define this_cpu_add_1 (pcp, val) percpu_add_op(1, volatile, (pcp), val)#define this_cpu_add_2 (pcp, val) percpu_add_op(2, volatile, (pcp), val)#define this_cpu_add_4 (pcp, val) percpu_add_op(4, volatile, (pcp), val) -#define this_cpu_and_1 (pcp, val) percpu_to_op(1, volatile, "and", (pcp), val) -#define this_cpu_and_2 (pcp, val) percpu_to_op(2, volatile, "and", (pcp), val) -#define this_cpu_and_4 (pcp, val) percpu_to_op(4, volatile, "and", (pcp), val) -#define this_cpu_or_1 (pcp, val) percpu_to_op(1, volatile, "or", (pcp), val) -#define this_cpu_or_2 (pcp, val) percpu_to_op(2, volatile, "or", (pcp), val) -#define this_cpu_or_4 (pcp, val) percpu_to_op(4, volatile, "or", (pcp), val) +#define this_cpu_and_1 (pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) +#define this_cpu_and_2 (pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) +#define this_cpu_and_4 (pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) +#define this_cpu_or_1 (pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) +#define this_cpu_or_2 (pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) +#define this_cpu_or_4 (pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)#define this_cpu_xchg_1 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_xchg_2 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_xchg_4 (pcp, nval) this_percpu_xchg_op(pcp, nval)
@@ -543,16 +556,16 @@ do { \#define this_cpu_read_stable_8 (pcp) percpu_stable_op(8, "mov", pcp)#define raw_cpu_add_8 (pcp, val) percpu_add_op(8, , (pcp), val) -#define raw_cpu_and_8 (pcp, val) percpu_to_op(8, , "and", (pcp), val) -#define raw_cpu_or_8 (pcp, val) percpu_to_op(8, , "or", (pcp), val) +#define raw_cpu_and_8 (pcp, val) percpu_binary_op(8, , "and", (pcp), val) +#define raw_cpu_or_8 (pcp, val) percpu_binary_op(8, , "or", (pcp), val)#define raw_cpu_add_return_8 (pcp, val) percpu_add_return_op(8, , pcp, val)#define raw_cpu_xchg_8 (pcp, nval) raw_percpu_xchg_op(pcp, nval)#define raw_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)#define this_cpu_add_8 (pcp, val) percpu_add_op(8, volatile, (pcp), val) -#define this_cpu_and_8 (pcp, val) percpu_to_op(8, volatile, "and", (pcp), val) -#define this_cpu_or_8 (pcp, val) percpu_to_op(8, volatile, "or", (pcp), val) +#define this_cpu_and_8 (pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) +#define this_cpu_or_8 (pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(8, volatile, pcp, val)#define this_cpu_xchg_8 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
Signature · 1 line
2.44.0
PATCH 2/5 x86/percpu: Move some percpu macros around
Move some percpu macros around to make a follow-up patch more readable.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 63 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 29 deletions(-)
arch/x86/include/asm/percpu.h +34 -29
@@ -144,6 +144,29 @@#define __pcpu_reg_imm_4 (x) "ri" (x)#define __pcpu_reg_imm_8 (x) "re" (x) +#ifdef CONFIG_USE_X86_SEG_SUPPORT + +#define __raw_cpu_read (qual, pcp) \ +({ \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ +}) + +#define __raw_cpu_write (qual, pcp, val) \ +do { \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ +} while (0) + +#else /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_from_op (size, qual, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "m" (__my_cpu_var(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) +#define percpu_to_op (size, qual, op, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
@@ -157,6 +180,17 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_stable_op (size, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "i" (&(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) +#define percpu_unary_op (size, qual, op, _var) \ ({ \ asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
@@ -198,24 +232,6 @@ do { \percpu_binary_op (size, qual,"add" , var, val); \ }while (0 ) -#define percpu_from_op (size, qual, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "m" (__my_cpu_var(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) - -#define percpu_stable_op (size, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "i" (&(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) -/* * Add return operation */
@@ -433,17 +449,6 @@ do { \#define this_cpu_read_stable (pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)#ifdef CONFIG_USE_X86_SEG_SUPPORT - -#define __raw_cpu_read (qual, pcp) \ -({ \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ -}) - -#define __raw_cpu_write (qual, pcp, val) \ -do { \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ -} while (0) -#define raw_cpu_read_1 (pcp) __raw_cpu_read(, pcp)#define raw_cpu_read_2 (pcp) __raw_cpu_read(, pcp)#define raw_cpu_read_4 (pcp) __raw_cpu_read(, pcp)
Signature · 1 line
2.44.0
[tip: x86/percpu] x86/percpu: Move some percpu macros around for readability
The following commit has been merged into the x86/percpu branch of tip:
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 08d564ad699ef32ceaf99d238b3d9c1f4ce5c998 Gitweb: https://git.kernel.org/tip/08d564ad699ef32ceaf99d238b3d9c1f4ce5c998 Author: Uros Bizjak <[email protected]> AuthorDate: Tue, 30 Apr 2024 11:17:21 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 18 May 2024 11:18:41 +02:00
x86/percpu: Move some percpu macros around for readability
Move some percpu macros around to make a follow-up patch more readable.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 63 ++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 29 deletions(-)
arch/x86/include/asm/percpu.h +34 -29
@@ -144,6 +144,29 @@#define __pcpu_reg_imm_4 (x) "ri" (x)#define __pcpu_reg_imm_8 (x) "re" (x) +#ifdef CONFIG_USE_X86_SEG_SUPPORT + +#define __raw_cpu_read (qual, pcp) \ +({ \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ +}) + +#define __raw_cpu_write (qual, pcp, val) \ +do { \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ +} while (0) + +#else /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_from_op (size, qual, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "m" (__my_cpu_var(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) +#define percpu_to_op (size, qual, op, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
@@ -157,6 +180,17 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_stable_op (size, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "i" (&(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) +#define percpu_unary_op (size, qual, op, _var) \ ({ \ asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
@@ -198,24 +232,6 @@ do { \percpu_binary_op (size, qual,"add" , var, val); \ }while (0 ) -#define percpu_from_op (size, qual, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "m" (__my_cpu_var(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) - -#define percpu_stable_op (size, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "i" (&(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) -/* * Add return operation */
@@ -433,17 +449,6 @@ do { \#define this_cpu_read_stable (pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)#ifdef CONFIG_USE_X86_SEG_SUPPORT - -#define __raw_cpu_read (qual, pcp) \ -({ \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ -}) - -#define __raw_cpu_write (qual, pcp, val) \ -do { \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ -} while (0) -#define raw_cpu_read_1 (pcp) __raw_cpu_read(, pcp)#define raw_cpu_read_2 (pcp) __raw_cpu_read(, pcp)#define raw_cpu_read_4 (pcp) __raw_cpu_read(, pcp)
PATCH 3/5 x86/percpu: Unify percpu read-write accessors
In reply to Uros BizjakRedefine percpu_from_op() and percpu_to_op() as __raw_cpu_read() and __raw_cpu_write(). Unify __raw_cpu_{read,write}() macros between configs w/ and w/o USE_X86_SEG_SUPPORT in order to unify {raw,this}_cpu{read_write}_N() accessors between configs.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 72 ++++++++++++----------------------- 1 file changed, 25 insertions(+), 47 deletions(-)
arch/x86/include/asm/percpu.h +25 -47
@@ -146,28 +146,28 @@#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define __raw_cpu_read (qual, pcp) \ +#define __raw_cpu_read (size, qual, pcp) \ ({ \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ }) -#define __raw_cpu_write (qual, pcp, val) \ +#define __raw_cpu_write (size, qual, pcp, val) \ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0)#else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define percpu_from_op (size, qual, op, _var) \ +#define __raw_cpu_read (size, qual, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "m" (__my_cpu_var(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ }) -#define percpu_to_op (size, qual, op, _var, _val) \ +#define __raw_cpu_write (size, qual, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ if (0) { \
@@ -175,7 +175,7 @@ do { \ pto_tmp__ = (_val); \ (void )pto_tmp__; \ } \ - asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ : [var] "+m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0)
@@ -448,54 +448,32 @@ do { \*/ #define this_cpu_read_stable (pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) -#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define raw_cpu_read_1 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_2 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_4 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_1 (pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_2 (pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_4 (pcp, val) __raw_cpu_write(, pcp, val) - -#define this_cpu_read_1 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_2 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_4 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_1 (pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_2 (pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_4 (pcp, val) __raw_cpu_write(volatile, pcp, val) +#define raw_cpu_read_1 (pcp) __raw_cpu_read(1, , pcp) +#define raw_cpu_read_2 (pcp) __raw_cpu_read(2, , pcp) +#define raw_cpu_read_4 (pcp) __raw_cpu_read(4, , pcp) +#define raw_cpu_write_1 (pcp, val) __raw_cpu_write(1, , pcp, val) +#define raw_cpu_write_2 (pcp, val) __raw_cpu_write(2, , pcp, val) +#define raw_cpu_write_4 (pcp, val) __raw_cpu_write(4, , pcp, val) + +#define this_cpu_read_1 (pcp) __raw_cpu_read(1, volatile, pcp) +#define this_cpu_read_2 (pcp) __raw_cpu_read(2, volatile, pcp) +#define this_cpu_read_4 (pcp) __raw_cpu_read(4, volatile, pcp) +#define this_cpu_write_1 (pcp, val) __raw_cpu_write(1, volatile, pcp, val) +#define this_cpu_write_2 (pcp, val) __raw_cpu_write(2, volatile, pcp, val) +#define this_cpu_write_4 (pcp, val) __raw_cpu_write(4, volatile, pcp, val)#ifdef CONFIG_X86_64 -#define raw_cpu_read_8 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_8 (pcp, val) __raw_cpu_write(, pcp, val) +#define raw_cpu_read_8 (pcp) __raw_cpu_read(8, , pcp) +#define raw_cpu_write_8 (pcp, val) __raw_cpu_write(8, , pcp, val) -#define this_cpu_read_8 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_8 (pcp, val) __raw_cpu_write(volatile, pcp, val) +#define this_cpu_read_8 (pcp) __raw_cpu_read(8, volatile, pcp) +#define this_cpu_write_8 (pcp, val) __raw_cpu_write(8, volatile, pcp, val)#endif -#define this_cpu_read_const (pcp) __raw_cpu_read(, pcp) +#ifdef CONFIG_USE_X86_SEG_SUPPORT +#define this_cpu_read_const (pcp) __raw_cpu_read(, , pcp)#else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define raw_cpu_read_1 (pcp) percpu_from_op(1, , "mov", pcp) -#define raw_cpu_read_2 (pcp) percpu_from_op(2, , "mov", pcp) -#define raw_cpu_read_4 (pcp) percpu_from_op(4, , "mov", pcp) -#define raw_cpu_write_1 (pcp, val) percpu_to_op(1, , "mov", (pcp), val) -#define raw_cpu_write_2 (pcp, val) percpu_to_op(2, , "mov", (pcp), val) -#define raw_cpu_write_4 (pcp, val) percpu_to_op(4, , "mov", (pcp), val) - -#define this_cpu_read_1 (pcp) percpu_from_op(1, volatile, "mov", pcp) -#define this_cpu_read_2 (pcp) percpu_from_op(2, volatile, "mov", pcp) -#define this_cpu_read_4 (pcp) percpu_from_op(4, volatile, "mov", pcp) -#define this_cpu_write_1 (pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val) -#define this_cpu_write_2 (pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val) -#define this_cpu_write_4 (pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val) - -#ifdef CONFIG_X86_64 -#define raw_cpu_read_8 (pcp) percpu_from_op(8, , "mov", pcp) -#define raw_cpu_write_8 (pcp, val) percpu_to_op(8, , "mov", (pcp), val) - -#define this_cpu_read_8 (pcp) percpu_from_op(8, volatile, "mov", pcp) -#define this_cpu_write_8 (pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val) -#endif -/* * The generic per-cpu infrastrucutre is not suitable for * reading const-qualified variables.
Signature · 1 line
2.44.0
[tip: x86/percpu] x86/percpu: Unify percpu read-write accessors
The following commit has been merged into the x86/percpu branch of tip:
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: a50ea641296699af1947336c2e75f6234f53548a Gitweb: https://git.kernel.org/tip/a50ea641296699af1947336c2e75f6234f53548a Author: Uros Bizjak <[email protected]> AuthorDate: Tue, 30 Apr 2024 11:17:22 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 18 May 2024 11:18:41 +02:00
x86/percpu: Unify percpu read-write accessors
Redefine percpu_from_op() and percpu_to_op() as __raw_cpu_read() and __raw_cpu_write(). Unify __raw_cpu_{read,write}() macros between configs w/ and w/o USE_X86_SEG_SUPPORT in order to unify {raw,this}_cpu{read_write}_N() accessors between configs.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 72 +++++++++++----------------------- 1 file changed, 25 insertions(+), 47 deletions(-)
arch/x86/include/asm/percpu.h +25 -47
@@ -146,28 +146,28 @@#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define __raw_cpu_read (qual, pcp) \ +#define __raw_cpu_read (size, qual, pcp) \ ({ \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ }) -#define __raw_cpu_write (qual, pcp, val) \ +#define __raw_cpu_write (size, qual, pcp, val) \ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0)#else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define percpu_from_op (size, qual, op, _var) \ +#define __raw_cpu_read (size, qual, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "m" (__my_cpu_var(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ }) -#define percpu_to_op (size, qual, op, _var, _val) \ +#define __raw_cpu_write (size, qual, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ if (0) { \
@@ -175,7 +175,7 @@ do { \ pto_tmp__ = (_val); \ (void )pto_tmp__; \ } \ - asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ : [var] "+m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0)
@@ -448,54 +448,32 @@ do { \*/ #define this_cpu_read_stable (pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) -#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define raw_cpu_read_1 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_2 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_4 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_1 (pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_2 (pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_4 (pcp, val) __raw_cpu_write(, pcp, val) - -#define this_cpu_read_1 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_2 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_4 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_1 (pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_2 (pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_4 (pcp, val) __raw_cpu_write(volatile, pcp, val) +#define raw_cpu_read_1 (pcp) __raw_cpu_read(1, , pcp) +#define raw_cpu_read_2 (pcp) __raw_cpu_read(2, , pcp) +#define raw_cpu_read_4 (pcp) __raw_cpu_read(4, , pcp) +#define raw_cpu_write_1 (pcp, val) __raw_cpu_write(1, , pcp, val) +#define raw_cpu_write_2 (pcp, val) __raw_cpu_write(2, , pcp, val) +#define raw_cpu_write_4 (pcp, val) __raw_cpu_write(4, , pcp, val) + +#define this_cpu_read_1 (pcp) __raw_cpu_read(1, volatile, pcp) +#define this_cpu_read_2 (pcp) __raw_cpu_read(2, volatile, pcp) +#define this_cpu_read_4 (pcp) __raw_cpu_read(4, volatile, pcp) +#define this_cpu_write_1 (pcp, val) __raw_cpu_write(1, volatile, pcp, val) +#define this_cpu_write_2 (pcp, val) __raw_cpu_write(2, volatile, pcp, val) +#define this_cpu_write_4 (pcp, val) __raw_cpu_write(4, volatile, pcp, val)#ifdef CONFIG_X86_64 -#define raw_cpu_read_8 (pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_8 (pcp, val) __raw_cpu_write(, pcp, val) +#define raw_cpu_read_8 (pcp) __raw_cpu_read(8, , pcp) +#define raw_cpu_write_8 (pcp, val) __raw_cpu_write(8, , pcp, val) -#define this_cpu_read_8 (pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_8 (pcp, val) __raw_cpu_write(volatile, pcp, val) +#define this_cpu_read_8 (pcp) __raw_cpu_read(8, volatile, pcp) +#define this_cpu_write_8 (pcp, val) __raw_cpu_write(8, volatile, pcp, val)#endif -#define this_cpu_read_const (pcp) __raw_cpu_read(, pcp) +#ifdef CONFIG_USE_X86_SEG_SUPPORT +#define this_cpu_read_const (pcp) __raw_cpu_read(, , pcp)#else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define raw_cpu_read_1 (pcp) percpu_from_op(1, , "mov", pcp) -#define raw_cpu_read_2 (pcp) percpu_from_op(2, , "mov", pcp) -#define raw_cpu_read_4 (pcp) percpu_from_op(4, , "mov", pcp) -#define raw_cpu_write_1 (pcp, val) percpu_to_op(1, , "mov", (pcp), val) -#define raw_cpu_write_2 (pcp, val) percpu_to_op(2, , "mov", (pcp), val) -#define raw_cpu_write_4 (pcp, val) percpu_to_op(4, , "mov", (pcp), val) - -#define this_cpu_read_1 (pcp) percpu_from_op(1, volatile, "mov", pcp) -#define this_cpu_read_2 (pcp) percpu_from_op(2, volatile, "mov", pcp) -#define this_cpu_read_4 (pcp) percpu_from_op(4, volatile, "mov", pcp) -#define this_cpu_write_1 (pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val) -#define this_cpu_write_2 (pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val) -#define this_cpu_write_4 (pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val) - -#ifdef CONFIG_X86_64 -#define raw_cpu_read_8 (pcp) percpu_from_op(8, , "mov", pcp) -#define raw_cpu_write_8 (pcp, val) percpu_to_op(8, , "mov", (pcp), val) - -#define this_cpu_read_8 (pcp) percpu_from_op(8, volatile, "mov", pcp) -#define this_cpu_write_8 (pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val) -#endif -/* * The generic per-cpu infrastrucutre is not suitable for * reading const-qualified variables.
PATCH 4/5 x86/percpu: Introduce __raw_cpu_read_const() macro
In reply to Uros BizjakIntroduce __raw_cpu_read_const() macro to further reduce ifdeffery and differences between configs w/ and w/o USE_X86_SEG_SUPPORT.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
arch/x86/include/asm/percpu.h +9 -10
@@ -156,6 +156,8 @@ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0) +#define __raw_cpu_read_const (pcp) __raw_cpu_read(, , pcp) +#else /* CONFIG_USE_X86_SEG_SUPPORT */ #define __raw_cpu_read (size, qual, _var) \
@@ -180,6 +182,12 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +/* +* The generic per-cpu infrastrucutre is not suitable for +* reading const-qualified variables. +*/ +#define __raw_cpu_read_const (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ #define percpu_stable_op (size, op, _var) \
@@ -470,16 +478,7 @@ do { \#define this_cpu_write_8 (pcp, val) __raw_cpu_write(8, volatile, pcp, val)#endif -#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define this_cpu_read_const (pcp) __raw_cpu_read(, , pcp) -#else /* CONFIG_USE_X86_SEG_SUPPORT */ - -/* -* The generic per-cpu infrastrucutre is not suitable for -* reading const-qualified variables. -*/ -#define this_cpu_read_const (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) -#endif /* CONFIG_USE_X86_SEG_SUPPORT */ +#define this_cpu_read_const (pcp) __raw_cpu_read_const(pcp)#define this_cpu_read_stable_1 (pcp) percpu_stable_op(1, "mov", pcp)#define this_cpu_read_stable_2 (pcp) percpu_stable_op(2, "mov", pcp)
Signature · 1 line
2.44.0
[tip: x86/percpu] x86/percpu: Introduce the __raw_cpu_read_const() macro
The following commit has been merged into the x86/percpu branch of tip:
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 539615de7004a46778020183622856f4ca14e4ac Gitweb: https://git.kernel.org/tip/539615de7004a46778020183622856f4ca14e4ac Author: Uros Bizjak <[email protected]> AuthorDate: Tue, 30 Apr 2024 11:17:23 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 18 May 2024 11:18:42 +02:00
x86/percpu: Introduce the __raw_cpu_read_const() macro
Introduce the __raw_cpu_read_const() macro to further reduce ifdeffery and differences between configs w/ and w/o USE_X86_SEG_SUPPORT.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
arch/x86/include/asm/percpu.h +9 -10
@@ -156,6 +156,8 @@ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0) +#define __raw_cpu_read_const (pcp) __raw_cpu_read(, , pcp) +#else /* CONFIG_USE_X86_SEG_SUPPORT */ #define __raw_cpu_read (size, qual, _var) \
@@ -180,6 +182,12 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +/* +* The generic per-cpu infrastrucutre is not suitable for +* reading const-qualified variables. +*/ +#define __raw_cpu_read_const (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ #define percpu_stable_op (size, op, _var) \
@@ -470,16 +478,7 @@ do { \#define this_cpu_write_8 (pcp, val) __raw_cpu_write(8, volatile, pcp, val)#endif -#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define this_cpu_read_const (pcp) __raw_cpu_read(, , pcp) -#else /* CONFIG_USE_X86_SEG_SUPPORT */ - -/* -* The generic per-cpu infrastrucutre is not suitable for -* reading const-qualified variables. -*/ -#define this_cpu_read_const (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) -#endif /* CONFIG_USE_X86_SEG_SUPPORT */ +#define this_cpu_read_const (pcp) __raw_cpu_read_const(pcp)#define this_cpu_read_stable_1 (pcp) percpu_stable_op(1, "mov", pcp)#define this_cpu_read_stable_2 (pcp) percpu_stable_op(2, "mov", pcp)
PATCH 5/5 x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
In reply to Uros Bizjak__raw_cpu_write() with !USE_X86_SEG_SUPPORT config uses read/write operand constraint modifier "+" for its memory location. This signals the compiler that the location is both read and written by the asm. This is not true, because MOV insn only writes to the output.
Correct the modifier to "=" to inform the compiler that the memory location is only written to. This also prevents the compiler from value tracking the undefined value from the uninitialized memory.
--- arch/x86/include/asm/percpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/include/asm/percpu.h +1 -1
@@ -178,7 +178,7 @@ do { \ (void )pto_tmp__; \ } \ asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ - : [var] "+m" (__my_cpu_var(_var)) \ + : [var] "=m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0)
Signature · 1 line
2.44.0
[tip: x86/percpu] x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
The following commit has been merged into the x86/percpu branch of tip:
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 1fe67aee8ab3fdab4357afc983a9e9ff3892d694 Gitweb: https://git.kernel.org/tip/1fe67aee8ab3fdab4357afc983a9e9ff3892d694 Author: Uros Bizjak <[email protected]> AuthorDate: Tue, 30 Apr 2024 11:17:24 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 18 May 2024 11:18:42 +02:00
x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
__raw_cpu_write() with !USE_X86_SEG_SUPPORT config uses read/write operand constraint modifier "+" for its memory location. This signals the compiler that the location is both read and written by the asm. This is not true, because MOV insn only writes to the output.
Correct the modifier to "=" to inform the compiler that the memory location is only written to. This also prevents the compiler from value tracking the undefined value from the uninitialized memory.
--- arch/x86/include/asm/percpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/include/asm/percpu.h +1 -1
@@ -178,7 +178,7 @@ do { \ (void )pto_tmp__; \ } \ asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ - : [var] "+m" (__my_cpu_var(_var)) \ + : [var] "=m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0)
[tip: x86/percpu] x86/percpu: Introduce the pcpu_binary_op() macro
In reply to Uros BizjakThe following commit has been merged into the x86/percpu branch of tip:
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 455ca134a7b2121dd739d425d3be313fb52f0651 Gitweb: https://git.kernel.org/tip/455ca134a7b2121dd739d425d3be313fb52f0651 Author: Uros Bizjak <[email protected]> AuthorDate: Tue, 30 Apr 2024 11:17:20 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 18 May 2024 11:18:40 +02:00
x86/percpu: Introduce the pcpu_binary_op() macro
Introduce the pcpu_binary_op() macro, a copy of the percpu_to_op() macro.
Update percpu binary operators to use the new macro, since percpu_to_op() will be re-purposed as a raw percpu write accessor in a follow-up patch.
No functional change intended.
--- arch/x86/include/asm/percpu.h | 47 +++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 17 deletions(-)
arch/x86/include/asm/percpu.h +30 -17
@@ -163,6 +163,19 @@ do { \ : [var] "+m" (__my_cpu_var(_var))); \ }) +#define percpu_binary_op (size, qual, op, _var, _val) \ +do { \ + __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ + if (0) { \ + typeof(_var) pto_tmp__; \ + pto_tmp__ = (_val); \ + (void)pto_tmp__; \ + } \ + asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + : [var] "+m" (__my_cpu_var(_var)) \ + : [val] __pcpu_reg_imm_##size(pto_val__)); \ +} while (0) +/* * Generate a percpu add to memory instruction and optimize code * if one is added or subtracted.
@@ -182,7 +195,7 @@ do { \else if (pao_ID__ ==-1 ) \percpu_unary_op (size, qual,"dec" , var); \else \ -percpu_to_op (size, qual,"add" , var, val); \ +percpu_binary_op (size, qual,"add" , var, val); \ }while (0 ) #define percpu_from_op(size, qual, op, _var) \
@@ -492,12 +505,12 @@ do { \#define raw_cpu_add_1 (pcp, val) percpu_add_op(1, , (pcp), val)#define raw_cpu_add_2 (pcp, val) percpu_add_op(2, , (pcp), val)#define raw_cpu_add_4 (pcp, val) percpu_add_op(4, , (pcp), val) -#define raw_cpu_and_1 (pcp, val) percpu_to_op(1, , "and", (pcp), val) -#define raw_cpu_and_2 (pcp, val) percpu_to_op(2, , "and", (pcp), val) -#define raw_cpu_and_4 (pcp, val) percpu_to_op(4, , "and", (pcp), val) -#define raw_cpu_or_1 (pcp, val) percpu_to_op(1, , "or", (pcp), val) -#define raw_cpu_or_2 (pcp, val) percpu_to_op(2, , "or", (pcp), val) -#define raw_cpu_or_4 (pcp, val) percpu_to_op(4, , "or", (pcp), val) +#define raw_cpu_and_1 (pcp, val) percpu_binary_op(1, , "and", (pcp), val) +#define raw_cpu_and_2 (pcp, val) percpu_binary_op(2, , "and", (pcp), val) +#define raw_cpu_and_4 (pcp, val) percpu_binary_op(4, , "and", (pcp), val) +#define raw_cpu_or_1 (pcp, val) percpu_binary_op(1, , "or", (pcp), val) +#define raw_cpu_or_2 (pcp, val) percpu_binary_op(2, , "or", (pcp), val) +#define raw_cpu_or_4 (pcp, val) percpu_binary_op(4, , "or", (pcp), val)#define raw_cpu_xchg_1 (pcp, val) raw_percpu_xchg_op(pcp, val)#define raw_cpu_xchg_2 (pcp, val) raw_percpu_xchg_op(pcp, val)#define raw_cpu_xchg_4 (pcp, val) raw_percpu_xchg_op(pcp, val)
@@ -505,12 +518,12 @@ do { \#define this_cpu_add_1 (pcp, val) percpu_add_op(1, volatile, (pcp), val)#define this_cpu_add_2 (pcp, val) percpu_add_op(2, volatile, (pcp), val)#define this_cpu_add_4 (pcp, val) percpu_add_op(4, volatile, (pcp), val) -#define this_cpu_and_1 (pcp, val) percpu_to_op(1, volatile, "and", (pcp), val) -#define this_cpu_and_2 (pcp, val) percpu_to_op(2, volatile, "and", (pcp), val) -#define this_cpu_and_4 (pcp, val) percpu_to_op(4, volatile, "and", (pcp), val) -#define this_cpu_or_1 (pcp, val) percpu_to_op(1, volatile, "or", (pcp), val) -#define this_cpu_or_2 (pcp, val) percpu_to_op(2, volatile, "or", (pcp), val) -#define this_cpu_or_4 (pcp, val) percpu_to_op(4, volatile, "or", (pcp), val) +#define this_cpu_and_1 (pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) +#define this_cpu_and_2 (pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) +#define this_cpu_and_4 (pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) +#define this_cpu_or_1 (pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) +#define this_cpu_or_2 (pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) +#define this_cpu_or_4 (pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)#define this_cpu_xchg_1 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_xchg_2 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_xchg_4 (pcp, nval) this_percpu_xchg_op(pcp, nval)
@@ -543,16 +556,16 @@ do { \#define this_cpu_read_stable_8 (pcp) percpu_stable_op(8, "mov", pcp)#define raw_cpu_add_8 (pcp, val) percpu_add_op(8, , (pcp), val) -#define raw_cpu_and_8 (pcp, val) percpu_to_op(8, , "and", (pcp), val) -#define raw_cpu_or_8 (pcp, val) percpu_to_op(8, , "or", (pcp), val) +#define raw_cpu_and_8 (pcp, val) percpu_binary_op(8, , "and", (pcp), val) +#define raw_cpu_or_8 (pcp, val) percpu_binary_op(8, , "or", (pcp), val)#define raw_cpu_add_return_8 (pcp, val) percpu_add_return_op(8, , pcp, val)#define raw_cpu_xchg_8 (pcp, nval) raw_percpu_xchg_op(pcp, nval)#define raw_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)#define this_cpu_add_8 (pcp, val) percpu_add_op(8, volatile, (pcp), val) -#define this_cpu_and_8 (pcp, val) percpu_to_op(8, volatile, "and", (pcp), val) -#define this_cpu_or_8 (pcp, val) percpu_to_op(8, volatile, "or", (pcp), val) +#define this_cpu_and_8 (pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) +#define this_cpu_or_8 (pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)#define this_cpu_add_return_8 (pcp, val) percpu_add_return_op(8, volatile, pcp, val)#define this_cpu_xchg_8 (pcp, nval) this_percpu_xchg_op(pcp, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)