Skip to contentSkip to conversation

x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template

PATCH Applied Uros Bizjak 2 participants 6 messages Apr 4, 2024 – Apr 6, 2024

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/3 x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template

Fix x86_this_cpu_variable_test_bit(), which is implemented with wrong asm template, where argument 2 (count argument) is considered as percpu variable. However, x86_this_cpu_test_bit() is currently used exclusively with constant bit number argument, so the called x86_this_cpu_variable_test_bit() function is never instantiated. The fix introduces named assembler operands to prevent this kind of errors.

Cc Ingo Molnar <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Josh Poimboeuf <[email protected]>

---
v2: Split from the original v1 patch.

---
 arch/x86/include/asm/percpu.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
arch/x86/include/asm/percpu.h +3 -2
index 20696df5d567..cbfbbe836ee2 100644
@@ -586,10 +586,11 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
 {
 	bool oldbit;
 
-	asm volatile("btl "__percpu_arg(2)",%1"
+	asm volatile("btl %[nr], " __percpu_arg([var])
 			CC_SET(c)
 			: CC_OUT(c) (oldbit)
-			: "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr));
+			: [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
+			  [nr] "Ir" (nr));
 
 	return oldbit;
 }
Signature · 1 line
2.44.0
#1 lore raw

PATCH v2 2/3 x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros

Rewrite the whole family of x86_this_cpu_test_bit() functions as macros, so standard __my_cpu_var() and raw_cpu_read() macros can be used on percpu variables. This approach considerably simplifies implementation of functions and also introduces standard checks on accessed percpu variables.

No functional changes intended.

Cc Ingo Molnar <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Josh Poimboeuf <[email protected]>

---
v2: Split from the original v1 patch.

---
 arch/um/include/asm/cpufeature.h  |  3 +-
 arch/x86/include/asm/cpufeature.h |  3 +-
 arch/x86/include/asm/percpu.h     | 54 +++++++++++++------------------
 3 files changed, 25 insertions(+), 35 deletions(-)
arch/um/include/asm/cpufeature.h +1 -2
index 66fe06db872f..1eb8b834fbec 100644
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)						\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
-	 x86_this_cpu_test_bit(bit,					\
-		(unsigned long __percpu *)&cpu_info.x86_capability))
+	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
 
 /*
  * This macro is for detection of features which need kernel
arch/x86/include/asm/cpufeature.h +1 -2
index f95e1c8b7c01..7d527467ab9c 100644
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)						\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
-	 x86_this_cpu_test_bit(bit,					\
-		(unsigned long __percpu *)&cpu_info.x86_capability))
+	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
 
 /*
  * This macro is for detection of features which need kernel
arch/x86/include/asm/percpu.h +23 -31

67 lines not on this page · raw message

Signature · 1 line
2.44.0
#3 lore raw

PATCH v2 3/3 x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery

In reply to Uros Bizjak

Introduce raw_cpu_read_long() macro to slightly reduce ifdeffery in arch/x86/include/asm/percpu.h.

No functional changes intended.

Cc Ingo Molnar <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Josh Poimboeuf <[email protected]>

---
v2: Merge the patch with the rest of percpu series.

---
 arch/x86/include/asm/percpu.h | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)
arch/x86/include/asm/percpu.h +6 -8
index d6ff0db32209..3bedee1801e2 100644
@@ -59,12 +59,6 @@
 #define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
 #define __my_cpu_offset		this_cpu_read(this_cpu_off)
 
-#ifdef CONFIG_X86_64
-#define __raw_my_cpu_offset	raw_cpu_read_8(this_cpu_off);
-#else
-#define __raw_my_cpu_offset	raw_cpu_read_4(this_cpu_off);
-#endif
-
 /*
  * Compared to the generic __my_cpu_offset version, the following
  * saves one instruction and avoids clobbering a temp register.
@@ -76,7 +70,7 @@
 #ifndef BUILD_VDSO32_64
 #define arch_raw_cpu_ptr(_ptr)					\
 ({								\
-	unsigned long tcp_ptr__ = __raw_my_cpu_offset;		\
+	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
 	tcp_ptr__ += (__force unsigned long)(_ptr);		\
 	(typeof(*(_ptr)) __kernel __force *)tcp_ptr__;		\
 })
@@ -563,9 +557,13 @@ do {									\
 #define this_cpu_xchg_8(pcp, nval)		this_percpu_xchg_op(pcp, nval)
 #define this_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval)	percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
+
+#define raw_cpu_read_long(pcp)		raw_cpu_read_8(pcp)
 #else
 /* There is no generic 64 bit read stable operation for 32 bit targets. */
-#define this_cpu_read_stable_8(pcp)    ({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp)	({ BUILD_BUG(); (typeof(pcp))0; })
+
+#define raw_cpu_read_long(pcp)		raw_cpu_read_4(pcp)
 #endif
 
 #define x86_this_cpu_constant_test_bit(_nr, _var)			\
Signature · 1 line
2.44.0
merge bot Applied
#4 lore raw

[tip: x86/percpu] x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery

The following commit has been merged into the x86/percpu branch of tip:

The following commit has been merged into the x86/percpu branch of tip:

Commit-ID:     93cfa544cf9e4771def159002304a2e366cd97af
Gitweb:        https://git.kernel.org/tip/93cfa544cf9e4771def159002304a2e366cd97af
Author:        Uros Bizjak <[email protected]>
AuthorDate:    Thu, 04 Apr 2024 11:42:03 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00

x86/percpu: Introduce raw_cpu_read_long() to reduce ifdeffery

Introduce raw_cpu_read_long() macro to slightly reduce ifdeffery in <asm/percpu.h>.

No functional changes intended.

---
 arch/x86/include/asm/percpu.h | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)
arch/x86/include/asm/percpu.h +6 -8
index d6ff0db..3bedee1 100644
@@ -59,12 +59,6 @@
 #define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
 #define __my_cpu_offset		this_cpu_read(this_cpu_off)
 
-#ifdef CONFIG_X86_64
-#define __raw_my_cpu_offset	raw_cpu_read_8(this_cpu_off);
-#else
-#define __raw_my_cpu_offset	raw_cpu_read_4(this_cpu_off);
-#endif
-
 /*
  * Compared to the generic __my_cpu_offset version, the following
  * saves one instruction and avoids clobbering a temp register.
@@ -76,7 +70,7 @@
 #ifndef BUILD_VDSO32_64
 #define arch_raw_cpu_ptr(_ptr)					\
 ({								\
-	unsigned long tcp_ptr__ = __raw_my_cpu_offset;		\
+	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
 	tcp_ptr__ += (__force unsigned long)(_ptr);		\
 	(typeof(*(_ptr)) __kernel __force *)tcp_ptr__;		\
 })
@@ -563,9 +557,13 @@ do {									\
 #define this_cpu_xchg_8(pcp, nval)		this_percpu_xchg_op(pcp, nval)
 #define this_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval)	percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
+
+#define raw_cpu_read_long(pcp)		raw_cpu_read_8(pcp)
 #else
 /* There is no generic 64 bit read stable operation for 32 bit targets. */
-#define this_cpu_read_stable_8(pcp)    ({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp)	({ BUILD_BUG(); (typeof(pcp))0; })
+
+#define raw_cpu_read_long(pcp)		raw_cpu_read_4(pcp)
 #endif
 
 #define x86_this_cpu_constant_test_bit(_nr, _var)			\
merge bot Applied
#2 lore raw

[tip: x86/percpu] x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros

In reply to Uros Bizjak
The following commit has been merged into the x86/percpu branch of tip:

The following commit has been merged into the x86/percpu branch of tip:

Commit-ID:     a3f8a3a2cf0b7b3ccb51ee60d51c0b5435c7135a
Gitweb:        https://git.kernel.org/tip/a3f8a3a2cf0b7b3ccb51ee60d51c0b5435c7135a
Author:        Uros Bizjak <[email protected]>
AuthorDate:    Thu, 04 Apr 2024 11:42:02 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00

x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros

Rewrite the whole family of x86_this_cpu_test_bit() functions as macros, so standard __my_cpu_var() and raw_cpu_read() macros can be used on percpu variables. This approach considerably simplifies implementation of functions and also introduces standard checks on accessed percpu variables.

No functional changes intended.

---
 arch/um/include/asm/cpufeature.h  |  3 +--
 arch/x86/include/asm/cpufeature.h |  3 +--
 arch/x86/include/asm/percpu.h     | 54 ++++++++++++------------------
 3 files changed, 25 insertions(+), 35 deletions(-)
arch/um/include/asm/cpufeature.h +1 -2
index 66fe06d..1eb8b83 100644
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)						\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
-	 x86_this_cpu_test_bit(bit,					\
-		(unsigned long __percpu *)&cpu_info.x86_capability))
+	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
 
 /*
  * This macro is for detection of features which need kernel
arch/x86/include/asm/cpufeature.h +1 -2
index 42157dd..cd4c02d 100644
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)						\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
-	 x86_this_cpu_test_bit(bit,					\
-		(unsigned long __percpu *)&cpu_info.x86_capability))
+	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))
 
 /*
  * This macro is for detection of features which need kernel
arch/x86/include/asm/percpu.h +23 -31

67 lines not on this page · raw message

merge bot Applied
#5 lore raw
In reply to Uros Bizjak
The following commit has been merged into the x86/percpu branch of tip:

The following commit has been merged into the x86/percpu branch of tip:

Commit-ID:     4c3677c077582f8665806def3f6dd35587793c69
Gitweb:        https://git.kernel.org/tip/4c3677c077582f8665806def3f6dd35587793c69
Author:        Uros Bizjak <[email protected]>
AuthorDate:    Thu, 04 Apr 2024 11:42:01 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Sat, 06 Apr 2024 12:42:17 +02:00

x86/percpu: Fix x86_this_cpu_variable_test_bit() asm template

Fix x86_this_cpu_variable_test_bit(), which is implemented with an incorrect asm template, where argument 2 (count argument) is considered a percpu variable. However, x86_this_cpu_test_bit() is currently used exclusively with constant bit number argument, so the called x86_this_cpu_variable_test_bit() function is never instantiated.

The fix introduces named assembler operands to prevent this kind of error.

---
 arch/x86/include/asm/percpu.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
arch/x86/include/asm/percpu.h +3 -2
index 20696df..cbfbbe8 100644
@@ -586,10 +586,11 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
 {
 	bool oldbit;
 
-	asm volatile("btl "__percpu_arg(2)",%1"
+	asm volatile("btl %[nr], " __percpu_arg([var])
 			CC_SET(c)
 			: CC_OUT(c) (oldbit)
-			: "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr));
+			: [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
+			  [nr] "Ir" (nr));
 
 	return oldbit;
 }