Skip to contentSkip to conversation

Enable strict percpu address space checks

RFC PATCH Superseded Uros Bizjak 3 participants 11 messages Aug 12, 2024 – Aug 14, 2024

Subsystems (12)
Networking Memory management x86 VFS +8 more subsystems

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover Enable strict percpu address space checks

This patchset enables strict percpu address space checks via x86 named address space qualifiers. Percpu variables are declared in __seg_gs/__seg_fs named AS and kept named AS qualified until they are dereferenced via percpu accessor. This approach enables various compiler checks for cross-namespace variable assignments.

Please note that sparse doesn't know anything about __typeof_unqual__() operator, so the usage of __typeof_unqual__() breaks sparse checking.

Cc Dennis Zhou <[email protected]>
Cc Tejun Heo <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Luc Van Oostenryck <[email protected]>

Uros Bizjak (4):
percpu: Define pcpu_typeof()
percpu: Assorted fixes found by strict percpu address space checks
percpu: Repurpose __percpu tag as a named address space qualifier percpu/x86: Enable strict percpu checks via named AS qualifiers

 arch/x86/include/asm/percpu.h  | 53 ++++++++++++++++++++++++----------
 arch/x86/kernel/head64.c       |  3 +-
 drivers/base/devres.c          |  2 +-
 fs/aio.c                       |  2 +-
 include/asm-generic/percpu.h   | 41 +++++++++++++++++---------
 include/linux/cleanup.h        |  4 +--
 include/linux/compiler_types.h |  2 +-
 include/linux/part_stat.h      |  2 +-
 include/linux/percpu-defs.h    | 39 ++++++++++++++++++++-----
 include/linux/prandom.h        |  1 +
 init/Kconfig                   |  3 ++
 kernel/events/hw_breakpoint.c  |  4 +--
 kernel/locking/percpu-rwsem.c  |  2 +-
 kernel/workqueue.c             |  2 +-
 lib/percpu_counter.c           |  2 +-
 net/core/dev.c                 |  2 +-
 16 files changed, 115 insertions(+), 49 deletions(-)
Signature · 1 line
2.46.0
#1 lore raw

RFC PATCH v2 1/4 percpu: Define pcpu_typeof()

Define pcpu_typeof() to use __typeof_unqual__() as typeof operator when available, to return unqualified type of the expression.

If an arch defines __percpu variables in their own named address space, then pcpu_typeof() returns unqualified type of the expression without named address space quialifier when CONFIG_CC_HAS_TYPEOF_UNQUAL is defined.

Cc Dennis Zhou <[email protected]>
Cc Tejun Heo <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Luc Van Oostenryck <[email protected]>
---
 arch/x86/include/asm/percpu.h | 38 ++++++++++++++++++++++------------
 include/asm-generic/percpu.h  | 26 +++++++++++------------
 include/linux/part_stat.h     |  2 +-
 include/linux/percpu-defs.h   | 39 ++++++++++++++++++++++++++++-------
 init/Kconfig                  |  3 +++
 kernel/locking/percpu-rwsem.c |  2 +-
 6 files changed, 75 insertions(+), 35 deletions(-)
arch/x86/include/asm/percpu.h +25 -13
index c55a79d5feae..4ab4ad60d767 100644
@@ -73,10 +73,14 @@
 	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off);	\
 									\
 	tcp_ptr__ += (__force unsigned long)(_ptr);			\
-	(typeof(*(_ptr)) __kernel __force *)tcp_ptr__;			\
+	(pcpu_typeof(*(_ptr)) __kernel __force *)tcp_ptr__;		\
 })
 #else
-#define arch_raw_cpu_ptr(_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; })
+#define arch_raw_cpu_ptr(_ptr)						\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(*(_ptr)) __kernel __force *)0;			\
+})
 #endif
 
 #define PER_CPU_VAR(var)	%__percpu_seg:(var)__percpu_rel
@@ -172,7 +176,7 @@ do {									\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "m" (__my_cpu_var(_var)));				\
 									\
-	(typeof(_var))(unsigned long) pfo_val__;			\
+	(pcpu_typeof(_var))(unsigned long) pfo_val__;			\
 })
 
 #define __raw_cpu_write(size, qual, _var, _val)				\
@@ -180,7 +184,7 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		pcpu_typeof(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
@@ -193,7 +197,11 @@ do {									\
  * The generic per-CPU infrastrucutre is not suitable for
  * reading const-qualified variables.
  */
-#define __raw_cpu_read_const(pcp)	({ BUILD_BUG(); (typeof(pcp))0; })
+#define __raw_cpu_read_const(pcp)					\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(pcp))0;						\
+})
 
 #endif /* CONFIG_USE_X86_SEG_SUPPORT */
 
@@ -205,7 +213,7 @@ do {									\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "i" (&(_var)));					\
 									\
-	(typeof(_var))(unsigned long) pfo_val__;			\
+	(pcpu_typeof(_var))(unsigned long) pfo_val__;			\
 })
 
 #define percpu_unary_op(size, qual, op, _var)				\
@@ -219,7 +227,7 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		pcpu_typeof(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
@@ -239,7 +247,7 @@ do {									\
 				(int)(val) : 0;				\
 									\
 	if (0) {							\
-		typeof(var) pao_tmp__;					\
+		pcpu_typeof(var) pao_tmp__;				\
 		pao_tmp__ = (val);					\
 		(void)pao_tmp__;					\
 	}								\
@@ -263,7 +271,7 @@ do {									\
 		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
 		    [var] "+m" (__my_cpu_var(_var))			\
 		  : : "memory");					\
-	(typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
+	(pcpu_typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
 })
 
 /*
@@ -272,7 +280,7 @@ do {									\
  */
 #define raw_percpu_xchg_op(_var, _nval)					\
 ({									\
-	typeof(_var) pxo_old__ = raw_cpu_read(_var);			\
+	pcpu_typeof(_var) pxo_old__ = raw_cpu_read(_var);		\
 									\
 	raw_cpu_write(_var, _nval);					\
 									\
@@ -286,7 +294,7 @@ do {									\
  */
 #define this_percpu_xchg_op(_var, _nval)				\
 ({									\
-	typeof(_var) pxo_old__ = this_cpu_read(_var);			\
+	pcpu_typeof(_var) pxo_old__ = this_cpu_read(_var);		\
 									\
 	do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval));	\
 									\
@@ -309,7 +317,7 @@ do {									\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
 		  : "memory");						\
 									\
-	(typeof(_var))(unsigned long) pco_old__;			\
+	(pcpu_typeof(_var))(unsigned long) pco_old__;			\
 })
 
 #define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval)		\
@@ -568,7 +576,11 @@ do {									\
 #else /* !CONFIG_X86_64: */
 
 /* There is no generic 64-bit read stable operation for 32-bit targets. */
-#define this_cpu_read_stable_8(pcp)			({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp)					\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(pcp))0;						\
+})
 
 #define raw_cpu_read_long(pcp)				raw_cpu_read_4(pcp)
 
include/asm-generic/percpu.h +13 -13
index 94cbd50cc870..c091e09046c5 100644
@@ -74,7 +74,7 @@ do {									\
 
 #define raw_cpu_generic_add_return(pcp, val)				\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
 									\
 	*__p += val;							\
 	*__p;								\
@@ -82,8 +82,8 @@ do {									\
 
 #define raw_cpu_generic_xchg(pcp, nval)					\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
+	pcpu_typeof(pcp) __ret;						\
 	__ret = *__p;							\
 	*__p = nval;							\
 	__ret;								\
@@ -91,7 +91,7 @@ do {									\
 
 #define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg)		\
 ({									\
-	typeof(pcp) __val, __old = *(ovalp);				\
+	pcpu_typeof(pcp) __val, __old = *(ovalp);			\
 	__val = _cmpxchg(pcp, __old, nval);				\
 	if (__val != __old)						\
 		*(ovalp) = __val;					\
@@ -100,8 +100,8 @@ do {									\
 
 #define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)			\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
-	typeof(pcp) __val = *__p, ___old = *(ovalp);			\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
+	pcpu_typeof(pcp) __val = *__p, ___old = *(ovalp);		\
 	bool __ret;							\
 	if (__val == ___old) {						\
 		*__p = nval;						\
@@ -115,14 +115,14 @@ do {									\
 
 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
 ({									\
-	typeof(pcp) __old = (oval);					\
+	pcpu_typeof(pcp) __old = (oval);				\
 	raw_cpu_generic_try_cmpxchg(pcp, &__old, nval);			\
 	__old;								\
 })
 
 #define __this_cpu_generic_read_nopreempt(pcp)				\
 ({									\
-	typeof(pcp) ___ret;						\
+	pcpu_typeof(pcp) ___ret;					\
 	preempt_disable_notrace();					\
 	___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));			\
 	preempt_enable_notrace();					\
@@ -131,7 +131,7 @@ do {									\
 
 #define __this_cpu_generic_read_noirq(pcp)				\
 ({									\
-	typeof(pcp) ___ret;						\
+	pcpu_typeof(pcp) ___ret;					\
 	unsigned long ___flags;						\
 	raw_local_irq_save(___flags);					\
 	___ret = raw_cpu_generic_read(pcp);				\
@@ -141,7 +141,7 @@ do {									\
 
 #define this_cpu_generic_read(pcp)					\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	if (__native_word(pcp))						\
 		__ret = __this_cpu_generic_read_nopreempt(pcp);		\
 	else								\
@@ -160,7 +160,7 @@ do {									\
 
 #define this_cpu_generic_add_return(pcp, val)				\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_add_return(pcp, val);			\
@@ -170,7 +170,7 @@ do {									\
 
 #define this_cpu_generic_xchg(pcp, nval)				\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_xchg(pcp, nval);			\
@@ -190,7 +190,7 @@ do {									\
 
 #define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
include/linux/part_stat.h +1 -1
index ac8c44dd8237..e228e1087f0c 100644
@@ -33,7 +33,7 @@ struct disk_stats {
 
 #define part_stat_read(part, field)					\
 ({									\
-	typeof((part)->bd_stats->field) res = 0;			\
+	pcpu_typeof((part)->bd_stats->field) res = 0;			\
 	unsigned int _cpu;						\
 	for_each_possible_cpu(_cpu)					\
 		res += per_cpu_ptr((part)->bd_stats, _cpu)->field; \
include/linux/percpu-defs.h +32 -7
index 8efce7414fad..aa824c6edb82 100644
@@ -220,6 +220,21 @@ do {									\
 	(void)__vpp_verify;						\
 } while (0)
 
+/*
+ * Define percpu_typeof() to use __typeof_unqual__() as typeof
+ * operator when available, to return unqualified type of the exp.
+ *
+ * If an arch defines __percpu variables in their own named address
+ * space, then pcpu_typeof() returns unqualified type of the
+ * expression without named address space qualifier when
+ * CONFIG_CC_HAS_TYPEOF_UNQUAL is defined.
+ */
+#ifdef CONFIG_CC_HAS_TYPEOF_UNQUAL
+#define pcpu_typeof(exp) __typeof_unqual__(exp)
+#else
+#define pcpu_typeof(exp) __typeof__(exp)
+#endif
+
 #ifdef CONFIG_SMP
 
 /*
@@ -228,7 +243,10 @@ do {									\
  * pointer value.  The weird cast keeps both GCC and sparse happy.
  */
 #define SHIFT_PERCPU_PTR(__p, __offset)					\
-	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
+	uintptr_t ptr__ = (__force uintptr_t)(__p);			\
+									\
+	RELOC_HIDE((pcpu_typeof(*(__p)) __kernel __force *)(ptr__),	\
+		   (__offset))
 
 #define per_cpu_ptr(ptr, cpu)						\
 ({									\
@@ -254,13 +272,20 @@ do {									\
 
 #else	/* CONFIG_SMP */
 
-#define VERIFY_PERCPU_PTR(__p)						\
+#define PERCPU_PTR(__p)							\
 ({									\
-	__verify_pcpu_ptr(__p);						\
-	(typeof(*(__p)) __kernel __force *)(__p);			\
+	uintptr_t ptr__ = (__force uintptr_t)(__p);			\
+									\
+	(pcpu_typeof(*(__p)) __kernel __force *)(ptr__);		\
+})
+
+#define per_cpu_ptr(ptr, cpu)						\
+({									\
+	__verify_pcpu_ptr(ptr);						\
+	(void)(cpu);							\
+	PERCPU_PTR(ptr);						\
 })
 
-#define per_cpu_ptr(ptr, cpu)	({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
 #define raw_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
 #define this_cpu_ptr(ptr)	raw_cpu_ptr(ptr)
 
@@ -315,7 +340,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }
 
 #define __pcpu_size_call_return(stem, variable)				\
 ({									\
-	typeof(variable) pscr_ret__;					\
+	pcpu_typeof(variable) pscr_ret__;				\
 	__verify_pcpu_ptr(&(variable));					\
 	switch(sizeof(variable)) {					\
 	case 1: pscr_ret__ = stem##1(variable); break;			\
@@ -330,7 +355,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }
 
 #define __pcpu_size_call_return2(stem, variable, ...)			\
 ({									\
-	typeof(variable) pscr2_ret__;					\
+	pcpu_typeof(variable) pscr2_ret__;				\
 	__verify_pcpu_ptr(&(variable));					\
 	switch(sizeof(variable)) {					\
 	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\
init/Kconfig +3 -0
index 37260d17267e..a9a04d0683b6 100644
@@ -873,6 +873,9 @@ config ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
 config CC_HAS_INT128
 	def_bool !$(cc-option,$(m64-flag) -D__SIZEOF_INT128__=0) && 64BIT
 
+config CC_HAS_TYPEOF_UNQUAL
+	def_bool $(success,echo 'int foo (int a) { __typeof_unqual__(a) b = a; return b; }' | $(CC) -x c - -S -o /dev/null)
+
 config CC_IMPLICIT_FALLTHROUGH
 	string
 	default "-Wimplicit-fallthrough=5" if CC_IS_GCC && $(cc-option,-Wimplicit-fallthrough=5)
kernel/locking/percpu-rwsem.c +1 -1
index 6083883c4fe0..8345238ba428 100644
@@ -184,7 +184,7 @@ EXPORT_SYMBOL_GPL(__percpu_down_read);
 
 #define per_cpu_sum(var)						\
 ({									\
-	typeof(var) __sum = 0;						\
+	pcpu_typeof(var) __sum = 0;					\
 	int cpu;							\
 	compiletime_assert_atomic_type(__sum);				\
 	for_each_possible_cpu(cpu)					\
Signature · 1 line
2.46.0
#2 lore raw

RFC PATCH v2 2/4 percpu: Assorted fixes found by strict percpu address space checks

In reply to Uros Bizjak

Assorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.

These show effeciveness of strict percpu address space checks.

Cc Dennis Zhou <[email protected]>
Cc Tejun Heo <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Luc Van Oostenryck <[email protected]>
---
 arch/x86/kernel/head64.c      | 3 ++-
 drivers/base/devres.c         | 2 +-
 fs/aio.c                      | 2 +-
 include/linux/cleanup.h       | 4 ++--
 include/linux/prandom.h       | 1 +
 kernel/events/hw_breakpoint.c | 4 ++--
 kernel/workqueue.c            | 2 +-
 lib/percpu_counter.c          | 2 +-
 net/core/dev.c                | 2 +-
 9 files changed, 12 insertions(+), 10 deletions(-)
arch/x86/kernel/head64.c +2 -1
index a817ed0724d1..f5d6ad351cc4 100644
@@ -560,9 +560,10 @@ void early_setup_idt(void)
 void __head startup_64_setup_gdt_idt(void)
 {
 	void *handler = NULL;
+	struct desc_struct *gdt = (struct desc_struct *)(uintptr_t)init_per_cpu_var(gdt_page.gdt);
 
 	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(init_per_cpu_var(gdt_page.gdt)),
+		.address = (unsigned long)&RIP_REL_REF(*gdt),
 		.size    = GDT_SIZE - 1,
 	};
 
drivers/base/devres.c +1 -1
index a2ce0ead06a6..894f5a1e6d18 100644
@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
 	 * devm_free_pages() does.
 	 */
 	WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
-			       (__force void *)pdata));
+			       (__force void *)(uintptr_t)pdata));
 }
 EXPORT_SYMBOL_GPL(devm_free_percpu);
fs/aio.c +1 -1
index 6066f64967b3..e8920178b50f 100644
@@ -100,7 +100,7 @@ struct kioctx {
 
 	unsigned long		user_id;
 
-	struct __percpu kioctx_cpu *cpu;
+	struct kioctx_cpu __percpu *cpu;
 
 	/*
 	 * For percpu reqs_available, number of slots we move to/from global
include/linux/cleanup.h +2 -2
index d9e613803df1..f7e1158cbacf 100644
@@ -154,7 +154,7 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \
 	DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
 	static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \
-	{ return *_T; }
+	{ return (void *)(uintptr_t)*_T; }
 
 #define DEFINE_GUARD_COND(_name, _ext, _condlock) \
 	EXTEND_CLASS(_name, _ext, \
@@ -211,7 +211,7 @@ static inline void class_##_name##_destructor(class_##_name##_t *_T)	\
 									\
 static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T)	\
 {									\
-	return _T->lock;						\
+	return (void *)(uintptr_t)_T->lock;				\
 }
 
 
include/linux/prandom.h +1 -0
index f7f1e5251c67..f2ed5b72b3d6 100644
@@ -10,6 +10,7 @@
 
 #include <linux/types.h>
 #include <linux/once.h>
+#include <linux/percpu.h>
 #include <linux/random.h>
 
 struct rnd_state {
kernel/events/hw_breakpoint.c +2 -2
index 6c2cb4e4f48d..d82fe78f0658 100644
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 
 	cpu_events = alloc_percpu(typeof(*cpu_events));
 	if (!cpu_events)
-		return (void __percpu __force *)ERR_PTR(-ENOMEM);
+		return (void __percpu __force *)(uintptr_t)ERR_PTR(-ENOMEM);
 
 	cpus_read_lock();
 	for_each_online_cpu(cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 		return cpu_events;
 
 	unregister_wide_hw_breakpoint(cpu_events);
-	return (void __percpu __force *)ERR_PTR(err);
+	return (void __percpu __force *)(uintptr_t)ERR_PTR(err);
 }
 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
 
kernel/workqueue.c +1 -1
index 1745ca788ede..32729a2e93af 100644
@@ -377,7 +377,7 @@ struct workqueue_struct {
 
 	/* hot fields used during command issue, aligned to cacheline */
 	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */
-	struct pool_workqueue __percpu __rcu **cpu_pwq; /* I: per-cpu pwqs */
+	struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */
 	struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
 };
 
lib/percpu_counter.c +1 -1
index 51bc5246986d..3d0613ac7e73 100644
@@ -209,7 +209,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
 		INIT_LIST_HEAD(&fbc[i].list);
 #endif
 		fbc[i].count = amount;
-		fbc[i].counters = (void *)counters + (i * counter_size);
+		fbc[i].counters = (void __percpu *)counters + (i * counter_size);
 
 		debug_percpu_counter_activate(&fbc[i]);
 	}
net/core/dev.c +1 -1
index 751d9b70e6ad..5cad88cf029c 100644
@@ -10860,7 +10860,7 @@ noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
 			return;
 	}
 
-	field = (__force unsigned long __percpu *)((__force void *)p + offset);
+	field = (unsigned long __percpu *)(void __percpu *)(p + offset);
 	this_cpu_inc(*field);
 }
 EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
Signature · 1 line
2.46.0
#3 lore raw

Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

Quoted 6 lines · Uros Bizjak

Assorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.

These show effeciveness of strict percpu address space checks.

[snip]

drivers/base/devres.c void devm_free_percpu(struct device *dev, void __percpu *pdata) · line 1235

Quoted 9 lines
drivers/base/devres.c +1 -1
@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
 	 * devm_free_pages() does.
 	 */
 	WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
-			       (__force void *)pdata));
+			       (__force void *)(uintptr_t)pdata));
 

Since this pattern of casting appears multiple times (sometimes slightly different), I think it would be best to give a name for this operation and put it behind a macro.

This would allow both to audit the cases developers move data between address-spaces, and also make them think whether what they do makes sense.

#4 lore raw

Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

Quoted 10 lines · Nadav Amit
Quoted 7 lines · Uros Bizjak

Assorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.

These show effeciveness of strict percpu address space checks.

[snip]

Quoted 13 lines

drivers/base/devres.c void devm_free_percpu(struct device *dev, void __percpu *pdata) · line 1235

Quoted 9 lines
drivers/base/devres.c +1 -1
@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
       * devm_free_pages() does.
       */
      WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
-                            (__force void *)pdata));
+                            (__force void *)(uintptr_t)pdata));
 

Since this pattern of casting appears multiple times (sometimes slightly different), I think it would be best to give a name for this operation and put it behind a macro.

The macro would not be flexible enough to also cover const qualified (const void __percpu *)(const uintptr_t) casts, required in e.g. [1].

[1] https://lore.kernel.org/lkml/[email protected]/

Also, some casts are decorated with __force. According to sparse documentation [2], there is no need to use __force when the destination type is uintptr_t or unsigned long, but sparse seems to not be consistent with this exception, leading to spurious warnings and fixes like the one in [3].

[2] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name
[3] https://lore.kernel.org/lkml/[email protected]/

OTOH, in a full allyesconfig this pattern of casting appears maybe a dozen of times (which is a surprisingly small number).

Quoted 3 lines

This would allow both to audit the cases developers move data between address-spaces, and also make them think whether what they do makes sense.

Looking through the fixes required for allyesconfig build, the remaining couple of casts are mostly required for ERR_PTR return with __percpu return type function, like:

--cut here--

kernel/events/hw_breakpoint.c +2 -2
index 6c2cb4e4f48d..d82fe78f0658 100644
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 
     cpu_events = alloc_percpu(typeof(*cpu_events));
     if (!cpu_events)
-        return (void __percpu __force *)ERR_PTR(-ENOMEM);
+        return (void __percpu __force *)(uintptr_t)ERR_PTR(-ENOMEM);
 
     cpus_read_lock();
     for_each_online_cpu(cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
         return cpu_events;
 
     unregister_wide_hw_breakpoint(cpu_events);
-    return (void __percpu __force *)ERR_PTR(err);
+    return (void __percpu __force *)(uintptr_t)ERR_PTR(err);
 }
 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
 

--cut here--

While the casts are somehow ugly, I think that the number of different types (pcpu -> generic and generic -> pcpu casts with possible const qualifier and still needed __force sparse attribute) and low number of occurrences currently do not warrant a separate macro.

Uros.

#5 lore raw

Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

In reply to Uros Bizjak
kernel test robot noticed the following build errors:

Hi Uros,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:

[auto build test ERROR on arnd-asm-generic/master]
[also build test ERROR on tip/x86/core driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus perf-tools-next/perf-tools-next tip/perf/core perf-tools/perf-tools tj-wq/for-next linus/master v6.11-rc3 next-20240814] [cannot apply to tip/locking/core dennis-percpu/for-next acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Uros-Bizjak/percpu-Define-pcpu_typeof/20240812-200316
base:   https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git master
patch link:    https://lore.kernel.org/r/20240812115945.484051-3-ubizjak%40gmail.com
patch subject: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks
config: arm64-allnoconfig (https://download.01.org/0day-ci/archive/20240815/[email protected]/config)
compiler: aarch64-linux-gcc (GCC) 14.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240815/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from include/linux/percpu.h:12,

                    from include/linux/prandom.h:13,
                    from include/linux/random.h:153,
                    from arch/arm64/include/asm/pointer_auth.h:7,
                    from arch/arm64/include/asm/processor.h:43,
                    from include/asm-generic/qrwlock.h:18,
                    from ./arch/arm64/include/generated/asm/qrwlock.h:1,
                    from arch/arm64/include/asm/spinlock.h:9,
                    from include/linux/spinlock.h:95,
                    from include/linux/mmzone.h:8,
                    from include/linux/gfp.h:7,
                    from include/linux/slab.h:16,
                    from include/linux/resource_ext.h:11,
                    from include/linux/acpi.h:13,
                    from include/acpi/apei.h:9,
                    from include/acpi/ghes.h:5,
                    from include/linux/arm_sdei.h:8,
                    from arch/arm64/kernel/asm-offsets.c:10:
   include/linux/sched.h:1578:41: error: field 'thread' has incomplete type
    1578 |         struct thread_struct            thread;
         |                                         ^~~~~~
>> arch/arm64/include/asm/spinlock.h:21:27: error: redefinition of 'vcpu_is_preempted'

21 | #define vcpu_is_preempted vcpu_is_preempted
| ^~~~~~~~~~~~~~~~~
arch/arm64/include/asm/spinlock.h:22:20: note: in expansion of macro 'vcpu_is_preempted'
22 | static inline bool vcpu_is_preempted(int cpu)
| ^~~~~~~~~~~~~~~~~
include/linux/sched.h:2127:20: note: previous definition of 'vcpu_is_preempted' with type 'bool(int)' {aka '_Bool(int)'}
2127 | static inline bool vcpu_is_preempted(int cpu)
| ^~~~~~~~~~~~~~~~~
make[3]: *** [scripts/Makefile.build:117: arch/arm64/kernel/asm-offsets.s] Error 1
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1193: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.

vim +/vcpu_is_preempted +21 arch/arm64/include/asm/spinlock.h

872c63fbf9e153 Will Deacon 2016-09-05  13  
f5bfdc8e3947a7 Waiman Long 2020-01-13  14  /*
f5bfdc8e3947a7 Waiman Long 2020-01-13  15   * Changing this will break osq_lock() thanks to the call inside
f5bfdc8e3947a7 Waiman Long 2020-01-13  16   * smp_cond_load_relaxed().
f5bfdc8e3947a7 Waiman Long 2020-01-13  17   *
f5bfdc8e3947a7 Waiman Long 2020-01-13  18   * See:
f5bfdc8e3947a7 Waiman Long 2020-01-13  19   * https://lore.kernel.org/lkml/[email protected]
f5bfdc8e3947a7 Waiman Long 2020-01-13  20   */
345d52c184dc7d Qian Cai    2020-01-23 @21  #define vcpu_is_preempted vcpu_is_preempted
345d52c184dc7d Qian Cai    2020-01-23  22  static inline bool vcpu_is_preempted(int cpu)
345d52c184dc7d Qian Cai    2020-01-23  23  {
345d52c184dc7d Qian Cai    2020-01-23  24  	return false;
345d52c184dc7d Qian Cai    2020-01-23  25  }

f5bfdc8e3947a7 Waiman Long 2020-01-13 26

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#6 lore raw

RFC PATCH v2 3/4 percpu: Repurpose __percpu tag as a named address space qualifier

In reply to Uros Bizjak

The patch introduces per_cpu_qual define and repurposes __percpu tag as a named address space qualifier using the new define.

Arches can now conditionally define __per_cpu_qual as their named address space qualifier for percpu variables.

Cc Dennis Zhou <[email protected]>
Cc Tejun Heo <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Luc Van Oostenryck <[email protected]>
---
 include/asm-generic/percpu.h   | 15 +++++++++++++++
 include/linux/compiler_types.h |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)
include/asm-generic/percpu.h +15 -0
index c091e09046c5..5f85d7ba8e2d 100644
@@ -6,6 +6,21 @@
 #include <linux/threads.h>
 #include <linux/percpu-defs.h>
 
+/*
+ * per_cpu_qual is the qualifier for the percpu named address space.
+ *
+ * Most arches use generic named address space for percpu variables but
+ * some arches define percpu variables in different named address space
+ * (on the x86 arch, percpu variables may be declared as being relative
+ * to the %fs or %gs segments using __seg_fs or __seg_gs named address
+ * space qualifiers).
+ */
+#ifdef __per_cpu_qual
+#define per_cpu_qual __per_cpu_qual
+#else
+#define per_cpu_qual /* nothing */
+#endif
+
 #ifdef CONFIG_SMP
 
 /*
include/linux/compiler_types.h +1 -1
index f14c275950b5..4c5917b97bc3 100644
@@ -57,7 +57,7 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { }
 #  define __user	BTF_TYPE_TAG(user)
 # endif
 # define __iomem
-# define __percpu	BTF_TYPE_TAG(percpu)
+# define __percpu	per_cpu_qual BTF_TYPE_TAG(percpu)
 # define __rcu		BTF_TYPE_TAG(rcu)
 
 # define __chk_user_ptr(x)	(void)0
Signature · 1 line
2.46.0
#7 lore raw

Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

Quoted 5 lines · Uros Bizjak

+#define per_cpu_qual /* nothing */

I wouldn't do that (the “/* nothing */“ part). It might cause a mess unnecessarily is used with a comment.

#8 lore raw

Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

Quoted 9 lines · Nadav Amit
Quoted 6 lines · Uros Bizjak

+#define per_cpu_qual /* nothing */

I wouldn't do that (the “/* nothing */“ part). It might cause a mess unnecessarily is used with a comment.

I took this /* nothing */ idea from the definition of BTF_TYPE_TAG in compiler_types.h. But yes, it doesn't bring anything here so let's remove it.

Thanks,
Uros.

#9 lore raw

Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

In reply to Uros Bizjak
kernel test robot noticed the following build errors:

Hi Uros,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:

[auto build test ERROR on arnd-asm-generic/master]
[also build test ERROR on tip/x86/core driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus perf-tools-next/perf-tools-next tip/perf/core perf-tools/perf-tools tj-wq/for-next linus/master v6.11-rc3 next-20240814] [cannot apply to tip/locking/core dennis-percpu/for-next acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Uros-Bizjak/percpu-Define-pcpu_typeof/20240812-200316
base:   https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git master
patch link:    https://lore.kernel.org/r/20240812115945.484051-4-ubizjak%40gmail.com
patch subject: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier
config: loongarch-allnoconfig (https://download.01.org/0day-ci/archive/20240815/[email protected]/config)
compiler: loongarch64-linux-gcc (GCC) 14.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240815/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All error/warnings (new ones prefixed by >>):

In file included from include/linux/irqflags.h:19,

                    from include/linux/spinlock.h:59,
                    from include/linux/sched.h:2116,
                    from arch/loongarch/kernel/asm-offsets.c:8:
>> arch/loongarch/include/asm/percpu.h:71:66: error: expected ';', ',' or ')' before '*' token
      71 | static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
         |                                                                  ^
   arch/loongarch/include/asm/percpu.h:108:58: error: expected ';', ',' or ')' before '*' token
     108 | static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
         |                                                          ^
   include/linux/interrupt.h: In function 'this_cpu_ksoftirqd':
>> arch/loongarch/include/asm/percpu.h:173:33: error: implicit declaration of function '__percpu_read'; did you mean '_percpu_read'? [-Wimplicit-function-declaration]
     173 |         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
         |                                 ^~~~~~~~~~~~~
   arch/loongarch/include/asm/percpu.h:218:30: note: in expansion of macro '_percpu_read'
     218 | #define this_cpu_read_1(pcp) _percpu_read(pcp)
         |                              ^~~~~~~~~~~~
   include/linux/percpu-defs.h:346:30: note: in expansion of macro 'this_cpu_read_1'
     346 |         case 1: pscr_ret__ = stem##1(variable); break;                  \
         |                              ^~~~
   include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
     520 | #define this_cpu_read(pcp)              __pcpu_size_call_return(this_cpu_read_, pcp)
         |                                         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
     623 |         return this_cpu_read(ksoftirqd);
         |                ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
     173 |         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
         |                    ^
   arch/loongarch/include/asm/percpu.h:218:30: note: in expansion of macro '_percpu_read'
     218 | #define this_cpu_read_1(pcp) _percpu_read(pcp)
         |                              ^~~~~~~~~~~~
   include/linux/percpu-defs.h:346:30: note: in expansion of macro 'this_cpu_read_1'
     346 |         case 1: pscr_ret__ = stem##1(variable); break;                  \
         |                              ^~~~
   include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
     520 | #define this_cpu_read(pcp)              __pcpu_size_call_return(this_cpu_read_, pcp)
         |                                         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
     623 |         return this_cpu_read(ksoftirqd);
         |                ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
     173 |         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
         |                    ^
   arch/loongarch/include/asm/percpu.h:219:30: note: in expansion of macro '_percpu_read'
     219 | #define this_cpu_read_2(pcp) _percpu_read(pcp)
         |                              ^~~~~~~~~~~~
   include/linux/percpu-defs.h:347:30: note: in expansion of macro 'this_cpu_read_2'
     347 |         case 2: pscr_ret__ = stem##2(variable); break;                  \
         |                              ^~~~
   include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
     520 | #define this_cpu_read(pcp)              __pcpu_size_call_return(this_cpu_read_, pcp)
         |                                         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
     623 |         return this_cpu_read(ksoftirqd);
         |                ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
     173 |         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
         |                    ^
   arch/loongarch/include/asm/percpu.h:220:30: note: in expansion of macro '_percpu_read'
     220 | #define this_cpu_read_4(pcp) _percpu_read(pcp)
         |                              ^~~~~~~~~~~~
   include/linux/percpu-defs.h:348:30: note: in expansion of macro 'this_cpu_read_4'
     348 |         case 4: pscr_ret__ = stem##4(variable); break;                  \
         |                              ^~~~
   include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
     520 | #define this_cpu_read(pcp)              __pcpu_size_call_return(this_cpu_read_, pcp)
         |                                         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
     623 |         return this_cpu_read(ksoftirqd);
         |                ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
     173 |         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
         |                    ^
   arch/loongarch/include/asm/percpu.h:221:30: note: in expansion of macro '_percpu_read'
     221 | #define this_cpu_read_8(pcp) _percpu_read(pcp)
         |                              ^~~~~~~~~~~~
   include/linux/percpu-defs.h:349:30: note: in expansion of macro 'this_cpu_read_8'
     349 |         case 8: pscr_ret__ = stem##8(variable); break;                  \
         |                              ^~~~
   include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
     520 | #define this_cpu_read(pcp)              __pcpu_size_call_return(this_cpu_read_, pcp)
         |                                         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
     623 |         return this_cpu_read(ksoftirqd);
         |                ^~~~~~~~~~~~~

make[3]: *** [scripts/Makefile.build:117: arch/loongarch/kernel/asm-offsets.s] Error 1 make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1193: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.

vim +71 arch/loongarch/include/asm/percpu.h

46859ac8af52ae Huacai Chen 2022-05-31 70

ee2daf7102f420 Huacai Chen 2023-11-21  @71  static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
46859ac8af52ae Huacai Chen 2022-05-31   72  {
46859ac8af52ae Huacai Chen 2022-05-31   73  	unsigned long ret;
46859ac8af52ae Huacai Chen 2022-05-31   74  
46859ac8af52ae Huacai Chen 2022-05-31   75  	switch (size) {
46859ac8af52ae Huacai Chen 2022-05-31   76  	case 1:
46859ac8af52ae Huacai Chen 2022-05-31   77  		__asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31   78  		: [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31   79  		: [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31   80  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31   81  		break;
46859ac8af52ae Huacai Chen 2022-05-31   82  	case 2:
46859ac8af52ae Huacai Chen 2022-05-31   83  		__asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31   84  		: [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31   85  		: [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31   86  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31   87  		break;
46859ac8af52ae Huacai Chen 2022-05-31   88  	case 4:
46859ac8af52ae Huacai Chen 2022-05-31   89  		__asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31   90  		: [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31   91  		: [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31   92  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31   93  		break;
46859ac8af52ae Huacai Chen 2022-05-31   94  	case 8:
46859ac8af52ae Huacai Chen 2022-05-31   95  		__asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31   96  		: [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31   97  		: [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31   98  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31   99  		break;
46859ac8af52ae Huacai Chen 2022-05-31  100  	default:
46859ac8af52ae Huacai Chen 2022-05-31  101  		ret = 0;
46859ac8af52ae Huacai Chen 2022-05-31  102  		BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31  103  	}
46859ac8af52ae Huacai Chen 2022-05-31  104  
46859ac8af52ae Huacai Chen 2022-05-31  105  	return ret;
46859ac8af52ae Huacai Chen 2022-05-31  106  }
46859ac8af52ae Huacai Chen 2022-05-31  107  
ee2daf7102f420 Huacai Chen 2023-11-21  108  static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
46859ac8af52ae Huacai Chen 2022-05-31  109  {
46859ac8af52ae Huacai Chen 2022-05-31  110  	switch (size) {
46859ac8af52ae Huacai Chen 2022-05-31  111  	case 1:
46859ac8af52ae Huacai Chen 2022-05-31  112  		__asm__ __volatile__("stx.b %[val], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31  113  		:
46859ac8af52ae Huacai Chen 2022-05-31  114  		: [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31  115  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31  116  		break;
46859ac8af52ae Huacai Chen 2022-05-31  117  	case 2:
46859ac8af52ae Huacai Chen 2022-05-31  118  		__asm__ __volatile__("stx.h %[val], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31  119  		:
46859ac8af52ae Huacai Chen 2022-05-31  120  		: [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31  121  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31  122  		break;
46859ac8af52ae Huacai Chen 2022-05-31  123  	case 4:
46859ac8af52ae Huacai Chen 2022-05-31  124  		__asm__ __volatile__("stx.w %[val], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31  125  		:
46859ac8af52ae Huacai Chen 2022-05-31  126  		: [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31  127  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31  128  		break;
46859ac8af52ae Huacai Chen 2022-05-31  129  	case 8:
46859ac8af52ae Huacai Chen 2022-05-31  130  		__asm__ __volatile__("stx.d %[val], $r21, %[ptr]	\n"
46859ac8af52ae Huacai Chen 2022-05-31  131  		:
46859ac8af52ae Huacai Chen 2022-05-31  132  		: [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31  133  		: "memory");
46859ac8af52ae Huacai Chen 2022-05-31  134  		break;
46859ac8af52ae Huacai Chen 2022-05-31  135  	default:
46859ac8af52ae Huacai Chen 2022-05-31  136  		BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31  137  	}
46859ac8af52ae Huacai Chen 2022-05-31  138  }
46859ac8af52ae Huacai Chen 2022-05-31  139  
ee2daf7102f420 Huacai Chen 2023-11-21  140  static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
46859ac8af52ae Huacai Chen 2022-05-31  141  {
46859ac8af52ae Huacai Chen 2022-05-31  142  	switch (size) {
720dc7ab252bbd Huacai Chen 2022-08-25  143  	case 1:
720dc7ab252bbd Huacai Chen 2022-08-25  144  	case 2:
720dc7ab252bbd Huacai Chen 2022-08-25  145  		return __xchg_small((volatile void *)ptr, val, size);
720dc7ab252bbd Huacai Chen 2022-08-25  146  
46859ac8af52ae Huacai Chen 2022-05-31  147  	case 4:
46859ac8af52ae Huacai Chen 2022-05-31  148  		return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
46859ac8af52ae Huacai Chen 2022-05-31  149  
46859ac8af52ae Huacai Chen 2022-05-31  150  	case 8:
46859ac8af52ae Huacai Chen 2022-05-31  151  		return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
46859ac8af52ae Huacai Chen 2022-05-31  152  
46859ac8af52ae Huacai Chen 2022-05-31  153  	default:
46859ac8af52ae Huacai Chen 2022-05-31  154  		BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31  155  	}
46859ac8af52ae Huacai Chen 2022-05-31  156  
46859ac8af52ae Huacai Chen 2022-05-31  157  	return 0;
46859ac8af52ae Huacai Chen 2022-05-31  158  }
46859ac8af52ae Huacai Chen 2022-05-31  159  
46859ac8af52ae Huacai Chen 2022-05-31  160  /* this_cpu_cmpxchg */
46859ac8af52ae Huacai Chen 2022-05-31  161  #define _protect_cmpxchg_local(pcp, o, n)			\
46859ac8af52ae Huacai Chen 2022-05-31  162  ({								\
46859ac8af52ae Huacai Chen 2022-05-31  163  	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
46859ac8af52ae Huacai Chen 2022-05-31  164  	preempt_disable_notrace();				\
46859ac8af52ae Huacai Chen 2022-05-31  165  	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
46859ac8af52ae Huacai Chen 2022-05-31  166  	preempt_enable_notrace();				\
46859ac8af52ae Huacai Chen 2022-05-31  167  	__ret;							\
46859ac8af52ae Huacai Chen 2022-05-31  168  })
46859ac8af52ae Huacai Chen 2022-05-31  169  
46859ac8af52ae Huacai Chen 2022-05-31  170  #define _percpu_read(pcp)						\
46859ac8af52ae Huacai Chen 2022-05-31  171  ({									\
46859ac8af52ae Huacai Chen 2022-05-31  172  	typeof(pcp) __retval;						\
46859ac8af52ae Huacai Chen 2022-05-31 @173  	__retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));	\
46859ac8af52ae Huacai Chen 2022-05-31  174  	__retval;							\
46859ac8af52ae Huacai Chen 2022-05-31  175  })

46859ac8af52ae Huacai Chen 2022-05-31 176

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#10 lore raw

RFC PATCH v2 4/4 percpu/x86: Enable strict percpu checks via named AS qualifiers

In reply to Uros Bizjak

Declare percpu variables in __seg_gs/__seg_fs named AS and keep them named AS qualified until they are dereferenced with percpu accessor. This approach enables various compiler check for cross-namespace variable assignments.

Cc Dennis Zhou <[email protected]>
Cc Tejun Heo <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Luc Van Oostenryck <[email protected]>
---
 arch/x86/include/asm/percpu.h | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
arch/x86/include/asm/percpu.h +12 -3
index 4ab4ad60d767..b54119398c4b 100644
@@ -95,9 +95,18 @@
 
 #endif /* CONFIG_SMP */
 
-#define __my_cpu_type(var)	typeof(var) __percpu_seg_override
-#define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
-#define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
+#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(CONFIG_CC_HAS_TYPEOF_UNQUAL)
+# define __my_cpu_type(var)	typeof(var)
+# define __my_cpu_ptr(ptr)	(ptr)
+# define __my_cpu_var(var)	(var)
+
+# define __per_cpu_qual		__percpu_seg_override
+#else
+# define __my_cpu_type(var)	typeof(var) __percpu_seg_override
+# define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
+# define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
+#endif
+
 #define __percpu_arg(x)		__percpu_prefix "%" #x
 #define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
 
Signature · 1 line
2.46.0