Enable strict percpu address space checks
Subsystems (12)
v2 cover Enable strict percpu address space checks
This patchset enables strict percpu address space checks via x86 named address space qualifiers. Percpu variables are declared in __seg_gs/__seg_fs named AS and kept named AS qualified until they are dereferenced via percpu accessor. This approach enables various compiler checks for cross-namespace variable assignments.
Please note that sparse doesn't know anything about __typeof_unqual__() operator, so the usage of __typeof_unqual__() breaks sparse checking.
Uros Bizjak (4):
percpu: Define pcpu_typeof()
percpu: Assorted fixes found by strict percpu address space checks
percpu: Repurpose __percpu tag as a named address space qualifier percpu/x86: Enable strict percpu checks via named AS qualifiers
arch/x86/include/asm/percpu.h | 53 ++++++++++++++++++++++++---------- arch/x86/kernel/head64.c | 3 +- drivers/base/devres.c | 2 +- fs/aio.c | 2 +- include/asm-generic/percpu.h | 41 +++++++++++++++++--------- include/linux/cleanup.h | 4 +-- include/linux/compiler_types.h | 2 +- include/linux/part_stat.h | 2 +- include/linux/percpu-defs.h | 39 ++++++++++++++++++++----- include/linux/prandom.h | 1 + init/Kconfig | 3 ++ kernel/events/hw_breakpoint.c | 4 +-- kernel/locking/percpu-rwsem.c | 2 +- kernel/workqueue.c | 2 +- lib/percpu_counter.c | 2 +- net/core/dev.c | 2 +- 16 files changed, 115 insertions(+), 49 deletions(-)
Signature · 1 line
2.46.0
RFC PATCH v2 1/4 percpu: Define pcpu_typeof()
Define pcpu_typeof() to use __typeof_unqual__() as typeof operator when available, to return unqualified type of the expression.
If an arch defines __percpu variables in their own named address space, then pcpu_typeof() returns unqualified type of the expression without named address space quialifier when CONFIG_CC_HAS_TYPEOF_UNQUAL is defined.
--- arch/x86/include/asm/percpu.h | 38 ++++++++++++++++++++++------------ include/asm-generic/percpu.h | 26 +++++++++++------------ include/linux/part_stat.h | 2 +- include/linux/percpu-defs.h | 39 ++++++++++++++++++++++++++++------- init/Kconfig | 3 +++ kernel/locking/percpu-rwsem.c | 2 +- 6 files changed, 75 insertions(+), 35 deletions(-)
arch/x86/include/asm/percpu.h +25 -13
@@ -73,10 +73,14 @@unsigned long tcp_ptr__ =raw_cpu_read_long (this_cpu_off); \ \ tcp_ptr__ += (__force unsigned long )(_ptr); \ - (typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \ + (pcpu_typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \ })#else -#define arch_raw_cpu_ptr (_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; }) +#define arch_raw_cpu_ptr (_ptr) \ +({ \ + BUILD_BUG(); \ + (pcpu_typeof(*(_ptr)) __kernel __force *)0; \ +})#endif #define PER_CPU_VAR (var) %__percpu_seg:(var)__percpu_rel
@@ -172,7 +176,7 @@ do { \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "m" (__my_cpu_var(_var))); \ \ - (typeof(_var))(unsigned long) pfo_val__; \ + (pcpu_typeof(_var))(unsigned long) pfo_val__; \ })#define __raw_cpu_write (size, qual, _var, _val) \
@@ -180,7 +184,7 @@ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ \ if (0) { \ - typeof(_var) pto_tmp__; \ + pcpu_typeof(_var) pto_tmp__; \ pto_tmp__ = (_val); \ (void)pto_tmp__; \ } \
@@ -193,7 +197,11 @@ do { \* The generic per-CPU infrastrucutre is not suitable for * reading const-qualified variables. */ -#define __raw_cpu_read_const (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) +#define __raw_cpu_read_const (pcp) \ +({ \ + BUILD_BUG(); \ + (pcpu_typeof(pcp))0; \ +})#endif /* CONFIG_USE_X86_SEG_SUPPORT */
@@ -205,7 +213,7 @@ do { \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "i" (&(_var))); \ \ - (typeof(_var))(unsigned long) pfo_val__; \ + (pcpu_typeof(_var))(unsigned long) pfo_val__; \ })#define percpu_unary_op (size, qual, op, _var) \
@@ -219,7 +227,7 @@ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ \ if (0) { \ - typeof(_var) pto_tmp__; \ + pcpu_typeof(_var) pto_tmp__; \ pto_tmp__ = (_val); \ (void)pto_tmp__; \ } \
@@ -239,7 +247,7 @@ do { \ (int)(val) : 0; \ \if (0 ) { \ -typeof (var) pao_tmp__; \ +pcpu_typeof (var) pao_tmp__; \ pao_tmp__ = (val); \ (void )pao_tmp__; \ } \
@@ -263,7 +271,7 @@ do { \ : [tmp] __pcpu_reg_##size("+", paro_tmp__), \ [var] "+m" (__my_cpu_var(_var)) \ : : "memory"); \ - (typeof(_var))(unsigned long) (paro_tmp__ + _val); \ + (pcpu_typeof(_var))(unsigned long) (paro_tmp__ + _val); \ })/*
@@ -272,7 +280,7 @@ do { \*/ #define raw_percpu_xchg_op (_var, _nval) \ ({ \ - typeof(_var) pxo_old__ = raw_cpu_read(_var); \ + pcpu_typeof(_var) pxo_old__ = raw_cpu_read(_var); \ \ raw_cpu_write(_var, _nval); \ \
@@ -286,7 +294,7 @@ do { \*/ #define this_percpu_xchg_op (_var, _nval) \ ({ \ - typeof(_var) pxo_old__ = this_cpu_read(_var); \ + pcpu_typeof(_var) pxo_old__ = this_cpu_read(_var); \ \ do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval)); \ \
@@ -309,7 +317,7 @@ do { \ : [nval] __pcpu_reg_##size(, pco_new__) \ : "memory"); \ \ - (typeof(_var))(unsigned long) pco_old__; \ + (pcpu_typeof(_var))(unsigned long) pco_old__; \ })#define percpu_try_cmpxchg_op (size, qual, _var, _ovalp, _nval) \
@@ -568,7 +576,11 @@ do { \#else /* !CONFIG_X86_64: */ /* There is no generic 64-bit read stable operation for 32-bit targets. */ -#define this_cpu_read_stable_8 (pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) +#define this_cpu_read_stable_8 (pcp) \ +({ \ + BUILD_BUG(); \ + (pcpu_typeof(pcp))0; \ +})#define raw_cpu_read_long (pcp) raw_cpu_read_4(pcp)
include/asm-generic/percpu.h +13 -13
@@ -74,7 +74,7 @@ do { \#define raw_cpu_generic_add_return (pcp, val) \ ({ \ - typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ \ *__p += val; \ *__p; \
@@ -82,8 +82,8 @@ do { \#define raw_cpu_generic_xchg (pcp, nval) \ ({ \ - typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ + pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + pcpu_typeof(pcp) __ret; \ __ret = *__p; \ *__p = nval; \ __ret; \
@@ -91,7 +91,7 @@ do { \#define __cpu_fallback_try_cmpxchg (pcp, ovalp, nval, _cmpxchg) \ ({ \ - typeof(pcp) __val, __old = *(ovalp); \ + pcpu_typeof(pcp) __val, __old = *(ovalp); \ __val = _cmpxchg(pcp, __old, nval); \ if (__val != __old) \ *(ovalp) = __val; \
@@ -100,8 +100,8 @@ do { \#define raw_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ ({ \ - typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __val = *__p, ___old = *(ovalp); \ + pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + pcpu_typeof(pcp) __val = *__p, ___old = *(ovalp); \ bool __ret; \ if (__val == ___old) { \ *__p = nval; \
@@ -115,14 +115,14 @@ do { \#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ - typeof(pcp) __old = (oval); \ + pcpu_typeof(pcp) __old = (oval); \ raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ __old; \ })#define __this_cpu_generic_read_nopreempt (pcp) \ ({ \ - typeof(pcp) ___ret; \ + pcpu_typeof(pcp) ___ret; \ preempt_disable_notrace(); \ ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \ preempt_enable_notrace(); \
@@ -131,7 +131,7 @@ do { \#define __this_cpu_generic_read_noirq (pcp) \ ({ \ - typeof(pcp) ___ret; \ + pcpu_typeof(pcp) ___ret; \ unsigned long ___flags; \ raw_local_irq_save(___flags); \ ___ret = raw_cpu_generic_read(pcp); \
@@ -141,7 +141,7 @@ do { \#define this_cpu_generic_read (pcp) \ ({ \ - typeof(pcp) __ret; \ + pcpu_typeof(pcp) __ret; \ if (__native_word(pcp)) \ __ret = __this_cpu_generic_read_nopreempt(pcp); \ else \
@@ -160,7 +160,7 @@ do { \#define this_cpu_generic_add_return (pcp, val) \ ({ \ - typeof(pcp) __ret; \ + pcpu_typeof(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ __ret = raw_cpu_generic_add_return(pcp, val); \
@@ -170,7 +170,7 @@ do { \#define this_cpu_generic_xchg (pcp, nval) \ ({ \ - typeof(pcp) __ret; \ + pcpu_typeof(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ __ret = raw_cpu_generic_xchg(pcp, nval); \
@@ -190,7 +190,7 @@ do { \#define this_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ - typeof(pcp) __ret; \ + pcpu_typeof(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
include/linux/part_stat.h +1 -1
@@ -33,7 +33,7 @@ struct disk_stats {#define part_stat_read (part, field) \ ({ \ - typeof((part)->bd_stats->field) res = 0; \ + pcpu_typeof((part)->bd_stats->field) res = 0; \ unsigned int _cpu; \ for_each_possible_cpu(_cpu) \ res += per_cpu_ptr((part)->bd_stats, _cpu)->field; \
include/linux/percpu-defs.h +32 -7
@@ -220,6 +220,21 @@ do { \ (void )__vpp_verify; \ }while (0 ) +/* +* Define percpu_typeof() to use __typeof_unqual__() as typeof +* operator when available, to return unqualified type of the exp. +* +* If an arch defines __percpu variables in their own named address +* space, then pcpu_typeof() returns unqualified type of the +* expression without named address space qualifier when +* CONFIG_CC_HAS_TYPEOF_UNQUAL is defined. + */ +#ifdef CONFIG_CC_HAS_TYPEOF_UNQUAL +#define pcpu_typeof (exp) __typeof_unqual__(exp) +#else +#define pcpu_typeof (exp) __typeof__(exp) +#endif +#ifdef CONFIG_SMP /*
@@ -228,7 +243,10 @@ do { \* pointer value. The weird cast keeps both GCC and sparse happy. */ #define SHIFT_PERCPU_PTR (__p, __offset) \ - RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)) + uintptr_t ptr__ = (__force uintptr_t)(__p); \ + \ + RELOC_HIDE((pcpu_typeof(*(__p)) __kernel __force *)(ptr__), \ + (__offset))#define per_cpu_ptr (ptr, cpu) \ ({ \
@@ -254,13 +272,20 @@ do { \#else /* CONFIG_SMP */ -#define VERIFY_PERCPU_PTR (__p) \ +#define PERCPU_PTR (__p) \ ({ \ - __verify_pcpu_ptr(__p); \ - (typeof(*(__p)) __kernel __force *)(__p); \ + uintptr_t ptr__ = (__force uintptr_t)(__p); \ + \ + (pcpu_typeof(*(__p)) __kernel __force *)(ptr__); \ +}) + +#define per_cpu_ptr (ptr, cpu) \ +({ \ + __verify_pcpu_ptr(ptr); \ + (void)(cpu); \ + PERCPU_PTR(ptr); \ }) -#define per_cpu_ptr (ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })#define raw_cpu_ptr (ptr) per_cpu_ptr(ptr, 0)#define this_cpu_ptr (ptr) raw_cpu_ptr(ptr)
@@ -315,7 +340,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }#define __pcpu_size_call_return (stem, variable) \ ({ \ - typeof(variable) pscr_ret__; \ + pcpu_typeof(variable) pscr_ret__; \ __verify_pcpu_ptr(&(variable)); \ switch(sizeof(variable)) { \ case 1: pscr_ret__ = stem##1(variable); break; \
@@ -330,7 +355,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }#define __pcpu_size_call_return2 (stem, variable, ...) \ ({ \ - typeof(variable) pscr2_ret__; \ + pcpu_typeof(variable) pscr2_ret__; \ __verify_pcpu_ptr(&(variable)); \ switch(sizeof(variable)) { \ case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
init/Kconfig +3 -0
@@ -873,6 +873,9 @@ config ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH config CC_HAS_INT128 def_bool !$(cc-option,$(m64-flag) -D__SIZEOF_INT128__=0) && 64BIT +config CC_HAS_TYPEOF_UNQUAL + def_bool $(success,echo 'int foo (int a) { __typeof_unqual__(a) b = a; return b; }' | $(CC) -x c - -S -o /dev/null) + config CC_IMPLICIT_FALLTHROUGH string default "-Wimplicit-fallthrough=5" if CC_IS_GCC && $(cc-option,-Wimplicit-fallthrough=5)
kernel/locking/percpu-rwsem.c +1 -1
@@ -184,7 +184,7 @@ EXPORT_SYMBOL_GPL(__percpu_down_read);#define per_cpu_sum (var) \ ({ \ - typeof(var) __sum = 0; \ + pcpu_typeof(var) __sum = 0; \ int cpu; \ compiletime_assert_atomic_type(__sum); \ for_each_possible_cpu(cpu) \
Signature · 1 line
2.46.0
RFC PATCH v2 2/4 percpu: Assorted fixes found by strict percpu address space checks
In reply to Uros BizjakAssorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.
These show effeciveness of strict percpu address space checks.
--- arch/x86/kernel/head64.c | 3 ++- drivers/base/devres.c | 2 +- fs/aio.c | 2 +- include/linux/cleanup.h | 4 ++-- include/linux/prandom.h | 1 + kernel/events/hw_breakpoint.c | 4 ++-- kernel/workqueue.c | 2 +- lib/percpu_counter.c | 2 +- net/core/dev.c | 2 +- 9 files changed, 12 insertions(+), 10 deletions(-)
arch/x86/kernel/head64.c +2 -1
@@ -560,9 +560,10 @@ void early_setup_idt(void) void __head startup_64_setup_gdt_idt(void) {void *handler =NULL ; +struct desc_struct *gdt = (struct desc_struct *)(uintptr_t )init_per_cpu_var (gdt_page.gdt);struct desc_ptr startup_gdt_descr = { - .address = (unsigned long )&RIP_REL_REF (init_per_cpu_var (gdt_page.gdt)), + .address = (unsigned long )&RIP_REL_REF (*gdt), .size =GDT_SIZE -1 , };
drivers/base/devres.c +1 -1
@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)* devm_free_pages() does. */ WARN_ON (devres_release (dev, devm_percpu_release, devm_percpu_match, - (__force void *)pdata)); + (__force void *)(uintptr_t )pdata)); }EXPORT_SYMBOL_GPL (devm_free_percpu);
fs/aio.c +1 -1
@@ -100,7 +100,7 @@ struct kioctx {unsigned long user_id; -struct __percpu kioctx_cpu *cpu; +struct kioctx_cpu __percpu *cpu;/* * For percpu reqs_available, number of slots we move to/from global
include/linux/cleanup.h +2 -2
@@ -154,7 +154,7 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \#define DEFINE_GUARD (_name, _type, _lock, _unlock) \ DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \ - { return *_T; } + { return (void *)(uintptr_t)*_T; }#define DEFINE_GUARD_COND (_name, _ext, _condlock) \ EXTEND_CLASS(_name, _ext, \
@@ -211,7 +211,7 @@ static inline void class_##_name##_destructor(class_##_name##_t *_T) \ \ static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T) \ { \ -return _T->lock; \ +return (void *)(uintptr_t )_T->lock; \ }
include/linux/prandom.h +1 -0
@@ -10,6 +10,7 @@#include <linux/types.h> #include <linux/once.h> +#include <linux/percpu.h> #include <linux/random.h> struct rnd_state {
kernel/events/hw_breakpoint.c +2 -2
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, cpu_events =alloc_percpu (typeof (*cpu_events));if (!cpu_events) -return (void __percpu __force *)ERR_PTR (-ENOMEM ); +return (void __percpu __force *)(uintptr_t )ERR_PTR (-ENOMEM );cpus_read_lock ();for_each_online_cpu (cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,return cpu_events;unregister_wide_hw_breakpoint (cpu_events); -return (void __percpu __force *)ERR_PTR (err); +return (void __percpu __force *)(uintptr_t )ERR_PTR (err); }EXPORT_SYMBOL_GPL (register_wide_hw_breakpoint);
kernel/workqueue.c +1 -1
@@ -377,7 +377,7 @@ struct workqueue_struct {/* hot fields used during command issue, aligned to cacheline */ unsigned int flags____cacheline_aligned ;/* WQ: WQ_* flags */ -struct pool_workqueue __percpu __rcu **cpu_pwq;/* I: per-cpu pwqs */ +struct pool_workqueue __rcu *__percpu *cpu_pwq;/* I: per-cpu pwqs */ struct wq_node_nr_active *node_nr_active[];/* I: per-node nr_active */ };
lib/percpu_counter.c +1 -1
@@ -209,7 +209,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,INIT_LIST_HEAD (&fbc[i].list);#endif fbc[i].count = amount; - fbc[i].counters = (void *)counters + (i * counter_size); + fbc[i].counters = (void __percpu *)counters + (i * counter_size);debug_percpu_counter_activate (&fbc[i]); }
net/core/dev.c +1 -1
@@ -10860,7 +10860,7 @@ noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)return ; } - field = (__force unsigned long __percpu *)((__force void *)p + offset); + field = (unsigned long __percpu *)(void __percpu *)(p + offset);this_cpu_inc (*field); }EXPORT_SYMBOL_GPL (netdev_core_stats_inc);
Signature · 1 line
2.46.0
Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks
Quoted 6 lines · Uros Bizjak
Assorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.
These show effeciveness of strict percpu address space checks.
[snip]
drivers/base/devres.c void devm_free_percpu(struct device *dev, void __percpu *pdata) · line 1235
Quoted 9 lines
drivers/base/devres.c +1
-1@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)* devm_free_pages() does. */ WARN_ON (devres_release (dev, devm_percpu_release, devm_percpu_match, - (__force void *)pdata)); + (__force void *)(uintptr_t )pdata));
Since this pattern of casting appears multiple times (sometimes slightly different), I think it would be best to give a name for this operation and put it behind a macro.
This would allow both to audit the cases developers move data between address-spaces, and also make them think whether what they do makes sense.
Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks
Quoted 10 lines · Nadav Amit
Quoted 7 lines · Uros Bizjak
Assorted fixes to prevent defconfig build failures when strict percpu address space checks will be enabled.
These show effeciveness of strict percpu address space checks.
[snip]
Quoted 13 lines
drivers/base/devres.c void devm_free_percpu(struct device *dev, void __percpu *pdata) · line 1235
Quoted 9 lines
drivers/base/devres.c +1
-1@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)* devm_free_pages() does. */ WARN_ON (devres_release (dev, devm_percpu_release, devm_percpu_match, - (__force void *)pdata)); + (__force void *)(uintptr_t )pdata));Since this pattern of casting appears multiple times (sometimes slightly different), I think it would be best to give a name for this operation and put it behind a macro.
The macro would not be flexible enough to also cover const qualified (const void __percpu *)(const uintptr_t) casts, required in e.g. [1].
[1] https://lore.kernel.org/lkml/[email protected]/
Also, some casts are decorated with __force. According to sparse documentation [2], there is no need to use __force when the destination type is uintptr_t or unsigned long, but sparse seems to not be consistent with this exception, leading to spurious warnings and fixes like the one in [3].
[2] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name
[3] https://lore.kernel.org/lkml/[email protected]/
OTOH, in a full allyesconfig this pattern of casting appears maybe a dozen of times (which is a surprisingly small number).
Quoted 3 lines
This would allow both to audit the cases developers move data between address-spaces, and also make them think whether what they do makes sense.
Looking through the fixes required for allyesconfig build, the remaining couple of casts are mostly required for ERR_PTR return with __percpu return type function, like:
--cut here--
kernel/events/hw_breakpoint.c +2 -2
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, cpu_events =alloc_percpu (typeof (*cpu_events));if (!cpu_events) -return (void __percpu __force *)ERR_PTR (-ENOMEM ); +return (void __percpu __force *)(uintptr_t )ERR_PTR (-ENOMEM );cpus_read_lock ();for_each_online_cpu (cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,return cpu_events;unregister_wide_hw_breakpoint (cpu_events); -return (void __percpu __force *)ERR_PTR (err); +return (void __percpu __force *)(uintptr_t )ERR_PTR (err); }EXPORT_SYMBOL_GPL (register_wide_hw_breakpoint);
--cut here--
While the casts are somehow ugly, I think that the number of different types (pcpu -> generic and generic -> pcpu casts with possible const qualifier and still needed __force sparse attribute) and low number of occurrences currently do not warrant a separate macro.
Uros.
kernel test robot
build botRe: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks
In reply to Uros Bizjakkernel test robot noticed the following build errors:
Hi Uros,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:
[auto build test ERROR on arnd-asm-generic/master]
[also build test ERROR on tip/x86/core driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus perf-tools-next/perf-tools-next tip/perf/core perf-tools/perf-tools tj-wq/for-next linus/master v6.11-rc3 next-20240814] [cannot apply to tip/locking/core dennis-percpu/for-next acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Uros-Bizjak/percpu-Define-pcpu_typeof/20240812-200316 base: https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git master patch link: https://lore.kernel.org/r/20240812115945.484051-3-ubizjak%40gmail.com patch subject: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks config: arm64-allnoconfig (https://download.01.org/0day-ci/archive/20240815/[email protected]/config) compiler: aarch64-linux-gcc (GCC) 14.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240815/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from include/linux/percpu.h:12,
from include/linux/prandom.h:13,
from include/linux/random.h:153,
from arch/arm64/include/asm/pointer_auth.h:7,
from arch/arm64/include/asm/processor.h:43,
from include/asm-generic/qrwlock.h:18,
from ./arch/arm64/include/generated/asm/qrwlock.h:1,
from arch/arm64/include/asm/spinlock.h:9,
from include/linux/spinlock.h:95,
from include/linux/mmzone.h:8,
from include/linux/gfp.h:7,
from include/linux/slab.h:16,
from include/linux/resource_ext.h:11,
from include/linux/acpi.h:13,
from include/acpi/apei.h:9,
from include/acpi/ghes.h:5,
from include/linux/arm_sdei.h:8,
from arch/arm64/kernel/asm-offsets.c:10:
include/linux/sched.h:1578:41: error: field 'thread' has incomplete type
1578 | struct thread_struct thread;
| ^~~~~~
>> arch/arm64/include/asm/spinlock.h:21:27: error: redefinition of 'vcpu_is_preempted' 21 | #define vcpu_is_preempted vcpu_is_preempted
| ^~~~~~~~~~~~~~~~~
arch/arm64/include/asm/spinlock.h:22:20: note: in expansion of macro 'vcpu_is_preempted'
22 | static inline bool vcpu_is_preempted(int cpu)
| ^~~~~~~~~~~~~~~~~
include/linux/sched.h:2127:20: note: previous definition of 'vcpu_is_preempted' with type 'bool(int)' {aka '_Bool(int)'}
2127 | static inline bool vcpu_is_preempted(int cpu)
| ^~~~~~~~~~~~~~~~~
make[3]: *** [scripts/Makefile.build:117: arch/arm64/kernel/asm-offsets.s] Error 1
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1193: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +/vcpu_is_preempted +21 arch/arm64/include/asm/spinlock.h
872c63fbf9e153 Will Deacon 2016-09-05 13 f5bfdc8e3947a7 Waiman Long 2020-01-13 14 /* f5bfdc8e3947a7 Waiman Long 2020-01-13 15 * Changing this will break osq_lock() thanks to the call inside f5bfdc8e3947a7 Waiman Long 2020-01-13 16 * smp_cond_load_relaxed(). f5bfdc8e3947a7 Waiman Long 2020-01-13 17 * f5bfdc8e3947a7 Waiman Long 2020-01-13 18 * See: f5bfdc8e3947a7 Waiman Long 2020-01-13 19 * https://lore.kernel.org/lkml/[email protected] f5bfdc8e3947a7 Waiman Long 2020-01-13 20 */ 345d52c184dc7d Qian Cai 2020-01-23 @21 #define vcpu_is_preempted vcpu_is_preempted 345d52c184dc7d Qian Cai 2020-01-23 22 static inline bool vcpu_is_preempted(int cpu) 345d52c184dc7d Qian Cai 2020-01-23 23 { 345d52c184dc7d Qian Cai 2020-01-23 24 return false; 345d52c184dc7d Qian Cai 2020-01-23 25 }
f5bfdc8e3947a7 Waiman Long 2020-01-13 26
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
RFC PATCH v2 3/4 percpu: Repurpose __percpu tag as a named address space qualifier
In reply to Uros BizjakThe patch introduces per_cpu_qual define and repurposes __percpu tag as a named address space qualifier using the new define.
Arches can now conditionally define __per_cpu_qual as their named address space qualifier for percpu variables.
--- include/asm-generic/percpu.h | 15 +++++++++++++++ include/linux/compiler_types.h | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-)
include/asm-generic/percpu.h +15 -0
@@ -6,6 +6,21 @@#include <linux/threads.h> #include <linux/percpu-defs.h> +/* +* per_cpu_qual is the qualifier for the percpu named address space. +* +* Most arches use generic named address space for percpu variables but +* some arches define percpu variables in different named address space +* (on the x86 arch, percpu variables may be declared as being relative +* to the %fs or %gs segments using __seg_fs or __seg_gs named address +* space qualifiers). +*/ +#ifdef __per_cpu_qual +#define per_cpu_qual __per_cpu_qual +#else +#define per_cpu_qual/* nothing */ +#endif +#ifdef CONFIG_SMP /*
include/linux/compiler_types.h +1 -1
@@ -57,7 +57,7 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { }# define __user BTF_TYPE_TAG(user)# endif # define __iomem -# define __percpu BTF_TYPE_TAG(percpu) +# define __percpu per_cpu_qual BTF_TYPE_TAG(percpu)# define __rcu BTF_TYPE_TAG(rcu)# define __chk_user_ptr (x) (void)0
Signature · 1 line
2.46.0
Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier
Quoted 5 lines · Uros Bizjak
+#define
per_cpu_qual/* nothing */
I wouldn't do that (the “/* nothing */“ part). It might cause a mess unnecessarily is used with a comment.
Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier
Quoted 9 lines · Nadav Amit
Quoted 6 lines · Uros Bizjak
+#define
per_cpu_qual/* nothing */I wouldn't do that (the “/* nothing */“ part). It might cause a mess unnecessarily is used with a comment.
I took this /* nothing */ idea from the definition of BTF_TYPE_TAG in compiler_types.h. But yes, it doesn't bring anything here so let's remove it.
Thanks,
Uros.
kernel test robot
build botRe: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier
In reply to Uros Bizjakkernel test robot noticed the following build errors:
Hi Uros,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:
[auto build test ERROR on arnd-asm-generic/master]
[also build test ERROR on tip/x86/core driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus perf-tools-next/perf-tools-next tip/perf/core perf-tools/perf-tools tj-wq/for-next linus/master v6.11-rc3 next-20240814] [cannot apply to tip/locking/core dennis-percpu/for-next acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Uros-Bizjak/percpu-Define-pcpu_typeof/20240812-200316 base: https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git master patch link: https://lore.kernel.org/r/20240812115945.484051-4-ubizjak%40gmail.com patch subject: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier config: loongarch-allnoconfig (https://download.01.org/0day-ci/archive/20240815/[email protected]/config) compiler: loongarch64-linux-gcc (GCC) 14.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240815/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All error/warnings (new ones prefixed by >>):
In file included from include/linux/irqflags.h:19,
from include/linux/spinlock.h:59,
from include/linux/sched.h:2116,
from arch/loongarch/kernel/asm-offsets.c:8:
>> arch/loongarch/include/asm/percpu.h:71:66: error: expected ';', ',' or ')' before '*' token
71 | static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
| ^
arch/loongarch/include/asm/percpu.h:108:58: error: expected ';', ',' or ')' before '*' token
108 | static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
| ^
include/linux/interrupt.h: In function 'this_cpu_ksoftirqd':
>> arch/loongarch/include/asm/percpu.h:173:33: error: implicit declaration of function '__percpu_read'; did you mean '_percpu_read'? [-Wimplicit-function-declaration]
173 | __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
| ^~~~~~~~~~~~~
arch/loongarch/include/asm/percpu.h:218:30: note: in expansion of macro '_percpu_read'
218 | #define this_cpu_read_1(pcp) _percpu_read(pcp)
| ^~~~~~~~~~~~
include/linux/percpu-defs.h:346:30: note: in expansion of macro 'this_cpu_read_1'
346 | case 1: pscr_ret__ = stem##1(variable); break; \
| ^~~~
include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
520 | #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
| ^~~~~~~~~~~~~~~~~~~~~~~
include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
623 | return this_cpu_read(ksoftirqd);
| ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
173 | __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
| ^
arch/loongarch/include/asm/percpu.h:218:30: note: in expansion of macro '_percpu_read'
218 | #define this_cpu_read_1(pcp) _percpu_read(pcp)
| ^~~~~~~~~~~~
include/linux/percpu-defs.h:346:30: note: in expansion of macro 'this_cpu_read_1'
346 | case 1: pscr_ret__ = stem##1(variable); break; \
| ^~~~
include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
520 | #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
| ^~~~~~~~~~~~~~~~~~~~~~~
include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
623 | return this_cpu_read(ksoftirqd);
| ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
173 | __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
| ^
arch/loongarch/include/asm/percpu.h:219:30: note: in expansion of macro '_percpu_read'
219 | #define this_cpu_read_2(pcp) _percpu_read(pcp)
| ^~~~~~~~~~~~
include/linux/percpu-defs.h:347:30: note: in expansion of macro 'this_cpu_read_2'
347 | case 2: pscr_ret__ = stem##2(variable); break; \
| ^~~~
include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
520 | #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
| ^~~~~~~~~~~~~~~~~~~~~~~
include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
623 | return this_cpu_read(ksoftirqd);
| ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
173 | __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
| ^
arch/loongarch/include/asm/percpu.h:220:30: note: in expansion of macro '_percpu_read'
220 | #define this_cpu_read_4(pcp) _percpu_read(pcp)
| ^~~~~~~~~~~~
include/linux/percpu-defs.h:348:30: note: in expansion of macro 'this_cpu_read_4'
348 | case 4: pscr_ret__ = stem##4(variable); break; \
| ^~~~
include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
520 | #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
| ^~~~~~~~~~~~~~~~~~~~~~~
include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
623 | return this_cpu_read(ksoftirqd);
| ^~~~~~~~~~~~~
>> arch/loongarch/include/asm/percpu.h:173:20: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
173 | __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
| ^
arch/loongarch/include/asm/percpu.h:221:30: note: in expansion of macro '_percpu_read'
221 | #define this_cpu_read_8(pcp) _percpu_read(pcp)
| ^~~~~~~~~~~~
include/linux/percpu-defs.h:349:30: note: in expansion of macro 'this_cpu_read_8'
349 | case 8: pscr_ret__ = stem##8(variable); break; \
| ^~~~
include/linux/percpu-defs.h:520:41: note: in expansion of macro '__pcpu_size_call_return'
520 | #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
| ^~~~~~~~~~~~~~~~~~~~~~~
include/linux/interrupt.h:623:16: note: in expansion of macro 'this_cpu_read'
623 | return this_cpu_read(ksoftirqd);
| ^~~~~~~~~~~~~ make[3]: *** [scripts/Makefile.build:117: arch/loongarch/kernel/asm-offsets.s] Error 1 make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1193: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +71 arch/loongarch/include/asm/percpu.h
46859ac8af52ae Huacai Chen 2022-05-31 70
ee2daf7102f420 Huacai Chen 2023-11-21 @71 static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
46859ac8af52ae Huacai Chen 2022-05-31 72 {
46859ac8af52ae Huacai Chen 2022-05-31 73 unsigned long ret;
46859ac8af52ae Huacai Chen 2022-05-31 74
46859ac8af52ae Huacai Chen 2022-05-31 75 switch (size) {
46859ac8af52ae Huacai Chen 2022-05-31 76 case 1:
46859ac8af52ae Huacai Chen 2022-05-31 77 __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 78 : [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31 79 : [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31 80 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 81 break;
46859ac8af52ae Huacai Chen 2022-05-31 82 case 2:
46859ac8af52ae Huacai Chen 2022-05-31 83 __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 84 : [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31 85 : [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31 86 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 87 break;
46859ac8af52ae Huacai Chen 2022-05-31 88 case 4:
46859ac8af52ae Huacai Chen 2022-05-31 89 __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 90 : [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31 91 : [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31 92 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 93 break;
46859ac8af52ae Huacai Chen 2022-05-31 94 case 8:
46859ac8af52ae Huacai Chen 2022-05-31 95 __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 96 : [ret] "=&r"(ret)
46859ac8af52ae Huacai Chen 2022-05-31 97 : [ptr] "r"(ptr)
46859ac8af52ae Huacai Chen 2022-05-31 98 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 99 break;
46859ac8af52ae Huacai Chen 2022-05-31 100 default:
46859ac8af52ae Huacai Chen 2022-05-31 101 ret = 0;
46859ac8af52ae Huacai Chen 2022-05-31 102 BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31 103 }
46859ac8af52ae Huacai Chen 2022-05-31 104
46859ac8af52ae Huacai Chen 2022-05-31 105 return ret;
46859ac8af52ae Huacai Chen 2022-05-31 106 }
46859ac8af52ae Huacai Chen 2022-05-31 107
ee2daf7102f420 Huacai Chen 2023-11-21 108 static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
46859ac8af52ae Huacai Chen 2022-05-31 109 {
46859ac8af52ae Huacai Chen 2022-05-31 110 switch (size) {
46859ac8af52ae Huacai Chen 2022-05-31 111 case 1:
46859ac8af52ae Huacai Chen 2022-05-31 112 __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 113 :
46859ac8af52ae Huacai Chen 2022-05-31 114 : [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31 115 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 116 break;
46859ac8af52ae Huacai Chen 2022-05-31 117 case 2:
46859ac8af52ae Huacai Chen 2022-05-31 118 __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 119 :
46859ac8af52ae Huacai Chen 2022-05-31 120 : [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31 121 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 122 break;
46859ac8af52ae Huacai Chen 2022-05-31 123 case 4:
46859ac8af52ae Huacai Chen 2022-05-31 124 __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 125 :
46859ac8af52ae Huacai Chen 2022-05-31 126 : [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31 127 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 128 break;
46859ac8af52ae Huacai Chen 2022-05-31 129 case 8:
46859ac8af52ae Huacai Chen 2022-05-31 130 __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n"
46859ac8af52ae Huacai Chen 2022-05-31 131 :
46859ac8af52ae Huacai Chen 2022-05-31 132 : [val] "r" (val), [ptr] "r" (ptr)
46859ac8af52ae Huacai Chen 2022-05-31 133 : "memory");
46859ac8af52ae Huacai Chen 2022-05-31 134 break;
46859ac8af52ae Huacai Chen 2022-05-31 135 default:
46859ac8af52ae Huacai Chen 2022-05-31 136 BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31 137 }
46859ac8af52ae Huacai Chen 2022-05-31 138 }
46859ac8af52ae Huacai Chen 2022-05-31 139
ee2daf7102f420 Huacai Chen 2023-11-21 140 static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
46859ac8af52ae Huacai Chen 2022-05-31 141 {
46859ac8af52ae Huacai Chen 2022-05-31 142 switch (size) {
720dc7ab252bbd Huacai Chen 2022-08-25 143 case 1:
720dc7ab252bbd Huacai Chen 2022-08-25 144 case 2:
720dc7ab252bbd Huacai Chen 2022-08-25 145 return __xchg_small((volatile void *)ptr, val, size);
720dc7ab252bbd Huacai Chen 2022-08-25 146
46859ac8af52ae Huacai Chen 2022-05-31 147 case 4:
46859ac8af52ae Huacai Chen 2022-05-31 148 return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
46859ac8af52ae Huacai Chen 2022-05-31 149
46859ac8af52ae Huacai Chen 2022-05-31 150 case 8:
46859ac8af52ae Huacai Chen 2022-05-31 151 return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
46859ac8af52ae Huacai Chen 2022-05-31 152
46859ac8af52ae Huacai Chen 2022-05-31 153 default:
46859ac8af52ae Huacai Chen 2022-05-31 154 BUILD_BUG();
46859ac8af52ae Huacai Chen 2022-05-31 155 }
46859ac8af52ae Huacai Chen 2022-05-31 156
46859ac8af52ae Huacai Chen 2022-05-31 157 return 0;
46859ac8af52ae Huacai Chen 2022-05-31 158 }
46859ac8af52ae Huacai Chen 2022-05-31 159
46859ac8af52ae Huacai Chen 2022-05-31 160 /* this_cpu_cmpxchg */
46859ac8af52ae Huacai Chen 2022-05-31 161 #define _protect_cmpxchg_local(pcp, o, n) \
46859ac8af52ae Huacai Chen 2022-05-31 162 ({ \
46859ac8af52ae Huacai Chen 2022-05-31 163 typeof(*raw_cpu_ptr(&(pcp))) __ret; \
46859ac8af52ae Huacai Chen 2022-05-31 164 preempt_disable_notrace(); \
46859ac8af52ae Huacai Chen 2022-05-31 165 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
46859ac8af52ae Huacai Chen 2022-05-31 166 preempt_enable_notrace(); \
46859ac8af52ae Huacai Chen 2022-05-31 167 __ret; \
46859ac8af52ae Huacai Chen 2022-05-31 168 })
46859ac8af52ae Huacai Chen 2022-05-31 169
46859ac8af52ae Huacai Chen 2022-05-31 170 #define _percpu_read(pcp) \
46859ac8af52ae Huacai Chen 2022-05-31 171 ({ \
46859ac8af52ae Huacai Chen 2022-05-31 172 typeof(pcp) __retval; \
46859ac8af52ae Huacai Chen 2022-05-31 @173 __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
46859ac8af52ae Huacai Chen 2022-05-31 174 __retval; \
46859ac8af52ae Huacai Chen 2022-05-31 175 })46859ac8af52ae Huacai Chen 2022-05-31 176
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
RFC PATCH v2 4/4 percpu/x86: Enable strict percpu checks via named AS qualifiers
In reply to Uros BizjakDeclare percpu variables in __seg_gs/__seg_fs named AS and keep them named AS qualified until they are dereferenced with percpu accessor. This approach enables various compiler check for cross-namespace variable assignments.
--- arch/x86/include/asm/percpu.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
arch/x86/include/asm/percpu.h +12 -3
@@ -95,9 +95,18 @@#endif /* CONFIG_SMP */ -#define __my_cpu_type (var) typeof(var) __percpu_seg_override -#define __my_cpu_ptr (ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr) -#define __my_cpu_var (var) (*__my_cpu_ptr(&(var))) +#if defined(CONFIG_USE_X86_SEG_SUPPORT ) && defined(CONFIG_CC_HAS_TYPEOF_UNQUAL ) +# define __my_cpu_type (var) typeof(var) +# define __my_cpu_ptr (ptr) (ptr) +# define __my_cpu_var (var) (var) + +# define __per_cpu_qual __percpu_seg_override +#else +# define __my_cpu_type (var) typeof(var) __percpu_seg_override +# define __my_cpu_ptr (ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr) +# define __my_cpu_var (var) (*__my_cpu_ptr(&(var))) +#endif +#define __percpu_arg (x) __percpu_prefix "%" #x#define __force_percpu_arg (x) __force_percpu_prefix "%" #x
Signature · 1 line
2.46.0