smp: fix smp_call_function_single_async prototype
PATCH v2 1/1 smp: fix smp_call_function_single_async prototype
From: Arnd Bergmann <[email protected]>
As of commit 966a967116e6 ("smp: Avoid using two cache lines for struct call_single_data"), the smp code prefers 32-byte aligned call_single_data objects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see 4ccafe032005 ("block: unalign call_single_data in struct request").
The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]
smp_call_function_single_async(cpu, &rq->csd);
It does seem that the usage of the call_single_data without cache line alignment should still be allowed by the smp code, so just change the function prototype so it accepts both, but leave the default alignment unchanged for the other users. This seems better to me than adding a local hack to shut up an otherwise correct warning in the caller.
---
v2: avoid adding other warnings by a more thorough change
--- include/linux/smp.h | 2 +- kernel/smp.c | 26 +++++++++++++------------- kernel/up.c | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-)
include/linux/smp.h +1 -1
@@ -53,7 +53,7 @@ int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,void on_each_cpu_cond_mask (smp_cond_func_t cond_func,smp_call_func_t func,void *info,bool wait,const struct cpumask *mask); -int smp_call_function_single_async (int cpu,call_single_data_t *csd); +int smp_call_function_single_async (int cpu,struct __call_single_data *csd);/* * Cpus stopping functions in panic. All have default weak definitions.
kernel/smp.c +13 -13
@@ -211,7 +211,7 @@ static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type) }while (0 )/* Record current CSD work for current CPU, NULL to erase. */ -static void __csd_lock_record(call_single_data_t *csd) +static void __csd_lock_record(struct __call_single_data *csd) {if (!csd) {smp_mb ();/* NULL cur_csd after unlock. */
@@ -226,13 +226,13 @@ static void __csd_lock_record(call_single_data_t *csd)/* Or before unlock, as the case may be. */ } -static __always_inline void csd_lock_record(call_single_data_t *csd) +static __always_inline void csd_lock_record(struct __call_single_data *csd) {if (static_branch_unlikely (&csdlock_debug_enabled))__csd_lock_record (csd); } -static int csd_lock_wait_getcpu (call_single_data_t *csd) +static int csd_lock_wait_getcpu (struct __call_single_data *csd) {unsigned int csd_type;
@@ -282,7 +282,7 @@ static const char *csd_lock_get_type(unsigned int type)return (type >=ARRAY_SIZE (seq_type)) ?"?" : seq_type[type]; } -static void csd_lock_print_extended (call_single_data_t *csd,int cpu) +static void csd_lock_print_extended (struct __call_single_data *csd,int cpu) {struct cfd_seq_local *seq = &per_cpu (cfd_seq_local, cpu);unsigned int srccpu = csd->node.src;
@@ -321,7 +321,7 @@ static void csd_lock_print_extended(call_single_data_t *csd, int cpu)* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */ -static bool csd_lock_wait_toolong (call_single_data_t *csd,u64 ts0,u64 *ts1,int *bug_id) +static bool csd_lock_wait_toolong (struct __call_single_data *csd,u64 ts0,u64 *ts1,int *bug_id) {int cpu =-1 ;int cpux;
@@ -387,7 +387,7 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in* previous function call. For multi-cpu calls its even more interesting * as we'll have to ensure no other cpu is observing our csd. */ -static void __csd_lock_wait (call_single_data_t *csd) +static void __csd_lock_wait (struct __call_single_data *csd) {int bug_id =0 ;u64 ts0, ts1;
@@ -401,7 +401,7 @@ static void __csd_lock_wait(call_single_data_t *csd)smp_acquire__after_ctrl_dep (); } -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) {if (static_branch_unlikely (&csdlock_debug_enabled)) {__csd_lock_wait (csd);
@@ -431,17 +431,17 @@ static void __smp_call_single_queue_debug(int cpu, struct llist_node *node)#else #define cfd_seq_store (var, src, dst, type) -static void csd_lock_record (call_single_data_t *csd) +static void csd_lock_record (struct __call_single_data *csd) { } -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) {smp_cond_load_acquire (&csd->node.u_flags, !(VAL &CSD_FLAG_LOCK )); }#endif -static __always_inline void csd_lock(call_single_data_t *csd) +static __always_inline void csd_lock(struct __call_single_data *csd) {csd_lock_wait (csd); csd->node.u_flags |=CSD_FLAG_LOCK ;
@@ -454,7 +454,7 @@ static __always_inline void csd_lock(call_single_data_t *csd)smp_wmb (); } -static __always_inline void csd_unlock(call_single_data_t *csd) +static __always_inline void csd_unlock(struct __call_single_data *csd) {WARN_ON (!(csd->node.u_flags &CSD_FLAG_LOCK ));
@@ -501,7 +501,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)* for execution on the given CPU. data must already have * ->func, ->info, and ->flags set. */ -static int generic_exec_single (int cpu,call_single_data_t *csd) +static int generic_exec_single (int cpu,struct __call_single_data *csd) {if (cpu ==smp_processor_id ()) {smp_call_func_t func = csd->func;
@@ -784,7 +784,7 @@ EXPORT_SYMBOL(smp_call_function_single);* NOTE: Be careful, there is unfortunately no current debugging facility to * validate the correctness of this serialization. */ -int smp_call_function_single_async (int cpu,call_single_data_t *csd) +int smp_call_function_single_async (int cpu,struct __call_single_data *csd) {int err =0 ;
kernel/up.c +1 -1
@@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, }EXPORT_SYMBOL (smp_call_function_single); -int smp_call_function_single_async (int cpu,call_single_data_t *csd) +int smp_call_function_single_async (int cpu,struct __call_single_data *csd) {unsigned long flags;
Signature · 1 line
2.29.2
Quoted 9 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").
Quoted 5 lines
The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);
Can this be silenced by
smp_call_function_single_async(cpu, (call_single_data_t *)&rq->csd);
Best Regards,
Huang, Ying
[snip]
Quoted 15 lines · Ying
Quoted 12 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").The result is a violation of the calling conventions that clang correctly points out:
Quoted 6 lines
Quoted 2 lines
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);Can this be silenced by
smp_call_function_single_async(cpu, (call_single_data_t *)&rq->csd);
Probably, but casting from smaller alignment to larger alignment is undefined behavior and I'd rather not go there in case this triggers some runtime misbehavior or ubsan check in the future. Making the function accept a pointer with the smaller alignment avoids getting into undefined behavior and doesn't require a cast.
Arnd
Quoted 19 lines · Arnd Bergmann
Quoted 18 lines · Ying
Quoted 14 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);
Quoted 6 lines
Quoted 3 lines
Can this be silenced by
smp_call_function_single_async(cpu, (call_single_data_t *)&rq->csd);Probably, but casting from smaller alignment to larger alignment is undefined behavior
We cannot avoid type cast in Linux kernel, such as container_of(), is there some difference here?
Quoted 4 lines
and I'd rather not go there in case this triggers some runtime misbehavior or ubsan check in the future. Making the function accept a pointer with the smaller alignment avoids getting into undefined behavior and doesn't require a cast.
In its raw form as above, this looks bad. If we encapsulate it, it may look better, for example,
static inline int __smp_call_function_single_async(int cpu, struct __call_single_data *csd)
{
smp_call_function_single_async(cpu, (call_single_data_t *)csd);
}Then, we can do
__smp_call_function_single_async(cpu, &rq->csd);
Best Regards,
Huang, Ying
Quoted 24 lines · Ying
Quoted 21 lines · Arnd Bergmann
Quoted 20 lines · Ying
Quoted 9 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").Quoted 5 lines
The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);Can this be silenced by
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
smp_call_function_single_async(cpu, (call_single_data_t *)&rq->csd);Probably, but casting from smaller alignment to larger alignment is undefined behavior
We cannot avoid type cast in Linux kernel, such as
container_of(), is there some difference here?
container_of() does not cause any alignment problems. Assuming the outer structure is aligned correctly, then the inner structure also is.
Quoted 8 lines
Quoted 4 lines
and I'd rather not go there in case this triggers some runtime misbehavior or ubsan check in the future. Making the function accept a pointer with the smaller alignment avoids getting into undefined behavior and doesn't require a cast.
In its raw form as above, this looks bad. If we encapsulate it, it may look better, for example,
Quoted 8 lines
static inline int
__smp_call_function_single_async(int cpu, struct __call_single_data *csd){ smp_call_function_single_async(cpu, (call_single_data_t *)csd); }Then, we can do
__smp_call_function_single_async(cpu, &rq->csd);
Same problem, it's still calling a function that expects stricter alignment. It would work if we do it the other way around though:
static inline int smp_call_function_single_async(int cpu, call_single_data_t *csd)
{
return __smp_call_function_single_async(cpu, csd);
}That should even work without the cast.
Arnd
Quoted 30 lines · Arnd Bergmann
Quoted 29 lines · Ying
Quoted 26 lines · Arnd Bergmann
Quoted 24 lines · Ying
Quoted 9 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").Quoted 8 lines
The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);Can this be silenced by
smp_call_function_single_async(cpu, (call_single_data_t *)&rq->csd);
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Probably, but casting from smaller alignment to larger alignment is undefined behavior
We cannot avoid type cast in Linux kernel, such as
container_of(), is there some difference here?
container_of()does not cause any alignment problems. Assuming the outer structure is aligned correctly, then the inner structure also is.
So you think that the compiler may generate different code depends on the data structure alignment (8 vs. 32 here)? I think that it doesn't on x86. Do you know it does that on any architecture? But I understand that this is possible at least in theory.
Quoted 20 lines
Quoted 8 lines
Quoted 4 lines
and I'd rather not go there in case this triggers some runtime misbehavior or ubsan check in the future. Making the function accept a pointer with the smaller alignment avoids getting into undefined behavior and doesn't require a cast.
In its raw form as above, this looks bad. If we encapsulate it, it may look better, for example,
Quoted 8 lines
static inline int
__smp_call_function_single_async(int cpu, struct __call_single_data *csd){ smp_call_function_single_async(cpu, (call_single_data_t *)csd); }Then, we can do
__smp_call_function_single_async(cpu, &rq->csd);Same problem, it's still calling a function that expects stricter alignment. It would work if we do it the other way around though:
Quoted 7 lines
static inline int
smp_call_function_single_async(int cpu, call_single_data_t *csd){ return __smp_call_function_single_async(cpu, csd); }That should even work without the cast.
Yes. This looks good!
Best Regards,
Huang, Ying
Quoted 6 lines · Ying
Quoted 5 lines · Arnd Bergmann
Quoted 3 lines · Ying
We cannot avoid type cast in Linux kernel, such as
container_of(), is there some difference here?
Quoted 7 lines
Quoted 2 lines
container_of()does not cause any alignment problems. Assuming the outer structure is aligned correctly, then the inner structure also is.So you think that the compiler may generate different code depends on the data structure alignment (8 vs. 32 here)? I think that it doesn't on x86. Do you know it does that on any architecture? But I understand that this is possible at least in theory.
It probably won't generate any different code because that would be silly, but it's also not a good idea to rely on that. In theory the compiler might e.g. construct an offset into the structure using a bitwise-or instruction instead of an addition if the alignment tells it that the lower bits are always zero.
Arnd
Quoted 21 lines · Arnd Bergmann
From: Arnd Bergmann <[email protected]>
As of commit
966a967116e6("smp: Avoid using two cache lines for structcall_single_data"), the smp code prefers 32-byte alignedcall_single_dataobjects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see4ccafe032005("block: unaligncall_single_datain struct request").The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]smp_call_function_single_async(cpu, &rq->csd);It does seem that the usage of the
call_single_datawithout cache line alignment should still be allowed by the smp code, so just change the function prototype so it accepts both, but leave the default alignment unchanged for the other users. This seems better to me than adding a local hack to shut up an otherwise correct warning in the caller.
Quoted 13 lines
Acked-by Jens Axboe <[email protected]>Cc Jian Cai <[email protected]>Cc Guenter Roeck <[email protected]>Cc Peter Zijlstra <[email protected]>Cc "Huang, Ying" <[email protected]>Cc Borislav Petkov <[email protected]>Cc Eric Dumazet <[email protected]>Cc Juergen Gross <[email protected]>Cc Michael Ellerman <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Arnd Bergmann <[email protected]>
Thanks!
[tip: locking/urgent] smp: Fix smp_call_function_single_async prototype
In reply to Arnd BergmannThe following commit has been merged into the locking/urgent branch of tip:
The following commit has been merged into the locking/urgent branch of tip:
Commit-ID: 1139aeb1c521eb4a050920ce6c64c36c4f2a3ab7 Gitweb: https://git.kernel.org/tip/1139aeb1c521eb4a050920ce6c64c36c4f2a3ab7 Author: Arnd Bergmann <[email protected]> AuthorDate: Wed, 05 May 2021 23:12:42 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Thu, 06 May 2021 15:33:49 +02:00
smp: Fix smp_call_function_single_async prototype
As of commit 966a967116e6 ("smp: Avoid using two cache lines for struct call_single_data"), the smp code prefers 32-byte aligned call_single_data objects for performance reasons, but the block layer includes an instance of this structure in the main 'struct request' that is more senstive to size than to performance here, see 4ccafe032005 ("block: unalign call_single_data in struct request").
The result is a violation of the calling conventions that clang correctly points out:
block/blk-mq.c:630:39: warning: passing 8-byte aligned argument to 32-byte aligned parameter 2 of 'smp_call_function_single_async' may result in an unaligned pointer access [-Walign-mismatch]
smp_call_function_single_async(cpu, &rq->csd);
It does seem that the usage of the call_single_data without cache line alignment should still be allowed by the smp code, so just change the function prototype so it accepts both, but leave the default alignment unchanged for the other users. This seems better to me than adding a local hack to shut up an otherwise correct warning in the caller.
--- include/linux/smp.h | 2 +- kernel/smp.c | 26 +++++++++++++------------- kernel/up.c | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-)
include/linux/smp.h +1 -1
@@ -53,7 +53,7 @@ int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,void on_each_cpu_cond_mask (smp_cond_func_t cond_func,smp_call_func_t func,void *info,bool wait,const struct cpumask *mask); -int smp_call_function_single_async (int cpu,call_single_data_t *csd); +int smp_call_function_single_async (int cpu,struct __call_single_data *csd);/* * Call a function on all processors
kernel/smp.c +13 -13
@@ -211,7 +211,7 @@ static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type) }while (0 )/* Record current CSD work for current CPU, NULL to erase. */ -static void __csd_lock_record(call_single_data_t *csd) +static void __csd_lock_record(struct __call_single_data *csd) {if (!csd) {smp_mb ();/* NULL cur_csd after unlock. */
@@ -226,13 +226,13 @@ static void __csd_lock_record(call_single_data_t *csd)/* Or before unlock, as the case may be. */ } -static __always_inline void csd_lock_record(call_single_data_t *csd) +static __always_inline void csd_lock_record(struct __call_single_data *csd) {if (static_branch_unlikely (&csdlock_debug_enabled))__csd_lock_record (csd); } -static int csd_lock_wait_getcpu (call_single_data_t *csd) +static int csd_lock_wait_getcpu (struct __call_single_data *csd) {unsigned int csd_type;
@@ -282,7 +282,7 @@ static const char *csd_lock_get_type(unsigned int type)return (type >=ARRAY_SIZE (seq_type)) ?"?" : seq_type[type]; } -static void csd_lock_print_extended (call_single_data_t *csd,int cpu) +static void csd_lock_print_extended (struct __call_single_data *csd,int cpu) {struct cfd_seq_local *seq = &per_cpu (cfd_seq_local, cpu);unsigned int srccpu = csd->node.src;
@@ -321,7 +321,7 @@ static void csd_lock_print_extended(call_single_data_t *csd, int cpu)* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */ -static bool csd_lock_wait_toolong (call_single_data_t *csd,u64 ts0,u64 *ts1,int *bug_id) +static bool csd_lock_wait_toolong (struct __call_single_data *csd,u64 ts0,u64 *ts1,int *bug_id) {int cpu =-1 ;int cpux;
@@ -387,7 +387,7 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in* previous function call. For multi-cpu calls its even more interesting * as we'll have to ensure no other cpu is observing our csd. */ -static void __csd_lock_wait (call_single_data_t *csd) +static void __csd_lock_wait (struct __call_single_data *csd) {int bug_id =0 ;u64 ts0, ts1;
@@ -401,7 +401,7 @@ static void __csd_lock_wait(call_single_data_t *csd)smp_acquire__after_ctrl_dep (); } -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) {if (static_branch_unlikely (&csdlock_debug_enabled)) {__csd_lock_wait (csd);
@@ -431,17 +431,17 @@ static void __smp_call_single_queue_debug(int cpu, struct llist_node *node)#else #define cfd_seq_store (var, src, dst, type) -static void csd_lock_record (call_single_data_t *csd) +static void csd_lock_record (struct __call_single_data *csd) { } -static __always_inline void csd_lock_wait(call_single_data_t *csd) +static __always_inline void csd_lock_wait(struct __call_single_data *csd) {smp_cond_load_acquire (&csd->node.u_flags, !(VAL &CSD_FLAG_LOCK )); }#endif -static __always_inline void csd_lock(call_single_data_t *csd) +static __always_inline void csd_lock(struct __call_single_data *csd) {csd_lock_wait (csd); csd->node.u_flags |=CSD_FLAG_LOCK ;
@@ -454,7 +454,7 @@ static __always_inline void csd_lock(call_single_data_t *csd)smp_wmb (); } -static __always_inline void csd_unlock(call_single_data_t *csd) +static __always_inline void csd_unlock(struct __call_single_data *csd) {WARN_ON (!(csd->node.u_flags &CSD_FLAG_LOCK ));
@@ -501,7 +501,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)* for execution on the given CPU. data must already have * ->func, ->info, and ->flags set. */ -static int generic_exec_single (int cpu,call_single_data_t *csd) +static int generic_exec_single (int cpu,struct __call_single_data *csd) {if (cpu ==smp_processor_id ()) {smp_call_func_t func = csd->func;
@@ -784,7 +784,7 @@ EXPORT_SYMBOL(smp_call_function_single);* NOTE: Be careful, there is unfortunately no current debugging facility to * validate the correctness of this serialization. */ -int smp_call_function_single_async (int cpu,call_single_data_t *csd) +int smp_call_function_single_async (int cpu,struct __call_single_data *csd) {int err =0 ;
kernel/up.c +1 -1
@@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, }EXPORT_SYMBOL (smp_call_function_single); -int smp_call_function_single_async (int cpu,call_single_data_t *csd) +int smp_call_function_single_async (int cpu,struct __call_single_data *csd) {unsigned long flags;