trace,smp: Add tracepoints for csd
v5 cover trace,smp: Add tracepoints for csd
Changes since RFCv4:
- Moved from trace/events/smp.h to trace/events/csd.h
- Using TRACE_SYSTEM csd, instead of smp
- generic_exec_single() trace uses NULL instead of csd for current cpu.
- Trace function csd type moved from call_single_data_t to struct __call_single_data to avoid alignment errors on blk code.
Changes since RFCv3:
- Split the patch in 2: entry/exit and queue
- Fix 'struct __call_single_data' & call_single_data_t alignment issue
- Made all TP_printk follow the same pattern
Changes since RFCv2:
- Fixed some spacing issues and trace calls
Changes since RFCv1:
- Implemented trace_csd_queue_cpu() as suggested by Valentin Schneider
- Using EVENT_CLASS in order to avoid duplication
- Introduced new helper: csd_do_func()
- Name change from smp_call_function_* to csd_function_*
- Rebased on top of torvalds/master
Leonardo Bras (2):
trace,smp: Add tracepoints around remotelly called functions trace,smp: Add tracepoints for scheduling remotelly called functions
include/trace/events/csd.h | 72 ++++++++++++++++++++++++++++++++++++++ kernel/smp.c | 41 +++++++++++++--------- 2 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 include/trace/events/csd.h
Signature · 1 line
2.41.0
RFC PATCH v5 1/2 trace,smp: Add tracepoints around remotelly called functions
The recently added ipi_send_{cpu,cpumask} tracepoints allow finding sources of IPIs targeting CPUs running latency-sensitive applications.
For NOHZ_FULL CPUs, all IPIs are interference, and those tracepoints are sufficient to find them and work on getting rid of them. In some setups however, not *all* IPIs are to be suppressed, but long-running IPI callbacks can still be problematic.
Add a pair of tracepoints to mark the start and end of processing a CSD IPI callback, similar to what exists for softirq, workqueue or timer callbacks.
--- include/trace/events/csd.h | 45 ++++++++++++++++++++++++++++++++++++++ kernel/smp.c | 25 ++++++++++++++++----- 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 include/trace/events/csd.h
include/trace/events/csd.h new file +45 -0
@@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM csd + +#if !defined(_TRACE_CSD_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_CSD_H + +#include <linux/tracepoint.h> + +/* +* Tracepoints for a function which is called as an effect of smp_call_function.* +*/ +DECLARE_EVENT_CLASS (csd_function, + + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), + +TP_ARGS (func, csd), + +TP_STRUCT__entry ( + __field(void *, func) + __field(void *, csd) + ), + +TP_fast_assign ( + __entry->func = func; + __entry->csd = csd; + ), + +TP_printk ("func=%ps, csd=%p" , __entry->func, __entry->csd) +); + +DEFINE_EVENT (csd_function, csd_function_entry, + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), +TP_ARGS (func, csd) +); + +DEFINE_EVENT (csd_function, csd_function_exit, + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), +TP_ARGS (func, csd) +); + +#endif /* _TRACE_CSD_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h>
kernel/smp.c +19 -6
@@ -27,6 +27,9 @@#include <linux/jump_label.h> #include <trace/events/ipi.h> +#define CREATE_TRACE_POINTS +#include <trace/events/csd.h> +#undef CREATE_TRACE_POINTS#include "smpboot.h" #include "sched/smp.h"
@@ -121,6 +124,14 @@ send_call_function_ipi_mask(struct cpumask *mask)arch_send_call_function_ipi_mask (mask); } +static __always_inline void +csd_do_func (smp_call_func_t func,void *info,struct __call_single_data *csd) +{ +trace_csd_function_entry (func, csd); +func (info); +trace_csd_function_exit (func, csd); +} +#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
@@ -375,7 +386,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd)csd_lock_record (csd);csd_unlock (csd);local_irq_save (flags); -func (info); +csd_do_func (func, info,NULL );csd_lock_record (NULL );local_irq_restore (flags);return 0 ;
@@ -477,7 +488,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) }csd_lock_record (csd); -func (info); +csd_do_func (func, info, csd);csd_unlock (csd);csd_lock_record (NULL ); }else {
@@ -508,7 +519,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline)csd_lock_record (csd);csd_unlock (csd); -func (info); +csd_do_func (func, info, csd);csd_lock_record (NULL ); }else if (type ==CSD_TYPE_IRQ_WORK ) {irq_work_single (csd);
@@ -522,8 +533,10 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline)/* * Third; only CSD_TYPE_TTWU is left, issue those. */ -if (entry) -sched_ttwu_pending (entry); +if (entry) { + csd =llist_entry (entry,typeof (*csd), node.llist); +csd_do_func (sched_ttwu_pending, entry, csd); + } }
@@ -816,7 +829,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask,unsigned long flags;local_irq_save (flags); -func (info); +csd_do_func (func, info,NULL );local_irq_restore (flags); }
Signature · 1 line
2.41.0
Re: [RFC PATCH v5 1/2] trace,smp: Add tracepoints around remotelly called functions
Quoted 8 lines · Leonardo Bras
The recently added ipi_send_{cpu,cpumask} tracepoints allow finding sources of IPIs targeting CPUs running latency-sensitive applications.
For
NOHZ_FULLCPUs, all IPIs are interference, and those tracepoints are sufficient to find them and work on getting rid of them. In some setups however, not *all* IPIs are to be suppressed, but long-running IPI callbacks can still be problematic.
Quoted 4 lines
Add a pair of tracepoints to mark the start and end of processing a CSD IPI callback, similar to what exists for softirq, workqueue or timer callbacks.
Signed-off-by Leonardo Bras <[email protected]>
[tip: smp/core] trace,smp: Add tracepoints around remotelly called functions
In reply to Leonardo BrasThe following commit has been merged into the smp/core branch of tip:
The following commit has been merged into the smp/core branch of tip:
Commit-ID: 949fa3f11ced2a5c8e3737e73b09676adf4b322b Gitweb: https://git.kernel.org/tip/949fa3f11ced2a5c8e3737e73b09676adf4b322b Author: Leonardo Bras <[email protected]> AuthorDate: Thu, 15 Jun 2023 03:59:45 -03:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 16 Jun 2023 22:08:09 +02:00
trace,smp: Add tracepoints around remotelly called functions
The recently added ipi_send_{cpu,cpumask} tracepoints allow finding sources of IPIs targeting CPUs running latency-sensitive applications.
For NOHZ_FULL CPUs, all IPIs are interference, and those tracepoints are sufficient to find them and work on getting rid of them. In some setups however, not *all* IPIs are to be suppressed, but long-running IPI callbacks can still be problematic.
Add a pair of tracepoints to mark the start and end of processing a CSD IPI callback, similar to what exists for softirq, workqueue or timer callbacks.
--- include/trace/events/csd.h | 45 +++++++++++++++++++++++++++++++++++++- kernel/smp.c | 25 +++++++++++++++------ 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 include/trace/events/csd.h
include/trace/events/csd.h new file +45 -0
@@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM csd + +#if !defined(_TRACE_CSD_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_CSD_H + +#include <linux/tracepoint.h> + +/* +* Tracepoints for a function which is called as an effect of smp_call_function.* +*/ +DECLARE_EVENT_CLASS (csd_function, + + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), + +TP_ARGS (func, csd), + +TP_STRUCT__entry ( + __field(void *, func) + __field(void *, csd) + ), + +TP_fast_assign ( + __entry->func = func; + __entry->csd = csd; + ), + +TP_printk ("func=%ps, csd=%p" , __entry->func, __entry->csd) +); + +DEFINE_EVENT (csd_function, csd_function_entry, + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), +TP_ARGS (func, csd) +); + +DEFINE_EVENT (csd_function, csd_function_exit, + TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), +TP_ARGS (func, csd) +); + +#endif /* _TRACE_CSD_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h>
kernel/smp.c +19 -6
@@ -27,6 +27,9 @@#include <linux/jump_label.h> #include <trace/events/ipi.h> +#define CREATE_TRACE_POINTS +#include <trace/events/csd.h> +#undef CREATE_TRACE_POINTS#include "smpboot.h" #include "sched/smp.h"
@@ -121,6 +124,14 @@ send_call_function_ipi_mask(struct cpumask *mask)arch_send_call_function_ipi_mask (mask); } +static __always_inline void +csd_do_func (smp_call_func_t func,void *info,struct __call_single_data *csd) +{ +trace_csd_function_entry (func, csd); +func (info); +trace_csd_function_exit (func, csd); +} +#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
@@ -375,7 +386,7 @@ static int generic_exec_single(int cpu, struct __call_single_data *csd)csd_lock_record (csd);csd_unlock (csd);local_irq_save (flags); -func (info); +csd_do_func (func, info,NULL );csd_lock_record (NULL );local_irq_restore (flags);return 0 ;
@@ -477,7 +488,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) }csd_lock_record (csd); -func (info); +csd_do_func (func, info, csd);csd_unlock (csd);csd_lock_record (NULL ); }else {
@@ -508,7 +519,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline)csd_lock_record (csd);csd_unlock (csd); -func (info); +csd_do_func (func, info, csd);csd_lock_record (NULL ); }else if (type ==CSD_TYPE_IRQ_WORK ) {irq_work_single (csd);
@@ -522,8 +533,10 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline)/* * Third; only CSD_TYPE_TTWU is left, issue those. */ -if (entry) -sched_ttwu_pending (entry); +if (entry) { + csd =llist_entry (entry,typeof (*csd), node.llist); +csd_do_func (sched_ttwu_pending, entry, csd); + } }
@@ -816,7 +829,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask,unsigned long flags;local_irq_save (flags); -func (info); +csd_do_func (func, info,NULL );local_irq_restore (flags); }
RFC PATCH v5 2/2 trace,smp: Add tracepoints for scheduling remotelly called functions
In reply to Leonardo BrasAdd a tracepoint for when a CSD is queued to a remote CPU's call_single_queue. This allows finding exactly which CPU queued a given CSD when looking at a csd_function_{entry,exit} event, and also enables us to accurately measure IPI delivery time with e.g. a synthetic event:
$ echo 'hist:keys=cpu,csd.hex:ts=common_timestamp.usecs' >\
/sys/kernel/tracing/events/smp/csd_queue_cpu/trigger
$ echo 'csd_latency unsigned int dst_cpu; unsigned long csd; u64 time' >\
/sys/kernel/tracing/synthetic_events
$ echo \
'hist:keys=common_cpu,csd.hex:'\
'time=common_timestamp.usecs-$ts:'\
'onmatch(smp.csd_queue_cpu).trace(csd_latency,common_cpu,csd,$time)' >\
/sys/kernel/tracing/events/smp/csd_function_entry/trigger$ trace-cmd record -e 'synthetic:csd_latency' hackbench $ trace-cmd report <...>-467 [001] 21.824263: csd_queue_cpu: cpu=0 callsite=try_to_wake_up+0x2ea func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-467 [001] 21.824280: ipi_send_cpu: cpu=0 callsite=try_to_wake_up+0x2ea callback=generic_smp_call_function_single_interrupt+0x0 <...>-489 [000] 21.824299: csd_function_entry: func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-489 [000] 21.824320: csd_latency: dst_cpu=0, csd=18446612682193848504, time=36
--- include/trace/events/csd.h | 27 +++++++++++++++++++++++++++ kernel/smp.c | 16 +++++----------- 2 files changed, 32 insertions(+), 11 deletions(-)
include/trace/events/csd.h +27 -0
@@ -7,6 +7,33 @@#include <linux/tracepoint.h> +TRACE_EVENT (csd_queue_cpu, + + TP_PROTO(const unsigned int cpu, + unsigned long callsite, + smp_call_func_t func, + struct __call_single_data *csd), + +TP_ARGS (cpu, callsite, func, csd), + +TP_STRUCT__entry ( + __field(unsigned int, cpu) + __field(void *, callsite) + __field(void *, func) + __field(void *, csd) + ), + +TP_fast_assign ( + __entry->cpu = cpu; + __entry->callsite = (void *)callsite; + __entry->func = func; + __entry->csd = csd; + ), + +TP_printk ("cpu=%u callsite=%pS func=%ps csd=%p" , + __entry->cpu, __entry->callsite, __entry->func, __entry->csd) + ); +/* * Tracepoints for a function which is called as an effect of smp_call_function.* */
kernel/smp.c +5 -11
@@ -340,7 +340,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)* even if we haven't sent the smp_call IPI yet (e.g. the stopper * executes migration_cpu_stop() on the remote CPU). */ -if (trace_ipi_send_cpu_enabled ()) { +if (trace_csd_queue_cpu_enabled ()) {call_single_data_t *csd;smp_call_func_t func;
@@ -348,7 +348,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) func =CSD_TYPE (csd) ==CSD_TYPE_TTWU ? sched_ttwu_pending : csd->func; -trace_ipi_send_cpu (cpu, _RET_IP_, func); +trace_csd_queue_cpu (cpu, _RET_IP_, func, csd); }/*
@@ -741,7 +741,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask,int cpu, last_cpu, this_cpu =smp_processor_id ();struct call_function_data *cfd;bool wait = scf_flags &SCF_WAIT ; -int nr_cpus =0 , nr_queued =0 ; +int nr_cpus =0 ;bool run_remote = false;bool run_local = false;
@@ -799,21 +799,15 @@ static void smp_call_function_many_cond(const struct cpumask *mask, csd->node.src =smp_processor_id (); csd->node.dst = cpu;#endif +trace_csd_queue_cpu (cpu, _RET_IP_, func, csd); +if (llist_add (&csd->node.llist, &per_cpu (call_single_queue, cpu))) {__cpumask_set_cpu (cpu, cfd->cpumask_ipi); nr_cpus++; last_cpu = cpu; } - nr_queued++; } -/* -* Trace each smp_function_call_*() as an IPI, actual IPIs -* will be traced with func==generic_smp_call_function_single_ipi(). -*/ -if (nr_queued) -trace_ipi_send_cpumask (cfd->cpumask, _RET_IP_, func); -/* * Choose the most efficient way to send an IPI. Note that the * number of CPUs might be zero due to concurrent changes to the
Signature · 1 line
2.41.0
Re: [RFC PATCH v5 2/2] trace,smp: Add tracepoints for scheduling remotelly called functions
Quoted 15 lines · Leonardo Bras
Add a tracepoint for when a CSD is queued to a remote CPU's
call_single_queue. This allows finding exactly which CPU queued a given CSD when looking at a csd_function_{entry,exit} event, and also enables us to accurately measure IPI delivery time with e.g. a synthetic event:$ echo 'hist:keys=cpu,csd.hex:ts=common_timestamp.usecs' >\ /sys/kernel/tracing/events/smp/csd_queue_cpu/trigger $ echo 'csd_latency unsigned int dst_cpu; unsigned long csd; u64 time' >\ /sys/kernel/tracing/synthetic_events $ echo \ 'hist:keys=common_cpu,csd.hex:'\ 'time=common_timestamp.usecs-$ts:'\ 'onmatch(smp.csd_queue_cpu).trace(csd_latency,common_cpu,csd,$time)' >\ /sys/kernel/tracing/events/smp/csd_function_entry/trigger
Quoted 7 lines
$ trace-cmd record -e 'synthetic:csd_latency' hackbench $ trace-cmd report <...>-467 [001] 21.824263: csd_queue_cpu: cpu=0 callsite=try_to_wake_up+0x2ea func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-467 [001] 21.824280: ipi_send_cpu: cpu=0 callsite=try_to_wake_up+0x2ea callback=generic_smp_call_function_single_interrupt+0x0 <...>-489 [000] 21.824299: csd_function_entry: func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-489 [000] 21.824320: csd_latency: dst_cpu=0, csd=18446612682193848504, time=36
Nit: these commands now need a s/smp/csd/.
I played with those a little more and found out how to record the src CPU, so feel free to update the changelog with this:
$ echo 'hist:keys=cpu,csd.hex:ts=common_timestamp.usecs:src=common_cpu' >\
/sys/kernel/tracing/events/csd/csd_queue_cpu/trigger
$ echo 'csd_latency unsigned int src_cpu; '\
'unsigned int dst_cpu; '\
'unsigned long csd; u64 time' >\
/sys/kernel/tracing/synthetic_events $ echo 'hist:keys=common_cpu,csd.hex:
time=common_timestamp.usecs-$ts:
onmatch(csd.csd_queue_cpu).trace(csd_latency,$src,common_cpu,csd,$time)' >\
/sys/kernel/tracing/events/csd/csd_function_entry/trigger$ trace-cmd record -e 'synthetic:csd_latency' hackbench $ trace-cmd report <idle>-0 [001] 115.236810: csd_latency: src_cpu=7, dst_cpu=1, csd=18446612682588476192, time=134 <idle>-0 [000] 115.240676: csd_latency: src_cpu=7, dst_cpu=0, csd=18446612682588214048, time=103 <idle>-0 [009] 115.241320: csd_latency: src_cpu=7, dst_cpu=9, csd=18446612682143963384, time=83 <idle>-0 [007] 115.242817: csd_latency: src_cpu=8, dst_cpu=7, csd=18446612682150759032, time=93 <idle>-0 [005] 115.247802: csd_latency: src_cpu=7, dst_cpu=5, csd=18446612682144441144, time=114 <idle>-0 [005] 115.271775: csd_latency: src_cpu=7, dst_cpu=5, csd=18446612682144441144, time=151 <idle>-0 [000] 115.279620: csd_latency: src_cpu=7, dst_cpu=0, csd=18446612682588214048, time=87 <idle>-0 [000] 115.281727: csd_latency: src_cpu=7, dst_cpu=0, csd=18446612682588214048, time=101
Quoted 2 lines
Suggested-by Valentin Schneider <[email protected]>Signed-off-by Leonardo Bras <[email protected]>
Other than that:
[tip: smp/core] trace,smp: Add tracepoints for scheduling remotelly called functions
In reply to Leonardo BrasThe following commit has been merged into the smp/core branch of tip:
The following commit has been merged into the smp/core branch of tip:
Commit-ID: bf5a8c26ad7caf0772a1cd48c8a0924e48bdbaf0 Gitweb: https://git.kernel.org/tip/bf5a8c26ad7caf0772a1cd48c8a0924e48bdbaf0 Author: Leonardo Bras <[email protected]> AuthorDate: Thu, 15 Jun 2023 03:59:47 -03:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 16 Jun 2023 22:08:09 +02:00
trace,smp: Add tracepoints for scheduling remotelly called functions
Add a tracepoint for when a CSD is queued to a remote CPU's call_single_queue. This allows finding exactly which CPU queued a given CSD when looking at a csd_function_{entry,exit} event, and also enables us to accurately measure IPI delivery time with e.g. a synthetic event:
$ echo 'hist:keys=cpu,csd.hex:ts=common_timestamp.usecs' >\
/sys/kernel/tracing/events/smp/csd_queue_cpu/trigger
$ echo 'csd_latency unsigned int dst_cpu; unsigned long csd; u64 time' >\
/sys/kernel/tracing/synthetic_events
$ echo \
'hist:keys=common_cpu,csd.hex:'\
'time=common_timestamp.usecs-$ts:'\
'onmatch(smp.csd_queue_cpu).trace(csd_latency,common_cpu,csd,$time)' >\
/sys/kernel/tracing/events/smp/csd_function_entry/trigger$ trace-cmd record -e 'synthetic:csd_latency' hackbench $ trace-cmd report <...>-467 [001] 21.824263: csd_queue_cpu: cpu=0 callsite=try_to_wake_up+0x2ea func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-467 [001] 21.824280: ipi_send_cpu: cpu=0 callsite=try_to_wake_up+0x2ea callback=generic_smp_call_function_single_interrupt+0x0 <...>-489 [000] 21.824299: csd_function_entry: func=sched_ttwu_pending csd=0xffff8880076148b8 <...>-489 [000] 21.824320: csd_latency: dst_cpu=0, csd=18446612682193848504, time=36
--- include/trace/events/csd.h | 27 +++++++++++++++++++++++++++ kernel/smp.c | 16 +++++----------- 2 files changed, 32 insertions(+), 11 deletions(-)
include/trace/events/csd.h +27 -0
@@ -7,6 +7,33 @@#include <linux/tracepoint.h> +TRACE_EVENT (csd_queue_cpu, + + TP_PROTO(const unsigned int cpu, + unsigned long callsite, + smp_call_func_t func, + struct __call_single_data *csd), + +TP_ARGS (cpu, callsite, func, csd), + +TP_STRUCT__entry ( + __field(unsigned int, cpu) + __field(void *, callsite) + __field(void *, func) + __field(void *, csd) + ), + +TP_fast_assign ( + __entry->cpu = cpu; + __entry->callsite = (void *)callsite; + __entry->func = func; + __entry->csd = csd; + ), + +TP_printk ("cpu=%u callsite=%pS func=%ps csd=%p" , + __entry->cpu, __entry->callsite, __entry->func, __entry->csd) + ); +/* * Tracepoints for a function which is called as an effect of smp_call_function.* */
kernel/smp.c +5 -11
@@ -340,7 +340,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)* even if we haven't sent the smp_call IPI yet (e.g. the stopper * executes migration_cpu_stop() on the remote CPU). */ -if (trace_ipi_send_cpu_enabled ()) { +if (trace_csd_queue_cpu_enabled ()) {call_single_data_t *csd;smp_call_func_t func;
@@ -348,7 +348,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) func =CSD_TYPE (csd) ==CSD_TYPE_TTWU ? sched_ttwu_pending : csd->func; -trace_ipi_send_cpu (cpu, _RET_IP_, func); +trace_csd_queue_cpu (cpu, _RET_IP_, func, csd); }/*
@@ -741,7 +741,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask,int cpu, last_cpu, this_cpu =smp_processor_id ();struct call_function_data *cfd;bool wait = scf_flags &SCF_WAIT ; -int nr_cpus =0 , nr_queued =0 ; +int nr_cpus =0 ;bool run_remote = false;bool run_local = false;
@@ -799,22 +799,16 @@ static void smp_call_function_many_cond(const struct cpumask *mask, csd->node.src =smp_processor_id (); csd->node.dst = cpu;#endif +trace_csd_queue_cpu (cpu, _RET_IP_, func, csd); +if (llist_add (&csd->node.llist, &per_cpu (call_single_queue, cpu))) {__cpumask_set_cpu (cpu, cfd->cpumask_ipi); nr_cpus++; last_cpu = cpu; } - nr_queued++; }/* -* Trace each smp_function_call_*() as an IPI, actual IPIs -* will be traced with func==generic_smp_call_function_single_ipi(). -*/ -if (nr_queued) -trace_ipi_send_cpumask (cfd->cpumask, _RET_IP_, func); - -/* * Choose the most efficient way to send an IPI. Note that the * number of CPUs might be zero due to concurrent changes to the * provided mask.