v5 expedited "big hammer" RCU grace periods
Subsystems (4)
RFC PATCH 1/1 v5 expedited "big hammer" RCU grace periods
Fifth cut of "big hammer" expedited RCU grace periods. This uses per-CPU kthreads that are scheduled in parallel by a call to smp_call_function() by yet another kthread. The synchronize_sched(), synchronize_rcu(), and synchronize_bh() primitives wake this kthread up and then wait for it to force the grace period.
As before, this does nothing to expedite callbacks already registered with call_rcu() or call_rcu_bh(), but there is no need to. Just maps to synchronize_rcu() and a new synchronize_rcu_bh() on preemptable RCU, which has more complex grace-period detection -- this can be fixed later.
Passes light rcutorture testing. Grace periods take about 45 microseconds on an 8-CPU Power machine, which I believe is good enough from a performance viewpoint. Scalability may eventually need to be addressed in the smp_call_function() primitive and perhaps also in the scan through the CPUs that determines when all have completed. Although this is most definitely not ready for inclusion, it seems to be a promising approach.
Shortcomings:
o CPU hotplug testing would probably cause it to die horribly.
o It is not clear to me why I retained the single non-per-CPU
kthread, it can probably be dispensed with.
o The per-CPU kthreads do not boost themselves to real-time
priority, and thus could be blocked by real-time processes. Use of real-time priority might also speed things up a bit.
o Contains random debug statements.
o Does not address preemptable RCU.
Changes since v4:
o Use per-CPU kthreads to force the quiescent states in parallel.
Changes since v3:
o Use a kthread that schedules itself on each CPU in turn to
force a grace period. The synchronize_rcu() primitive wakes up the kthread in order to avoid messing with affinity masks on user tasks.
o Tried a number of additional variations on the v3 approach, none
of which helped much.
Changes since v2:
o Use reschedule IPIs rather than a softirq.
Changes since v1:
o Added rcutorture support, and added exports required by
rcutorture.
o Added comment stating that smp_call_function() implies a
memory barrier, suggested by Mathieu.
o Added #include for delay.h.
---
include/linux/rcuclassic.h | 16 ++ include/linux/rcupdate.h | 25 ++-- include/linux/rcupreempt.h | 10 + include/linux/rcutree.h | 13 +- kernel/rcupdate.c | 273 +++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupreempt.c | 1 kernel/rcutorture.c | 200 +++++++++++++++++--------------- 7 files changed, 432 insertions(+), 106 deletions(-)
include/linux/rcuclassic.h +15 -1
@@ -158,14 +158,28 @@ extern struct lockdep_map rcu_lock_map;#define call_rcu_sched (head, func) call_rcu(head, func) +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_sched_expedited (); +} + +static inline void synchronize_rcu_bh_expedited (void ) +{ +synchronize_sched_expedited (); +} +extern void __rcu_init (void ); -#define rcu_init_sched () do { } while (0)extern void rcu_check_callbacks (int cpu,int user);extern void rcu_restart_cpu (int cpu);extern long rcu_batches_completed (void );extern long rcu_batches_completed_bh (void ); +static inline void rcu_init_sched (void ) +{ +synchronize_sched_expedited_init (); +} +#define rcu_enter_nohz () do { } while (0)#define rcu_exit_nohz () do { } while (0)
include/linux/rcupdate.h +13 -12
@@ -51,7 +51,19 @@ struct rcu_head {void (*func)(struct rcu_head *head); }; -/* Internal to kernel, but needed by rcupreempt.h. */ +/* Exported common interfaces */ +extern void synchronize_rcu (void ); +extern void rcu_barrier (void ); +extern void rcu_barrier_bh (void ); +extern void rcu_barrier_sched (void ); +extern void synchronize_sched_expedited (void ); +extern int sched_expedited_torture_stats (char *page); + +/* Internal to kernel */ +extern void rcu_init (void ); +extern void rcu_scheduler_starting (void ); +extern void synchronize_sched_expedited_init (void ); +extern int rcu_needs_cpu (int cpu);extern int rcu_scheduler_active;#if defined(CONFIG_CLASSIC_RCU )
@@ -259,15 +271,4 @@ extern void call_rcu(struct rcu_head *head,extern void call_rcu_bh (struct rcu_head *head,void (*func)(struct rcu_head *head)); -/* Exported common interfaces */ -extern void synchronize_rcu (void ); -extern void rcu_barrier (void ); -extern void rcu_barrier_bh (void ); -extern void rcu_barrier_sched (void ); - -/* Internal to kernel */ -extern void rcu_init (void ); -extern void rcu_scheduler_starting (void ); -extern int rcu_needs_cpu (int cpu); -#endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +10 -0
@@ -74,6 +74,16 @@ extern int rcu_needs_cpu(int cpu);extern void __synchronize_sched (void ); +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_rcu ();/* Placeholder for new rcupreempt implementation. */ +} + +static inline void synchronize_rcu_bh_expedited (void ) +{ +synchronize_rcu ();/* Placeholder for new rcupreempt implementation. */ +} +extern void __rcu_init (void );extern void rcu_init_sched (void );extern void rcu_check_callbacks (int cpu,int user);
include/linux/rcutree.h +12 -1
@@ -279,8 +279,14 @@ static inline void __rcu_read_unlock_bh(void)#define call_rcu_sched (head, func) call_rcu(head, func) -static inline void rcu_init_sched (void ) +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_sched_expedited (); +} + +static inline void synchronize_rcu_bh_expedited (void ) { +synchronize_sched_expedited (); }extern void __rcu_init (void );
@@ -290,6 +296,11 @@ extern void rcu_restart_cpu(int cpu);extern long rcu_batches_completed (void );extern long rcu_batches_completed_bh (void ); +static inline void rcu_init_sched (void ) +{ +synchronize_sched_expedited_init (); +} +#ifdef CONFIG_NO_HZ void rcu_enter_nohz (void );void rcu_exit_nohz (void );
kernel/rcupdate.c +273 -0
294 lines not on this page · raw message
kernel/rcupreempt.c +1 -0
@@ -1507,6 +1507,7 @@ void __init rcu_init_sched(void)NULL , "rcu_sched_grace_period");WARN_ON (IS_ERR (rcu_sched_grace_period_task)); +synchronize_sched_expedited_init (); }#ifdef CONFIG_RCU_TRACE
kernel/rcutorture.c +108 -92
292 lines not on this page · raw message
Hi.
Quoted 5 lines · Paul E. McKenney
Fifth cut of "big hammer" expedited RCU grace periods. This uses per-CPU kthreads that are scheduled in parallel by a call to
smp_call_function()by yet another kthread. Thesynchronize_sched(),synchronize_rcu(), andsynchronize_bh()primitives wake this kthread up and then wait for it to force the grace period.
I'm curious, but doesn't the fact that registered 'barrier' callback is invoked mean grace period completion? I.e. why to bother with rescheduling, waiting for thread to complete and so on, when we only care in the fact that 'barrier' callback is invoked, and thus all previous ones are completed?
Or it is done just for the simplicity, since all rescheduling machinery already manages the rcu bits correctly, so you do not want to put it directly into 'barrier' callback?
Signature · 1 line
Evgeniy Polyakov
Quoted 1 line · Evgeniy Polyakov
Hi.
Hello, Evgeniy!
Quoted 7 lines · Paul E. McKenney
Quoted 5 lines
Fifth cut of "big hammer" expedited RCU grace periods. This uses per-CPU kthreads that are scheduled in parallel by a call to
smp_call_function()by yet another kthread. Thesynchronize_sched(),synchronize_rcu(), andsynchronize_bh()primitives wake this kthread up and then wait for it to force the grace period.
Quoted 8 lines
I'm curious, but doesn't the fact that registered 'barrier' callback is invoked mean grace period completion? I.e. why to bother with rescheduling, waiting for thread to complete and so on, when we only care in the fact that 'barrier' callback is invoked, and thus all previous ones are completed?
Or it is done just for the simplicity, since all rescheduling machinery already manages the rcu bits correctly, so you do not want to put it directly into 'barrier' callback?
It is a short-term expedient course of action. Longer term, I will drop rcuclassic in favor of rcutree, and then merge rcupreempt into rcutree. I will then add machinery to rcutree to handle expedited grace periods (somewhat) more naturally. Trying to expedite three very different RCU implementations seems a bit silly, hence the current off-on-the-side approach.
But even then I will avoid relying on a "barrier" callback, or, indeed, any sort of callback, because we don't want expedited grace periods to have to wait on invocation of earlier RCU callbacks. There will thus not be a call_rcu_expedited(), at least not unless someone comes up with a -really- compelling reason why.
But the exercise of going through several possible implementations was quite useful, as I learned a number of things that will improve the eventual rcutree implementation. Like the fact that expedited grace periods don't want to be waiting on invocation of prior callbacks. ;-)
And rcutiny is, as always, as special case. Here is the implementation of synchronize_rcu_expedited() in rcutiny:
void synchronize_rcu_expedited(void)
{
}Or even:
#define synchronize_rcu_expedited synchronize_rcu
;-)
Thanx, Paul
Quoted 11 lines · Paul E. McKenney
kernel/rcupdate.c +11
-0+void sched_expedited_wake (void *unused) +{ +mutex_lock (&__get_cpu_var (sched_expedited_done_mutex)); +if (__get_cpu_var (sched_expedited_done_qs) == +SCHED_EXPEDITED_QS_DONE_QS ) { +__get_cpu_var (sched_expedited_done_qs) = +SCHED_EXPEDITED_QS_NEED_QS ; +wake_up (&__get_cpu_var (sched_expedited_qs_wq)); + } +mutex_unlock (&__get_cpu_var (sched_expedited_done_mutex)); +}
[...]
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 382
Quoted 4 lines
kernel/rcupdate.c +4
-0+get_online_cpus (); +preempt_disable (); + mycpu =smp_processor_id (); +smp_call_function (sched_expedited_wake,NULL ,1 );
sched_expedited_wake() calls mutex_lock() which may sleep?
And I think you have re-implement workqueue.
Thanks, Lai.
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 359
Quoted 11 lines · Paul E. McKenney
kernel/rcupdate.c +11
-0+void sched_expedited_wake (void *unused) +{ +mutex_lock (&__get_cpu_var (sched_expedited_done_mutex)); +if (__get_cpu_var (sched_expedited_done_qs) == +SCHED_EXPEDITED_QS_DONE_QS ) { +__get_cpu_var (sched_expedited_done_qs) = +SCHED_EXPEDITED_QS_NEED_QS ; +wake_up (&__get_cpu_var (sched_expedited_qs_wq)); + } +mutex_unlock (&__get_cpu_var (sched_expedited_done_mutex)); +}
( hm, IPI handlers are supposed to be atomic. )
Quoted 13 lines
kernel/rcupdate.c +13
-0+/* +* Kernel thread that processes synchronize_sched_expedited() requests. +* This is implemented as a separate kernel thread to avoid the need +* to mess with other tasks' cpumasks. +*/ +static int krcu_sched_expedited(void *arg) +{ + int cpu; + int mycpu; + int nwait; + + do { + wait_event_interruptible(need_sched_expedited_wq,
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 383
Quoted 10 lines
kernel/rcupdate.c +10
-0+ need_sched_expedited); +smp_mb ();/* In case we didn't sleep. */ +if (!need_sched_expedited) +continue ; + need_sched_expedited =0 ; +get_online_cpus (); +preempt_disable (); + mycpu =smp_processor_id (); +smp_call_function (sched_expedited_wake,NULL ,1 ); +preempt_enable ();
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single wake_up() call? That would remove the SMP cross call (wakeups do immediate cross-calls already).
Even more - we already have a per-CPU, high RT priority helper thread that could be reused: the per CPU migration threads. Couldnt we queue these requests to them? RCU is arguably closely related to scheduling so there's no layering violation IMO.
There's already a struct migration_req machinery that performs something quite similar. (do work on behalf of another task, on a specific CPU, and then signal completion)
Also, per CPU workqueues have similar features as well.
Ingo
Quoted 15 lines · Lai Jiangshan
Quoted 11 lines · Paul E. McKenney
kernel/rcupdate.c +11
-0+void sched_expedited_wake (void *unused) +{ +mutex_lock (&__get_cpu_var (sched_expedited_done_mutex)); +if (__get_cpu_var (sched_expedited_done_qs) == +SCHED_EXPEDITED_QS_DONE_QS ) { +__get_cpu_var (sched_expedited_done_qs) = +SCHED_EXPEDITED_QS_NEED_QS ; +wake_up (&__get_cpu_var (sched_expedited_qs_wq)); + } +mutex_unlock (&__get_cpu_var (sched_expedited_done_mutex)); +}[...]
Quoted 6 lines
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 382
Quoted 4 lines
kernel/rcupdate.c +4
-0+get_online_cpus (); +preempt_disable (); + mycpu =smp_processor_id (); +smp_call_function (sched_expedited_wake,NULL ,1 );
sched_expedited_wake()callsmutex_lock()which may sleep?
Good eyes! Fixing this and the failure to release this lock in krcu_sched_expedited_percpu() allows it to survive 10 hours of rcutorture running in parallel with onlining/offlining random CPUs.
Quoted 1 line
And I think you have re-implement workqueue.
Hmmm... I suppose I could use schedule_work(), though I am concerned about interference from other work. But I will give this some thought.
Thanx, Paul
Quoted 16 lines · Ingo Molnar
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 359
Quoted 11 lines · Paul E. McKenney
kernel/rcupdate.c +11
-0+void sched_expedited_wake (void *unused) +{ +mutex_lock (&__get_cpu_var (sched_expedited_done_mutex)); +if (__get_cpu_var (sched_expedited_done_qs) == +SCHED_EXPEDITED_QS_DONE_QS ) { +__get_cpu_var (sched_expedited_done_qs) = +SCHED_EXPEDITED_QS_NEED_QS ; +wake_up (&__get_cpu_var (sched_expedited_qs_wq)); + } +mutex_unlock (&__get_cpu_var (sched_expedited_done_mutex)); +}( hm, IPI handlers are supposed to be atomic. )
<red face>
Quoted 20 lines
Quoted 20 lines
kernel/rcupdate.c +20
-0+/* +* Kernel thread that processes synchronize_sched_expedited() requests. +* This is implemented as a separate kernel thread to avoid the need +* to mess with other tasks' cpumasks. +*/ +static int krcu_sched_expedited (void *arg) +{ +int cpu; +int mycpu; +int nwait; + + do { + wait_event_interruptible(need_sched_expedited_wq, + need_sched_expedited); + smp_mb(); /* In case we didn't sleep. */ + if (!need_sched_expedited) + continue; + need_sched_expedited = 0; + get_online_cpus(); + preempt_disable();
Quoted 8 lines
kernel/rcupdate.c void rcu_scheduler_starting(void) · line 383
Quoted 3 lines
kernel/rcupdate.c +3
-0+ mycpu =smp_processor_id (); +smp_call_function (sched_expedited_wake,NULL ,1 ); +preempt_enable ();i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).
My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that smp_call_function() traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.
Quoted 10 lines
Even more - we already have a per-CPU, high RT priority helper thread that could be reused: the per CPU migration threads. Couldnt we queue these requests to them? RCU is arguably closely related to scheduling so there's no layering violation IMO.
There's already a struct
migration_reqmachinery that performs something quite similar. (do work on behalf of another task, on a specific CPU, and then signal completion)Also, per CPU workqueues have similar features as well.
Good points!!!
I will post a working patch using my current approach, then try out some of these approaches.
Thanx, Paul
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.
At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Ingo
Quoted 14 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.
Quoted 5 lines
At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Ah, so you are suggesting that I use smp_call_function() to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.
Thanx, Paul
Quoted 21 lines · Paul E. McKenney
Quoted 14 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.Quoted 5 lines
At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.
My suggestion was to queue up a dummy 'struct migration_req' up with it (change migration_req::task == NULL to mean 'nothing') and simply wake it up using wake_up_process().
That will force a quiescent state, without the need for any extra information, right?
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread);
wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into PER_CPU, and a loop could be done like this:
for_each_online_cpu(cpu)
wake_up_process(cpu_rq(cpu)->migration_thread);
for_each_online_cpu(cpu)
wait_for_completion(&per_cpu(req, cpu).done);hm?
Ingo
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().
OK. I was thinking of just using wake_up_process() without the migration_req structure, and unconditionally setting a per-CPU variable from within migration_thread() just before the list_empty() check. In your approach we would need a NULL-pointer check just before the call to __migrate_task().
Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 8 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:
Quoted 7 lines
for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Thanx, Paul
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);
Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
Also, no matter what scheme we use, the target CPU does have to be processed somehow and it does have to signal completion back somehow
- which generates cachemisses.
I think what probaby matters most is to go simple, and to use established kernel primitives - and the above is really typical pattern for things like TLB flushes to a process having a presence on every physical CPU. Those aspects will be kept reasonably fast and balanced on all hardware that matters. (and if not, people will notice any TLB flush/shootdown linear slowdowns and will address it)
I could be wrong though ... maybe someone can get some numbers from a really large system?
Ingo
Quoted 68 lines · Ingo Molnar
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Quoted 4 lines
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
With the current code, agreed. One could imagine making an IPI tree, so that a given CPU IPIs (say) eight subordinates. Making this work nice with CPU hotplug would be entertaining, to say the least.
Quoted 3 lines
Also, no matter what scheme we use, the target CPU does have to be processed somehow and it does have to signal completion back somehow
- which generates cachemisses.
One could in theory use a combining tree, so that results filter up, sort of like they do in rcutree. But given that rcutree already has a combining tree, I would like to do this part in rcutree.
Quoted 9 lines
I think what probaby matters most is to go simple, and to use established kernel primitives - and the above is really typical pattern for things like TLB flushes to a process having a presence on every physical CPU. Those aspects will be kept reasonably fast and balanced on all hardware that matters. (and if not, people will notice any TLB flush/shootdown linear slowdowns and will address it)
I could be wrong though ... maybe someone can get some numbers from a really large system?
In theory, I have access to a 64-way system. In practice, it is extremely heavily booked.
I will try your straightforward approach.
Thanx, Paul
Quoted 74 lines · Paul E. McKenney
Quoted 68 lines · Ingo Molnar
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Quoted 4 lines
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
Quoted 4 lines
With the current code, agreed. One could imagine making an IPI tree, so that a given CPU IPIs (say) eight subordinates. Making this work nice with CPU hotplug would be entertaining, to say the least.
Certainly! :-)
As a general note, unrelated to your patches: i think our CPU-hotplug related complexity seems to be a bit too much. This is really just a gut feeling - from having seen many patches that also have hotplug notifiers.
I'm wondering whether this is because it's structured in a suboptimal way, or because i'm (intuitively) under-estimating the complexity of what it takes to express what happens when a CPU is offlined and then onlined?
Ingo
Quoted 89 lines · Ingo Molnar
Quoted 74 lines · Paul E. McKenney
Quoted 68 lines · Ingo Molnar
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Quoted 4 lines
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
Quoted 4 lines
With the current code, agreed. One could imagine making an IPI tree, so that a given CPU IPIs (say) eight subordinates. Making this work nice with CPU hotplug would be entertaining, to say the least.
Certainly! :-)
As a general note, unrelated to your patches: i think our CPU-hotplug related complexity seems to be a bit too much. This is really just a gut feeling - from having seen many patches that also have hotplug notifiers.
Quoted 4 lines
I'm wondering whether this is because it's structured in a suboptimal way, or because i'm (intuitively) under-estimating the complexity of what it takes to express what happens when a CPU is offlined and then onlined?
I suppose that I could take this as a cue to reminisce about the old days in a past life with a different implementation of CPU online/offline, but life is just too short for that sort of thing. Not that guys my age let that stop them. ;-)
And in that past life, exercising CPU online/offline usually exposed painful bugs in new code, so I cannot claim that the old-life approach to CPU hotplug was perfect. Interestingly enough, running uniprocessor also exposed painful bugs more often than not. Of course, the only way to run uniprocessor was to offline all but one of the CPUs, so you would hit the online/offline bugs before hitting the uniprocessor-only bugs.
The thing that worries me most about CPU hotplug in Linux is that there is no clear hierarchy of CPU function in the offline process, given that the offlining process invokes notifiers in the same order as does the onlining process. Whether this is a real defect in the CPU hotplug design or is instead simply a symptom of my not yet being fully comfortable with the two-phase CPU-removal process is an interesting question to which I do not have an answer.
Either way, the thought process is different. In my old life, CPUs shed roles in the opposite order that they acquired them. This meant that a given CPU was naturally guaranteed to be correctly taking interrupts for the entire time that it was capable of running user-level processes. Later in the offlining process, it would still take interrupts, but would be unable to run user processes. Still later, it would no longer be taking interrupts, and would stop participating in RCU and in the global TLB-flush algorithm. There was no need to stop the whole machine to make a given CPU go offline, in fact, most of the work was done by the CPU in question.
In the case of RCU, this meant that there was no need for double-checking for offlined CPUs, because CPUs could reliably indicate a quiescent state on their way out.
On the other hand, there was no equivalent of dynticks in the old days. And it is dynticks that is responsible for most of the complexity present in force_quiescent_state(), not CPU hotplug.
So I cannot hold up RCU as something that would be greatly simplified by changing the CPU hotplug design, much as I might like to. ;-)
Thanx, Paul
Quoted 108 lines · Paul E. McKenney
Quoted 89 lines · Ingo Molnar
Quoted 74 lines · Paul E. McKenney
Quoted 68 lines · Ingo Molnar
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Quoted 4 lines
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
Quoted 4 lines
With the current code, agreed. One could imagine making an IPI tree, so that a given CPU IPIs (say) eight subordinates. Making this work nice with CPU hotplug would be entertaining, to say the least.
Certainly! :-)
As a general note, unrelated to your patches: i think our CPU-hotplug related complexity seems to be a bit too much. This is really just a gut feeling - from having seen many patches that also have hotplug notifiers.
Quoted 4 lines
I'm wondering whether this is because it's structured in a suboptimal way, or because i'm (intuitively) under-estimating the complexity of what it takes to express what happens when a CPU is offlined and then onlined?
I suppose that I could take this as a cue to reminisce about the old days in a past life with a different implementation of CPU online/offline, but life is just too short for that sort of thing. Not that guys my age let that stop them. ;-)
And in that past life, exercising CPU online/offline usually exposed painful bugs in new code, so I cannot claim that the old-life approach to CPU hotplug was perfect. Interestingly enough, running uniprocessor also exposed painful bugs more often than not. Of course, the only way to run uniprocessor was to offline all but one of the CPUs, so you would hit the online/offline bugs before hitting the uniprocessor-only bugs.
Quoted 8 lines
The thing that worries me most about CPU hotplug in Linux is that there is no clear hierarchy of CPU function in the offline process, given that the offlining process invokes notifiers in the same order as does the onlining process. Whether this is a real defect in the CPU hotplug design or is instead simply a symptom of my not yet being fully comfortable with the two-phase CPU-removal process is an interesting question to which I do not have an answer.
I strongly believe it's the former.
Quoted 2 lines
Either way, the thought process is different. In my old life, CPUs shed roles in the opposite order that they acquired them.
Yeah, that looks a whole lot more logical to do.
Quoted 14 lines
This meant that a given CPU was naturally guaranteed to be correctly taking interrupts for the entire time that it was capable of running user-level processes. Later in the offlining process, it would still take interrupts, but would be unable to run user processes. Still later, it would no longer be taking interrupts, and would stop participating in RCU and in the global TLB-flush algorithm. There was no need to stop the whole machine to make a given CPU go offline, in fact, most of the work was done by the CPU in question.
In the case of RCU, this meant that there was no need for double-checking for offlined CPUs, because CPUs could reliably indicate a quiescent state on their way out.
Quoted 7 lines
On the other hand, there was no equivalent of dynticks in the old days. And it is dynticks that is responsible for most of the complexity present in
force_quiescent_state(), not CPU hotplug.So I cannot hold up RCU as something that would be greatly simplified by changing the CPU hotplug design, much as I might like to. ;-)
We could probably remove a fair bit of dynticks complexity by removing non-dynticks and removing non-hrtimer. People could still force a 'periodic' interrupting mode (if they want, or if their hw forces that), but that would be a plain periodic hrtimer firing off all the time.
Ingo
Quoted 120 lines · Ingo Molnar
Quoted 108 lines · Paul E. McKenney
Quoted 89 lines · Ingo Molnar
Quoted 74 lines · Paul E. McKenney
Quoted 68 lines · Ingo Molnar
Quoted 58 lines · Paul E. McKenney
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · Paul E. McKenney
Quoted 19 lines · Ingo Molnar
Quoted 10 lines · Paul E. McKenney
Quoted 4 lines
i might be missing something fundamental here, but why not just have per CPU helper threads, all on the same waitqueue, and wake them up via a single
wake_up()call? That would remove the SMP cross call (wakeups do immediate cross-calls already).My concern with this is that the cache misses accessing all the processes on this single waitqueue would be serialized, slowing things down. In contrast, the bitmask that
smp_call_function()traverses delivers on the order of a thousand CPUs' worth of bits per cache miss. I will give it a try, though.At least if you go via the migration threads, you can queue up requests to them locally. But there's going to be cachemisses _anyway_, since you have to access them all from a single CPU, and then they have to fetch details about what to do, and then have to notify the originator about completion.
Quoted 7 lines
Quoted 3 lines
Ah, so you are suggesting that I use
smp_call_function()to run code on each CPU that wakes up that CPU's migration thread? I will take a look at this.My suggestion was to queue up a dummy 'struct
migration_req' up with it (changemigration_req::task== NULL to mean 'nothing') and simply wake it up usingwake_up_process().OK. I was thinking of just using
wake_up_process()without themigration_reqstructure, and unconditionally setting a per-CPU variable from withinmigration_thread()just before thelist_empty()check. In your approach we would need a NULL-pointer check just before the call to__migrate_task().Quoted 2 lines
That will force a quiescent state, without the need for any extra information, right?
Yep!
Quoted 14 lines
This is what the scheduler code does, roughly:
wake_up_process(rq->migration_thread); wait_for_completion(&req.done);and this will always have to perform well. The 'req' could be put into
PER_CPU, and a loop could be done like this:for_each_online_cpu(cpu) wake_up_process(cpu_rq(cpu)->migration_thread); for_each_online_cpu(cpu) wait_for_completion(&per_cpu(req, cpu).done);Quoted 6 lines
Quoted 1 line
hm?
My concern is the linear slowdown for large systems, but this should be OK for modest systems (a few 10s of CPUs). However, I will try it out -- it does not need to be a long-term solution, after all.
Quoted 4 lines
I think there is going to be a linear slowdown no matter what - because sending that many IPIs is going to be linear. (there are no 'broadcast to all' IPIs anymore - on x86 we only have them if all physical APIC IDs are 7 or smaller.)
Quoted 4 lines
With the current code, agreed. One could imagine making an IPI tree, so that a given CPU IPIs (say) eight subordinates. Making this work nice with CPU hotplug would be entertaining, to say the least.
Certainly! :-)
As a general note, unrelated to your patches: i think our CPU-hotplug related complexity seems to be a bit too much. This is really just a gut feeling - from having seen many patches that also have hotplug notifiers.
Quoted 4 lines
I'm wondering whether this is because it's structured in a suboptimal way, or because i'm (intuitively) under-estimating the complexity of what it takes to express what happens when a CPU is offlined and then onlined?
I suppose that I could take this as a cue to reminisce about the old days in a past life with a different implementation of CPU online/offline, but life is just too short for that sort of thing. Not that guys my age let that stop them. ;-)
And in that past life, exercising CPU online/offline usually exposed painful bugs in new code, so I cannot claim that the old-life approach to CPU hotplug was perfect. Interestingly enough, running uniprocessor also exposed painful bugs more often than not. Of course, the only way to run uniprocessor was to offline all but one of the CPUs, so you would hit the online/offline bugs before hitting the uniprocessor-only bugs.
Quoted 8 lines
The thing that worries me most about CPU hotplug in Linux is that there is no clear hierarchy of CPU function in the offline process, given that the offlining process invokes notifiers in the same order as does the onlining process. Whether this is a real defect in the CPU hotplug design or is instead simply a symptom of my not yet being fully comfortable with the two-phase CPU-removal process is an interesting question to which I do not have an answer.
Quoted 6 lines
I strongly believe it's the former.
Quoted 2 lines
Either way, the thought process is different. In my old life, CPUs shed roles in the opposite order that they acquired them.
Yeah, that looks a whole lot more logical to do.
Hmmm... Making the transition work nicely would require some thought. It might be good to retain the two-phase nature, even when reversing the order of offline notifications. This would address one disadvantage of the past-life version, which was unnecessary migration of processes off of the CPU in question, only to find that a later notifier aborted the offlining.
So only the first phase is permitted to abort the offlining of the CPU, and this first phase must also set whatever state is necessary to prevent some later operation from making it impossible to offline the CPU. The second phase would unconditionally take the CPU out of service. In theory, this approach would allow incremental conversion of the notifiers, waiting to remove the stop_machine stuff until all notifiers had been converted.
If this actually works out, the sequence of changes would be as follows:
1. Reverse the order of the offline notifications, fixing any
bugs induced/exposed by this change.
2. Incrementally convert notifiers to the new mechanism. This
will require more thought.
3. Get rid of the stop_machine and the CPU_DEAD once all are
converted.
Or we might find that simply reversing the order (#1 above) suffices.
Quoted 22 lines
Quoted 14 lines
This meant that a given CPU was naturally guaranteed to be correctly taking interrupts for the entire time that it was capable of running user-level processes. Later in the offlining process, it would still take interrupts, but would be unable to run user processes. Still later, it would no longer be taking interrupts, and would stop participating in RCU and in the global TLB-flush algorithm. There was no need to stop the whole machine to make a given CPU go offline, in fact, most of the work was done by the CPU in question.
In the case of RCU, this meant that there was no need for double-checking for offlined CPUs, because CPUs could reliably indicate a quiescent state on their way out.
Quoted 7 lines
On the other hand, there was no equivalent of dynticks in the old days. And it is dynticks that is responsible for most of the complexity present in
force_quiescent_state(), not CPU hotplug.So I cannot hold up RCU as something that would be greatly simplified by changing the CPU hotplug design, much as I might like to. ;-)
Quoted 5 lines
We could probably remove a fair bit of dynticks complexity by removing non-dynticks and removing non-hrtimer. People could still force a 'periodic' interrupting mode (if they want, or if their hw forces that), but that would be a plain periodic hrtimer firing off all the time.
Hmmm... That would not simplify RCU much, but on the other hand (1) the rcutree.c dynticks approach is already quite a bit simpler than the rcupreempt.c approach and (2) doing this could potentially simplify other things.
Thanx, Paul
Quoted 7 lines · Paul E. McKenney
Hmmm... Making the transition work nicely would require some thought. It might be good to retain the two-phase nature, even when reversing the order of offline notifications. This would address one disadvantage of the past-life version, which was unnecessary migration of processes off of the CPU in question, only to find that a later notifier aborted the offlining.
The notifiers handling CPU_DEAD cannot abort it from here since the operation has already completed, whether they like it or not!
If there exist notifiers which try to abort it from here, it's a BUG, as the code says:
/* CPU is completely dead: tell everyone. Too late to complain.
* */
if (raw_notifier_call_chain(&cpu_chain, CPU_DEAD | mod,
hcpu) == NOTIFY_BAD)
BUG();Also, one can thus consider the CPU_DEAD and the CPU_POST_DEAD parts to be the extensions of the second phase. Just that we do some additional cleanup once the CPU has actually gone down. migration of processes (while breaking their affinity if required) is one of them.
But there are other things as well, such as rebuilding the sched-domain which have to be done after the cpu has gone down. Currently this operation contributes to majority of time taken to bring a cpu-offline.
Quoted 10 lines
So only the first phase is permitted to abort the offlining of the CPU, and this first phase must also set whatever state is necessary to prevent some later operation from making it impossible to offline the CPU. The second phase would unconditionally take the CPU out of service. In theory, this approach would allow incremental conversion of the notifiers, waiting to remove the
stop_machinestuff until all notifiers had been converted.
If this actually works out, the sequence of changes would be as follows:
Quoted 8 lines
1. Reverse the order of the offline notifications, fixing any
bugs induced/exposed by this change.2. Incrementally convert notifiers to the new mechanism. This
will require more thought.3. Get rid of the
stop_machineand theCPU_DEADonce all are
converted.
I agree with this sequence. It seems quite logical.
However, I am not yet sure if we can completely get rid of stop_machine and CPU_DEAD in practice, unless we're okay with having an time-consuming rollback operation. Currently the rollback only consists of rolling back the actions done during CPU_UP_PREPARE/CPU_DOWN_PREPARE.
And from the notifiers profile (see attached file), UP_PREPARE/DOWN_PREPARE seem to consume a lot lesser time when compared to the post-hotplug notifications.
Quoted 43 lines
Or we might find that simply reversing the order (#1 above) suffices.
Quoted 22 lines
Quoted 14 lines
This meant that a given CPU was naturally guaranteed to be correctly taking interrupts for the entire time that it was capable of running user-level processes. Later in the offlining process, it would still take interrupts, but would be unable to run user processes. Still later, it would no longer be taking interrupts, and would stop participating in RCU and in the global TLB-flush algorithm. There was no need to stop the whole machine to make a given CPU go offline, in fact, most of the work was done by the CPU in question.
In the case of RCU, this meant that there was no need for double-checking for offlined CPUs, because CPUs could reliably indicate a quiescent state on their way out.
Quoted 7 lines
On the other hand, there was no equivalent of dynticks in the old days. And it is dynticks that is responsible for most of the complexity present in
force_quiescent_state(), not CPU hotplug.So I cannot hold up RCU as something that would be greatly simplified by changing the CPU hotplug design, much as I might like to. ;-)
Quoted 5 lines
We could probably remove a fair bit of dynticks complexity by removing non-dynticks and removing non-hrtimer. People could still force a 'periodic' interrupting mode (if they want, or if their hw forces that), but that would be a plain periodic hrtimer firing off all the time.
Hmmm... That would not simplify RCU much, but on the other hand (1) the
rcutree.cdynticks approach is already quite a bit simpler than thercupreempt.capproach and (2) doing this could potentially simplify other things.Thanx, Paul
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Signature · 2 lines
Thanks and Regards gautham
=============================================================================
statistics for CPU_DOWN_PREPARE
=============================================================================
410 ns: buffer_cpu_notify : CPU_DOWN_PREPARE
441 ns: radix_tree_callback : CPU_DOWN_PREPARE
473 ns: relay_hotcpu_callback : CPU_DOWN_PREPARE
486 ns: blk_cpu_notify : CPU_DOWN_PREPARE
563 ns: cpu_callback : CPU_DOWN_PREPARE
579 ns: hotplug_hrtick : CPU_DOWN_PREPARE
594 ns: cpu_callback : CPU_DOWN_PREPARE
605 ns: cpu_numa_callback : CPU_DOWN_PREPARE
611 ns: hrtimer_cpu_notify : CPU_DOWN_PREPARE
625 ns: flow_cache_cpu : CPU_DOWN_PREPARE
625 ns: rcu_barrier_cpu_hotplug : CPU_DOWN_PREPARE
639 ns: hotplug_cfd : CPU_DOWN_PREPARE
641 ns: pageset_cpuup_callback : CPU_DOWN_PREPARE
656 ns: rb_cpu_notify : CPU_DOWN_PREPARE
670 ns: dev_cpu_callback : CPU_DOWN_PREPARE
670 ns: topology_cpu_callback : CPU_DOWN_PREPARE
672 ns: remote_softirq_cpu_notify : CPU_DOWN_PREPARE
715 ns: ratelimit_handler : CPU_DOWN_PREPARE
715 ns: rcu_cpu_notify : CPU_DOWN_PREPARE
717 ns: timer_cpu_notify : CPU_DOWN_PREPARE
730 ns: page_alloc_cpu_notify : CPU_DOWN_PREPARE
746 ns: cpu_callback : CPU_DOWN_PREPARE
821 ns: cpuset_track_online_cpus : CPU_DOWN_PREPARE
824 ns: slab_cpuup_callback : CPU_DOWN_PREPARE
849 ns: sysfs_cpu_notify : CPU_DOWN_PREPARE 884 ns: percpu_counter_hotcpu_callback: CPU_DOWN_PREPARE
961 ns: update_runtime : CPU_DOWN_PREPARE
1323 ns: migration_call : CPU_DOWN_PREPARE
1918 ns: vmstat_cpuup_callback : CPU_DOWN_PREPARE
2072 ns: workqueue_cpu_callback : CPU_DOWN_PREPARE
=========================================================================
Total time for CPU_DOWN_PREPARE = .023235000 ms
=========================================================================
=============================================================================
statistics for CPU_DYING
=============================================================================
365 ns: remote_softirq_cpu_notify : CPU_DYING
365 ns: topology_cpu_callback : CPU_DYING
381 ns: blk_cpu_notify : CPU_DYING
381 ns: cpu_callback : CPU_DYING
381 ns: relay_hotcpu_callback : CPU_DYING
381 ns: update_runtime : CPU_DYING
394 ns: dev_cpu_callback : CPU_DYING
395 ns: hotplug_cfd : CPU_DYING
395 ns: vmstat_cpuup_callback : CPU_DYING
397 ns: cpuset_track_online_cpus : CPU_DYING
397 ns: flow_cache_cpu : CPU_DYING
397 ns: pageset_cpuup_callback : CPU_DYING
397 ns: rb_cpu_notify : CPU_DYING
398 ns: hotplug_hrtick : CPU_DYING
410 ns: cpu_callback : CPU_DYING
410 ns: page_alloc_cpu_notify : CPU_DYING
411 ns: rcu_cpu_notify : CPU_DYING
412 ns: slab_cpuup_callback : CPU_DYING
412 ns: sysfs_cpu_notify : CPU_DYING
412 ns: timer_cpu_notify : CPU_DYING
426 ns: buffer_cpu_notify : CPU_DYING
426 ns: radix_tree_callback : CPU_DYING
441 ns: cpu_callback : CPU_DYING
442 ns: cpu_numa_callback : CPU_DYING
473 ns: ratelimit_handler : CPU_DYING 531 ns: percpu_counter_hotcpu_callback: CPU_DYING
562 ns: workqueue_cpu_callback : CPU_DYING
730 ns: rcu_barrier_cpu_hotplug : CPU_DYING
1536 ns: migration_call : CPU_DYING
1873 ns: hrtimer_cpu_notify : CPU_DYING
=========================================================================
Total time for CPU_DYING = .015331000 ms
=========================================================================
=============================================================================
statistics for CPU_DOWN_CANCELED
=============================================================================
=========================================================================
Total time for CPU_DOWN_CANCELED = 0 ms
=========================================================================
=============================================================================
statistics for __stop_machine
=============================================================================
357983 ns: __stop_machine :
=========================================================================
Total time for __stop_machine = .357983000 ms
=========================================================================
=============================================================================
statistics for CPU_DEAD
=============================================================================
350 ns: update_runtime : CPU_DEAD
379 ns: hotplug_hrtick : CPU_DEAD
381 ns: cpu_callback : CPU_DEAD
381 ns: rb_cpu_notify : CPU_DEAD
426 ns: hotplug_cfd : CPU_DEAD
426 ns: relay_hotcpu_callback : CPU_DEAD
441 ns: rcu_barrier_cpu_hotplug : CPU_DEAD
442 ns: remote_softirq_cpu_notify : CPU_DEAD
609 ns: ratelimit_handler : CPU_DEAD
625 ns: cpu_numa_callback : CPU_DEAD
684 ns: dev_cpu_callback : CPU_DEAD
686 ns: workqueue_cpu_callback : CPU_DEAD
838 ns: rcu_cpu_notify : CPU_DEAD
898 ns: pageset_cpuup_callback : CPU_DEAD
1202 ns: vmstat_cpuup_callback : CPU_DEAD
1295 ns: blk_cpu_notify : CPU_DEAD
1554 ns: buffer_cpu_notify : CPU_DEAD
2588 ns: hrtimer_cpu_notify : CPU_DEAD
3274 ns: radix_tree_callback : CPU_DEAD
5246 ns: timer_cpu_notify : CPU_DEAD
8587 ns: flow_cache_cpu : CPU_DEAD
8645 ns: topology_cpu_callback : CPU_DEAD
12454 ns: cpu_callback : CPU_DEAD
12650 ns: cpu_callback : CPU_DEAD 45727 ns: percpu_counter_hotcpu_callback: CPU_DEAD
55242 ns: page_alloc_cpu_notify : CPU_DEAD
56766 ns: sysfs_cpu_notify : CPU_DEAD
58241 ns: slab_cpuup_callback : CPU_DEAD
78250 ns: migration_call : CPU_DEAD
10784759 ns: cpuset_track_online_cpus : CPU_DEAD
=========================================================================
Total time for CPU_DEAD = 11.144046000 ms
=========================================================================
=============================================================================
statistics for CPU_POST_DEAD
=============================================================================
350 ns: cpu_callback : CPU_POST_DEAD
365 ns: blk_cpu_notify : CPU_POST_DEAD
365 ns: buffer_cpu_notify : CPU_POST_DEAD
365 ns: cpu_numa_callback : CPU_POST_DEAD
365 ns: dev_cpu_callback : CPU_POST_DEAD
365 ns: flow_cache_cpu : CPU_POST_DEAD
365 ns: hrtimer_cpu_notify : CPU_POST_DEAD
365 ns: page_alloc_cpu_notify : CPU_POST_DEAD
365 ns: rb_cpu_notify : CPU_POST_DEAD
365 ns: rcu_cpu_notify : CPU_POST_DEAD
365 ns: timer_cpu_notify : CPU_POST_DEAD
365 ns: update_runtime : CPU_POST_DEAD
366 ns: cpu_callback : CPU_POST_DEAD
366 ns: hotplug_cfd : CPU_POST_DEAD
366 ns: pageset_cpuup_callback : CPU_POST_DEAD
366 ns: radix_tree_callback : CPU_POST_DEAD
367 ns: hotplug_hrtick : CPU_POST_DEAD
367 ns: topology_cpu_callback : CPU_POST_DEAD
367 ns: vmstat_cpuup_callback : CPU_POST_DEAD
381 ns: cpu_callback : CPU_POST_DEAD
381 ns: cpuset_track_online_cpus : CPU_POST_DEAD
381 ns: relay_hotcpu_callback : CPU_POST_DEAD
381 ns: sysfs_cpu_notify : CPU_POST_DEAD
383 ns: rcu_barrier_cpu_hotplug : CPU_POST_DEAD
410 ns: remote_softirq_cpu_notify : CPU_POST_DEAD
412 ns: slab_cpuup_callback : CPU_POST_DEAD
442 ns: migration_call : CPU_POST_DEAD 457 ns: percpu_counter_hotcpu_callback: CPU_POST_DEAD
502 ns: ratelimit_handler : CPU_POST_DEAD
86200 ns: workqueue_cpu_callback : CPU_POST_DEAD
=========================================================================
Total time for CPU_POST_DEAD = .097260000 ms
=========================================================================
=============================================================================
statistics for CPU_UP_PREPARE
=============================================================================
336 ns: hotplug_hrtick : CPU_UP_PREPARE
350 ns: cpu_callback : CPU_UP_PREPARE
365 ns: blk_cpu_notify : CPU_UP_PREPARE
381 ns: vmstat_cpuup_callback : CPU_UP_PREPARE
410 ns: buffer_cpu_notify : CPU_UP_PREPARE
410 ns: radix_tree_callback : CPU_UP_PREPARE
426 ns: dev_cpu_callback : CPU_UP_PREPARE
426 ns: remote_softirq_cpu_notify : CPU_UP_PREPARE
428 ns: cpuset_track_online_cpus : CPU_UP_PREPARE
441 ns: sysfs_cpu_notify : CPU_UP_PREPARE
471 ns: hotplug_cfd : CPU_UP_PREPARE
472 ns: rb_cpu_notify : CPU_UP_PREPARE
473 ns: flow_cache_cpu : CPU_UP_PREPARE
486 ns: page_alloc_cpu_notify : CPU_UP_PREPARE
488 ns: hrtimer_cpu_notify : CPU_UP_PREPARE
488 ns: update_runtime : CPU_UP_PREPARE
502 ns: rcu_barrier_cpu_hotplug : CPU_UP_PREPARE 531 ns: percpu_counter_hotcpu_callback: CPU_UP_PREPARE
547 ns: ratelimit_handler : CPU_UP_PREPARE
594 ns: relay_hotcpu_callback : CPU_UP_PREPARE
1125 ns: rcu_cpu_notify : CPU_UP_PREPARE
1309 ns: pageset_cpuup_callback : CPU_UP_PREPARE
1947 ns: timer_cpu_notify : CPU_UP_PREPARE
5389 ns: cpu_numa_callback : CPU_UP_PREPARE
6379 ns: topology_cpu_callback : CPU_UP_PREPARE
6436 ns: slab_cpuup_callback : CPU_UP_PREPARE
19879 ns: cpu_callback : CPU_UP_PREPARE
20227 ns: cpu_callback : CPU_UP_PREPARE
33940 ns: migration_call : CPU_UP_PREPARE
143731 ns: workqueue_cpu_callback : CPU_UP_PREPARE
=========================================================================
Total time for CPU_UP_PREPARE = .249387000 ms
=========================================================================
=============================================================================
statistics for CPU_UP_CANCELED
=============================================================================
=========================================================================
Total time for CPU_UP_CANCELED = 0 ms
=========================================================================
=============================================================================
statistics for __cpu_up
=============================================================================
205868908 ns: __cpu_up :
=========================================================================
Total time for __cpu_up = 205.868908000 ms
=========================================================================
=============================================================================
statistics for CPU_STARTING
=============================================================================
350 ns: hotplug_cfd : CPU_STARTING
352 ns: cpu_callback : CPU_STARTING
352 ns: remote_softirq_cpu_notify : CPU_STARTING
363 ns: vmstat_cpuup_callback : CPU_STARTING
365 ns: cpu_callback : CPU_STARTING
365 ns: dev_cpu_callback : CPU_STARTING
365 ns: hotplug_hrtick : CPU_STARTING
365 ns: radix_tree_callback : CPU_STARTING
365 ns: rb_cpu_notify : CPU_STARTING
368 ns: update_runtime : CPU_STARTING
379 ns: cpu_callback : CPU_STARTING
379 ns: cpu_numa_callback : CPU_STARTING
380 ns: rcu_barrier_cpu_hotplug : CPU_STARTING
380 ns: relay_hotcpu_callback : CPU_STARTING
381 ns: hrtimer_cpu_notify : CPU_STARTING
381 ns: pageset_cpuup_callback : CPU_STARTING
381 ns: slab_cpuup_callback : CPU_STARTING
382 ns: flow_cache_cpu : CPU_STARTING
394 ns: blk_cpu_notify : CPU_STARTING
397 ns: buffer_cpu_notify : CPU_STARTING 397 ns: percpu_counter_hotcpu_callback: CPU_STARTING
397 ns: sysfs_cpu_notify : CPU_STARTING
397 ns: topology_cpu_callback : CPU_STARTING
410 ns: rcu_cpu_notify : CPU_STARTING
412 ns: page_alloc_cpu_notify : CPU_STARTING
426 ns: cpuset_track_online_cpus : CPU_STARTING
455 ns: ratelimit_handler : CPU_STARTING
471 ns: timer_cpu_notify : CPU_STARTING
516 ns: migration_call : CPU_STARTING
549 ns: workqueue_cpu_callback : CPU_STARTING
=========================================================================
Total time for CPU_STARTING = .011874000 ms
=========================================================================
=============================================================================
statistics for CPU_ONLINE
=============================================================================
365 ns: radix_tree_callback : CPU_ONLINE
379 ns: hotplug_hrtick : CPU_ONLINE
381 ns: hrtimer_cpu_notify : CPU_ONLINE
381 ns: remote_softirq_cpu_notify : CPU_ONLINE
410 ns: slab_cpuup_callback : CPU_ONLINE
410 ns: timer_cpu_notify : CPU_ONLINE
412 ns: blk_cpu_notify : CPU_ONLINE
426 ns: dev_cpu_callback : CPU_ONLINE
426 ns: flow_cache_cpu : CPU_ONLINE
426 ns: topology_cpu_callback : CPU_ONLINE
428 ns: rcu_barrier_cpu_hotplug : CPU_ONLINE
428 ns: rcu_cpu_notify : CPU_ONLINE
440 ns: buffer_cpu_notify : CPU_ONLINE
455 ns: pageset_cpuup_callback : CPU_ONLINE
457 ns: relay_hotcpu_callback : CPU_ONLINE
473 ns: rb_cpu_notify : CPU_ONLINE
518 ns: update_runtime : CPU_ONLINE
549 ns: cpu_numa_callback : CPU_ONLINE
562 ns: ratelimit_handler : CPU_ONLINE
595 ns: page_alloc_cpu_notify : CPU_ONLINE
596 ns: hotplug_cfd : CPU_ONLINE 777 ns: percpu_counter_hotcpu_callback: CPU_ONLINE
1037 ns: cpu_callback : CPU_ONLINE
1280 ns: cpu_callback : CPU_ONLINE
1680 ns: cpu_callback : CPU_ONLINE
2043 ns: vmstat_cpuup_callback : CPU_ONLINE
3422 ns: migration_call : CPU_ONLINE
12344 ns: workqueue_cpu_callback : CPU_ONLINE
52879 ns: sysfs_cpu_notify : CPU_ONLINE
12287706 ns: cpuset_track_online_cpus : CPU_ONLINE
=========================================================================
Total time for CPU_ONLINE = 12.372685000 ms
=========================================================================
Quoted 11 lines · Gautham R Shenoy
Quoted 7 lines · Paul E. McKenney
Hmmm... Making the transition work nicely would require some thought. It might be good to retain the two-phase nature, even when reversing the order of offline notifications. This would address one disadvantage of the past-life version, which was unnecessary migration of processes off of the CPU in question, only to find that a later notifier aborted the offlining.
The notifiers handling
CPU_DEADcannot abort it from here since the operation has already completed, whether they like it or not!
Hello, Gautham,
We are talking past each other -- the past-life (not Linux) CPU-offlining scheme had but one phase for offlining, which meant that if a very late notifier-equivalent realized that the offlining could not proceed, it would have mostly shut the CPU down, only to have to restart it.
For example, it might have needlessly migrated processes off of that CPU. This did not happen often, but it was a bit of a disadvantage.
Thanx, Paul
Quoted 374 lines
If there exist notifiers which try to abort it from here, it's a BUG, as the code says:
/* CPU is completely dead: tell everyone. Too late to complain. * */ if (raw_notifier_call_chain(&cpu_chain, CPU_DEAD | mod, hcpu) == NOTIFY_BAD) BUG();Also, one can thus consider the
CPU_DEADand theCPU_POST_DEADparts to be the extensions of the second phase. Just that we do some additional cleanup once the CPU has actually gone down. migration of processes (while breaking their affinity if required) is one of them.But there are other things as well, such as rebuilding the sched-domain which have to be done after the cpu has gone down. Currently this operation contributes to majority of time taken to bring a cpu-offline.
Quoted 10 lines
So only the first phase is permitted to abort the offlining of the CPU, and this first phase must also set whatever state is necessary to prevent some later operation from making it impossible to offline the CPU. The second phase would unconditionally take the CPU out of service. In theory, this approach would allow incremental conversion of the notifiers, waiting to remove the
stop_machinestuff until all notifiers had been converted.
If this actually works out, the sequence of changes would be as follows:Quoted 8 lines
1. Reverse the order of the offline notifications, fixing any
bugs induced/exposed by this change.2. Incrementally convert notifiers to the new mechanism. This
will require more thought.3. Get rid of the
stop_machineand theCPU_DEADonce all are
converted.I agree with this sequence. It seems quite logical.
However, I am not yet sure if we can completely get rid of
stop_machineandCPU_DEADin practice, unless we're okay with having an time-consuming rollback operation. Currently the rollback only consists of rolling back the actions done during CPU_UP_PREPARE/CPU_DOWN_PREPARE.And from the notifiers profile (see attached file), UP_PREPARE/DOWN_PREPARE seem to consume a lot lesser time when compared to the post-hotplug notifications.
Quoted 43 lines
Or we might find that simply reversing the order (#1 above) suffices.
Quoted 22 lines
Quoted 14 lines
This meant that a given CPU was naturally guaranteed to be correctly taking interrupts for the entire time that it was capable of running user-level processes. Later in the offlining process, it would still take interrupts, but would be unable to run user processes. Still later, it would no longer be taking interrupts, and would stop participating in RCU and in the global TLB-flush algorithm. There was no need to stop the whole machine to make a given CPU go offline, in fact, most of the work was done by the CPU in question.
In the case of RCU, this meant that there was no need for double-checking for offlined CPUs, because CPUs could reliably indicate a quiescent state on their way out.
Quoted 7 lines
On the other hand, there was no equivalent of dynticks in the old days. And it is dynticks that is responsible for most of the complexity present in
force_quiescent_state(), not CPU hotplug.So I cannot hold up RCU as something that would be greatly simplified by changing the CPU hotplug design, much as I might like to. ;-)
Quoted 5 lines
We could probably remove a fair bit of dynticks complexity by removing non-dynticks and removing non-hrtimer. People could still force a 'periodic' interrupting mode (if they want, or if their hw forces that), but that would be a plain periodic hrtimer firing off all the time.
Hmmm... That would not simplify RCU much, but on the other hand (1) the
rcutree.cdynticks approach is already quite a bit simpler than thercupreempt.capproach and (2) doing this could potentially simplify other things.Thanx, Paul
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/Signature · 281 lines
Thanks and Regards gautham ============================================================================= statistics for CPU_DOWN_PREPARE ============================================================================= 410 ns: buffer_cpu_notify : CPU_DOWN_PREPARE 441 ns: radix_tree_callback : CPU_DOWN_PREPARE 473 ns: relay_hotcpu_callback : CPU_DOWN_PREPARE 486 ns: blk_cpu_notify : CPU_DOWN_PREPARE 563 ns: cpu_callback : CPU_DOWN_PREPARE 579 ns: hotplug_hrtick : CPU_DOWN_PREPARE 594 ns: cpu_callback : CPU_DOWN_PREPARE 605 ns: cpu_numa_callback : CPU_DOWN_PREPARE 611 ns: hrtimer_cpu_notify : CPU_DOWN_PREPARE 625 ns: flow_cache_cpu : CPU_DOWN_PREPARE 625 ns: rcu_barrier_cpu_hotplug : CPU_DOWN_PREPARE 639 ns: hotplug_cfd : CPU_DOWN_PREPARE 641 ns: pageset_cpuup_callback : CPU_DOWN_PREPARE 656 ns: rb_cpu_notify : CPU_DOWN_PREPARE 670 ns: dev_cpu_callback : CPU_DOWN_PREPARE 670 ns: topology_cpu_callback : CPU_DOWN_PREPARE 672 ns: remote_softirq_cpu_notify : CPU_DOWN_PREPARE 715 ns: ratelimit_handler : CPU_DOWN_PREPARE 715 ns: rcu_cpu_notify : CPU_DOWN_PREPARE 717 ns: timer_cpu_notify : CPU_DOWN_PREPARE 730 ns: page_alloc_cpu_notify : CPU_DOWN_PREPARE 746 ns: cpu_callback : CPU_DOWN_PREPARE 821 ns: cpuset_track_online_cpus : CPU_DOWN_PREPARE 824 ns: slab_cpuup_callback : CPU_DOWN_PREPARE 849 ns: sysfs_cpu_notify : CPU_DOWN_PREPARE 884 ns: percpu_counter_hotcpu_callback: CPU_DOWN_PREPARE 961 ns: update_runtime : CPU_DOWN_PREPARE 1323 ns: migration_call : CPU_DOWN_PREPARE 1918 ns: vmstat_cpuup_callback : CPU_DOWN_PREPARE 2072 ns: workqueue_cpu_callback : CPU_DOWN_PREPARE ========================================================================= Total time for CPU_DOWN_PREPARE = .023235000 ms ========================================================================= ============================================================================= statistics for CPU_DYING ============================================================================= 365 ns: remote_softirq_cpu_notify : CPU_DYING 365 ns: topology_cpu_callback : CPU_DYING 381 ns: blk_cpu_notify : CPU_DYING 381 ns: cpu_callback : CPU_DYING 381 ns: relay_hotcpu_callback : CPU_DYING 381 ns: update_runtime : CPU_DYING 394 ns: dev_cpu_callback : CPU_DYING 395 ns: hotplug_cfd : CPU_DYING 395 ns: vmstat_cpuup_callback : CPU_DYING 397 ns: cpuset_track_online_cpus : CPU_DYING 397 ns: flow_cache_cpu : CPU_DYING 397 ns: pageset_cpuup_callback : CPU_DYING 397 ns: rb_cpu_notify : CPU_DYING 398 ns: hotplug_hrtick : CPU_DYING 410 ns: cpu_callback : CPU_DYING 410 ns: page_alloc_cpu_notify : CPU_DYING 411 ns: rcu_cpu_notify : CPU_DYING 412 ns: slab_cpuup_callback : CPU_DYING 412 ns: sysfs_cpu_notify : CPU_DYING 412 ns: timer_cpu_notify : CPU_DYING 426 ns: buffer_cpu_notify : CPU_DYING 426 ns: radix_tree_callback : CPU_DYING 441 ns: cpu_callback : CPU_DYING 442 ns: cpu_numa_callback : CPU_DYING 473 ns: ratelimit_handler : CPU_DYING 531 ns: percpu_counter_hotcpu_callback: CPU_DYING 562 ns: workqueue_cpu_callback : CPU_DYING 730 ns: rcu_barrier_cpu_hotplug : CPU_DYING 1536 ns: migration_call : CPU_DYING 1873 ns: hrtimer_cpu_notify : CPU_DYING ========================================================================= Total time for CPU_DYING = .015331000 ms ========================================================================= ============================================================================= statistics for CPU_DOWN_CANCELED ============================================================================= ========================================================================= Total time for CPU_DOWN_CANCELED = 0 ms ========================================================================= ============================================================================= statistics for __stop_machine ============================================================================= 357983 ns: __stop_machine : ========================================================================= Total time for __stop_machine = .357983000 ms ========================================================================= ============================================================================= statistics for CPU_DEAD ============================================================================= 350 ns: update_runtime : CPU_DEAD 379 ns: hotplug_hrtick : CPU_DEAD 381 ns: cpu_callback : CPU_DEAD 381 ns: rb_cpu_notify : CPU_DEAD 426 ns: hotplug_cfd : CPU_DEAD 426 ns: relay_hotcpu_callback : CPU_DEAD 441 ns: rcu_barrier_cpu_hotplug : CPU_DEAD 442 ns: remote_softirq_cpu_notify : CPU_DEAD 609 ns: ratelimit_handler : CPU_DEAD 625 ns: cpu_numa_callback : CPU_DEAD 684 ns: dev_cpu_callback : CPU_DEAD 686 ns: workqueue_cpu_callback : CPU_DEAD 838 ns: rcu_cpu_notify : CPU_DEAD 898 ns: pageset_cpuup_callback : CPU_DEAD 1202 ns: vmstat_cpuup_callback : CPU_DEAD 1295 ns: blk_cpu_notify : CPU_DEAD 1554 ns: buffer_cpu_notify : CPU_DEAD 2588 ns: hrtimer_cpu_notify : CPU_DEAD 3274 ns: radix_tree_callback : CPU_DEAD 5246 ns: timer_cpu_notify : CPU_DEAD 8587 ns: flow_cache_cpu : CPU_DEAD 8645 ns: topology_cpu_callback : CPU_DEAD 12454 ns: cpu_callback : CPU_DEAD 12650 ns: cpu_callback : CPU_DEAD 45727 ns: percpu_counter_hotcpu_callback: CPU_DEAD 55242 ns: page_alloc_cpu_notify : CPU_DEAD 56766 ns: sysfs_cpu_notify : CPU_DEAD 58241 ns: slab_cpuup_callback : CPU_DEAD 78250 ns: migration_call : CPU_DEAD 10784759 ns: cpuset_track_online_cpus : CPU_DEAD ========================================================================= Total time for CPU_DEAD = 11.144046000 ms ========================================================================= ============================================================================= statistics for CPU_POST_DEAD ============================================================================= 350 ns: cpu_callback : CPU_POST_DEAD 365 ns: blk_cpu_notify : CPU_POST_DEAD 365 ns: buffer_cpu_notify : CPU_POST_DEAD 365 ns: cpu_numa_callback : CPU_POST_DEAD 365 ns: dev_cpu_callback : CPU_POST_DEAD 365 ns: flow_cache_cpu : CPU_POST_DEAD 365 ns: hrtimer_cpu_notify : CPU_POST_DEAD 365 ns: page_alloc_cpu_notify : CPU_POST_DEAD 365 ns: rb_cpu_notify : CPU_POST_DEAD 365 ns: rcu_cpu_notify : CPU_POST_DEAD 365 ns: timer_cpu_notify : CPU_POST_DEAD 365 ns: update_runtime : CPU_POST_DEAD 366 ns: cpu_callback : CPU_POST_DEAD 366 ns: hotplug_cfd : CPU_POST_DEAD 366 ns: pageset_cpuup_callback : CPU_POST_DEAD 366 ns: radix_tree_callback : CPU_POST_DEAD 367 ns: hotplug_hrtick : CPU_POST_DEAD 367 ns: topology_cpu_callback : CPU_POST_DEAD 367 ns: vmstat_cpuup_callback : CPU_POST_DEAD 381 ns: cpu_callback : CPU_POST_DEAD 381 ns: cpuset_track_online_cpus : CPU_POST_DEAD 381 ns: relay_hotcpu_callback : CPU_POST_DEAD 381 ns: sysfs_cpu_notify : CPU_POST_DEAD 383 ns: rcu_barrier_cpu_hotplug : CPU_POST_DEAD 410 ns: remote_softirq_cpu_notify : CPU_POST_DEAD 412 ns: slab_cpuup_callback : CPU_POST_DEAD 442 ns: migration_call : CPU_POST_DEAD 457 ns: percpu_counter_hotcpu_callback: CPU_POST_DEAD 502 ns: ratelimit_handler : CPU_POST_DEAD 86200 ns: workqueue_cpu_callback : CPU_POST_DEAD ========================================================================= Total time for CPU_POST_DEAD = .097260000 ms ========================================================================= ============================================================================= statistics for CPU_UP_PREPARE ============================================================================= 336 ns: hotplug_hrtick : CPU_UP_PREPARE 350 ns: cpu_callback : CPU_UP_PREPARE 365 ns: blk_cpu_notify : CPU_UP_PREPARE 381 ns: vmstat_cpuup_callback : CPU_UP_PREPARE 410 ns: buffer_cpu_notify : CPU_UP_PREPARE 410 ns: radix_tree_callback : CPU_UP_PREPARE 426 ns: dev_cpu_callback : CPU_UP_PREPARE 426 ns: remote_softirq_cpu_notify : CPU_UP_PREPARE 428 ns: cpuset_track_online_cpus : CPU_UP_PREPARE 441 ns: sysfs_cpu_notify : CPU_UP_PREPARE 471 ns: hotplug_cfd : CPU_UP_PREPARE 472 ns: rb_cpu_notify : CPU_UP_PREPARE 473 ns: flow_cache_cpu : CPU_UP_PREPARE 486 ns: page_alloc_cpu_notify : CPU_UP_PREPARE 488 ns: hrtimer_cpu_notify : CPU_UP_PREPARE 488 ns: update_runtime : CPU_UP_PREPARE 502 ns: rcu_barrier_cpu_hotplug : CPU_UP_PREPARE 531 ns: percpu_counter_hotcpu_callback: CPU_UP_PREPARE 547 ns: ratelimit_handler : CPU_UP_PREPARE 594 ns: relay_hotcpu_callback : CPU_UP_PREPARE 1125 ns: rcu_cpu_notify : CPU_UP_PREPARE 1309 ns: pageset_cpuup_callback : CPU_UP_PREPARE 1947 ns: timer_cpu_notify : CPU_UP_PREPARE 5389 ns: cpu_numa_callback : CPU_UP_PREPARE 6379 ns: topology_cpu_callback : CPU_UP_PREPARE 6436 ns: slab_cpuup_callback : CPU_UP_PREPARE 19879 ns: cpu_callback : CPU_UP_PREPARE 20227 ns: cpu_callback : CPU_UP_PREPARE 33940 ns: migration_call : CPU_UP_PREPARE 143731 ns: workqueue_cpu_callback : CPU_UP_PREPARE ========================================================================= Total time for CPU_UP_PREPARE = .249387000 ms ========================================================================= ============================================================================= statistics for CPU_UP_CANCELED ============================================================================= ========================================================================= Total time for CPU_UP_CANCELED = 0 ms ========================================================================= ============================================================================= statistics for __cpu_up ============================================================================= 205868908 ns: __cpu_up : ========================================================================= Total time for __cpu_up = 205.868908000 ms ========================================================================= ============================================================================= statistics for CPU_STARTING ============================================================================= 350 ns: hotplug_cfd : CPU_STARTING 352 ns: cpu_callback : CPU_STARTING 352 ns: remote_softirq_cpu_notify : CPU_STARTING 363 ns: vmstat_cpuup_callback : CPU_STARTING 365 ns: cpu_callback : CPU_STARTING 365 ns: dev_cpu_callback : CPU_STARTING 365 ns: hotplug_hrtick : CPU_STARTING 365 ns: radix_tree_callback : CPU_STARTING 365 ns: rb_cpu_notify : CPU_STARTING 368 ns: update_runtime : CPU_STARTING 379 ns: cpu_callback : CPU_STARTING 379 ns: cpu_numa_callback : CPU_STARTING 380 ns: rcu_barrier_cpu_hotplug : CPU_STARTING 380 ns: relay_hotcpu_callback : CPU_STARTING 381 ns: hrtimer_cpu_notify : CPU_STARTING 381 ns: pageset_cpuup_callback : CPU_STARTING 381 ns: slab_cpuup_callback : CPU_STARTING 382 ns: flow_cache_cpu : CPU_STARTING 394 ns: blk_cpu_notify : CPU_STARTING 397 ns: buffer_cpu_notify : CPU_STARTING 397 ns: percpu_counter_hotcpu_callback: CPU_STARTING 397 ns: sysfs_cpu_notify : CPU_STARTING 397 ns: topology_cpu_callback : CPU_STARTING 410 ns: rcu_cpu_notify : CPU_STARTING 412 ns: page_alloc_cpu_notify : CPU_STARTING 426 ns: cpuset_track_online_cpus : CPU_STARTING 455 ns: ratelimit_handler : CPU_STARTING 471 ns: timer_cpu_notify : CPU_STARTING 516 ns: migration_call : CPU_STARTING 549 ns: workqueue_cpu_callback : CPU_STARTING ========================================================================= Total time for CPU_STARTING = .011874000 ms ========================================================================= ============================================================================= statistics for CPU_ONLINE ============================================================================= 365 ns: radix_tree_callback : CPU_ONLINE 379 ns: hotplug_hrtick : CPU_ONLINE 381 ns: hrtimer_cpu_notify : CPU_ONLINE 381 ns: remote_softirq_cpu_notify : CPU_ONLINE 410 ns: slab_cpuup_callback : CPU_ONLINE 410 ns: timer_cpu_notify : CPU_ONLINE 412 ns: blk_cpu_notify : CPU_ONLINE 426 ns: dev_cpu_callback : CPU_ONLINE 426 ns: flow_cache_cpu : CPU_ONLINE 426 ns: topology_cpu_callback : CPU_ONLINE 428 ns: rcu_barrier_cpu_hotplug : CPU_ONLINE 428 ns: rcu_cpu_notify : CPU_ONLINE 440 ns: buffer_cpu_notify : CPU_ONLINE 455 ns: pageset_cpuup_callback : CPU_ONLINE 457 ns: relay_hotcpu_callback : CPU_ONLINE 473 ns: rb_cpu_notify : CPU_ONLINE 518 ns: update_runtime : CPU_ONLINE 549 ns: cpu_numa_callback : CPU_ONLINE 562 ns: ratelimit_handler : CPU_ONLINE 595 ns: page_alloc_cpu_notify : CPU_ONLINE 596 ns: hotplug_cfd : CPU_ONLINE 777 ns: percpu_counter_hotcpu_callback: CPU_ONLINE 1037 ns: cpu_callback : CPU_ONLINE 1280 ns: cpu_callback : CPU_ONLINE 1680 ns: cpu_callback : CPU_ONLINE 2043 ns: vmstat_cpuup_callback : CPU_ONLINE 3422 ns: migration_call : CPU_ONLINE 12344 ns: workqueue_cpu_callback : CPU_ONLINE 52879 ns: sysfs_cpu_notify : CPU_ONLINE 12287706 ns: cpuset_track_online_cpus : CPU_ONLINE ========================================================================= Total time for CPU_ONLINE = 12.372685000 ms =========================================================================