nohz: Tick dependency mask v3
Subsystems (5)
cover nohz: Tick dependency mask v3
This series changes the nohz full infrastructure to let subsystems notify when they have a tick dependency. Checking if we can stop the tick is thus perfomed in a push rather than a pull model, see previous iterations:
http://lkml.kernel.org/r/[email protected]
Changes here:
* Reuse fetch_or()
* Change posix cpu timers to a per thread and signal dependency (we may want to merge all that to thread level only) instead of global dependency.
* Scheduler tick dependency now is evaluated at the CPU level instead of the task level. We don't anymore depend on the current task policy but on the whole tasks of the runqueue. That too can be further optimized.
Only lightly tested, just checking for current state opinions.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
timers/core-v5
HEAD: ca9e998d0be7e6ef1ffab888436cbabf5446c865
Thanks,
Frederic
---
Frederic Weisbecker (7):
atomic: Export fetch_or()
nohz: New tick dependency mask
perf: Migrate perf to use new tick dependency mask model
sched: Account rr and fifo tasks separately
sched: Migrate sched to use new tick dependency mask model
posix-cpu-timers: Migrate to use new tick dependency mask model
sched-clock: Migrate to use new tick dependency mask model
include/linux/atomic.h | 18 ++++++ include/linux/perf_event.h | 6 -- include/linux/posix-timers.h | 3 - include/linux/sched.h | 11 +++- include/linux/tick.h | 21 ++++++ kernel/events/core.c | 22 ++----- kernel/sched/clock.c | 5 ++ kernel/sched/core.c | 47 ++++++-------- kernel/sched/rt.c | 34 ++++++++++ kernel/sched/sched.h | 49 ++++++++++---- kernel/time/posix-cpu-timers.c | 52 ++++----------- kernel/time/tick-sched.c | 142 +++++++++++++++++++++++++++++++++-------- kernel/time/tick-sched.h | 1 + 13 files changed, 274 insertions(+), 137 deletions(-)
PATCH 1/7 atomic: Export fetch_or()
Export fetch_or() that's implemented and used internally by the scheduler. We are going to use it for NO_HZ so make it generally available.
--- include/linux/atomic.h | 18 ++++++++++++++++++ kernel/sched/core.c | 14 -------------- 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/atomic.h +18 -0
@@ -451,6 +451,24 @@ static inline int atomic_dec_if_positive(atomic_t *v) }#endif +/** +* fetch_or - perform *ptr |= mask and return old value of *ptr +* @ptr: pointer to value +* @mask: mask to OR on the value +* +* cmpxchg based fetch_or, macro so it works for different integer types +*/ +#define fetch_or (ptr, mask) \ +({ typeof(*(ptr)) __old, __val = *(ptr); \ + for (;;) { \ + __old = cmpxchg((ptr), __val, __val | (mask)); \ + if (__old == __val) \ + break; \ + __val = __old; \ + } \ + __old; \ +}) +#include <asm-generic/atomic-long.h> #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h>
kernel/sched/core.c +0 -14
@@ -453,20 +453,6 @@ static inline void init_hrtick(void) }#endif /* CONFIG_SCHED_HRTICK */ -/* -* cmpxchg based fetch_or, macro so it works for different integer types -*/ -#define fetch_or (ptr, val) \ -({ typeof(*(ptr)) __old, __val = *(ptr); \ - for (;;) { \ - __old = cmpxchg((ptr), __val, __val | (val)); \ - if (__old == __val) \ - break; \ - __val = __old; \ - } \ - __old; \ -}) -#if defined(CONFIG_SMP ) && defined(TIF_POLLING_NRFLAG )/* * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
Signature · 1 line
2.5.3
Re: [PATCH 1/7] atomic: Export fetch_or()
Quoted 37 lines · Frederic Weisbecker
Export
fetch_or()that's implemented and used internally by the scheduler. We are going to use it forNO_HZso make it generally available.Cc Christoph Lameter<[email protected]>Cc Chris Metcalf<[email protected]>Cc Ingo Molnar<[email protected]>Cc Luiz Capitulino<[email protected]>Cc Peter Zijlstra<[email protected]>Cc Rik van Riel<[email protected]>Cc Thomas Gleixner<[email protected]>Cc Viresh Kumar<[email protected]>Signed-off-by Frederic Weisbecker<[email protected]>--- include/linux/atomic.h | 18 ++++++++++++++++++ kernel/sched/core.c | 14 -------------- 2 files changed, 18 insertions(+), 14 deletions(-)include/linux/atomic.h +11
-0@@ -451,6 +451,24 @@ static inline int atomic_dec_if_positive(atomic_t *v) }#endif +/** +* fetch_or - perform *ptr |= mask and return old value of *ptr +* @ptr: pointer to value +* @mask: mask to OR on the value +* +* cmpxchg based fetch_or, macro so it works for different integer types +*/ +#define fetch_or (ptr, mask) \ +({ typeof(*(ptr)) __old, __val = *(ptr); \ + for (;;) { \ + __old = cmpxchg((ptr), __val, __val | (mask)); \
include/linux/atomic.h static inline int atomic_dec_if_positive(atomic_t *v) · line 474
Quoted 10 lines
include/linux/atomic.h +7
-0+if (__old == __val) \ +break ; \ + __val = __old; \ + } \ + __old; \ +}) +#include <asm-generic/atomic-long.h> #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h>
I think this should be guarded by an "#ifndef" like other things in this file, so architectures can provide their own implementations, or can use the C11 atomic_fetch_or() for newer compilers.
Also, I wonder about the nomenclature here: other than cmpxchg and xchg, all the atomic ops are named "atomic_xxx". For something that returns the old value, I'd expect it to be atomic_or_return() and be otherwise like the existing atomic_or() routine, and thus you'd specify "atomic_t tick_dependency".
Avoiding all of these issues is probably why fetch_or() is not exported :-) I made some similar comments last time around:
https://lkml.kernel.org/r/[email protected]
Signature · 2 lines
Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Re: [PATCH 1/7] atomic: Export fetch_or()
Quoted 43 lines · Chris Metcalf
Quoted 42 lines · Frederic Weisbecker
Export
fetch_or()that's implemented and used internally by the scheduler. We are going to use it forNO_HZso make it generally available.Cc Christoph Lameter<[email protected]>Cc Chris Metcalf<[email protected]>Cc Ingo Molnar<[email protected]>Cc Luiz Capitulino<[email protected]>Cc Peter Zijlstra<[email protected]>Cc Rik van Riel<[email protected]>Cc Thomas Gleixner<[email protected]>Cc Viresh Kumar<[email protected]>Signed-off-by Frederic Weisbecker<[email protected]>--- include/linux/atomic.h | 18 ++++++++++++++++++ kernel/sched/core.c | 14 -------------- 2 files changed, 18 insertions(+), 14 deletions(-)include/linux/atomic.h +17
-0@@ -451,6 +451,24 @@ static inline int atomic_dec_if_positive(atomic_t *v) }#endif +/** +* fetch_or - perform *ptr |= mask and return old value of *ptr +* @ptr: pointer to value +* @mask: mask to OR on the value +* +* cmpxchg based fetch_or, macro so it works for different integer types +*/ +#define fetch_or (ptr, mask) \ +({ typeof(*(ptr)) __old, __val = *(ptr); \ + for (;;) { \ + __old = cmpxchg((ptr), __val, __val | (mask)); \ + if (__old == __val) \ + break; \ + __val = __old; \ + } \ + __old; \ +})
Quoted 8 lines
include/linux/atomic.h static inline int atomic_dec_if_positive(atomic_t *v) · line 474
Quoted 4 lines
include/linux/atomic.h +1
-0+#include <asm-generic/atomic-long.h> #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> I think this should be guarded by an "#ifndef" like other things in this file, so architectures can provide their own implementations, or can use the C11
atomic_fetch_or()for newer compilers.
Right.
Quoted 6 lines
Also, I wonder about the nomenclature here: other than cmpxchg and xchg, all the atomic ops are named "
atomic_xxx". For something that returns the old value, I'd expect it to beatomic_or_return()and be otherwise like the existingatomic_or()routine, and thus you'd specify "atomic_ttick_dependency".
I think Peterz needs it to be type-generic, like cmpxchg, such that he can use it on tsk->thread_info->flags which type can vary. But if we happen to need an atomic_t version, we can also provide an atomic_fetch_or() version.
Also note that or_return() means that you first do OR and then return the new value.
I remember debating a bit the name with Peterz and the current one now makes quite some sense to me too :-)
Thanks!
Re: [PATCH 1/7] atomic: Export fetch_or()
Quoted 8 lines · Frederic Weisbecker
Quoted 5 lines
Also, I wonder about the nomenclature here: other than cmpxchg
Quoted 4 lines
and xchg, all the atomic ops are named "
atomic_xxx". For something that returns the old value, I'd expect it to beatomic_or_return()and be otherwise like the existingatomic_or()routine, and thus you'd specify "atomic_ttick_dependency".I think Peterz needs it to be type-generic, like cmpxchg, such that he can use it on
tsk->thread_info->flagswhich type can vary. But if we happen to need anatomic_tversion, we can also provide anatomic_fetch_or()version.
Yes, I think my point is that Peter Z's version is what is needed for the scheduler, but it may not be the thing we want to start providing to the entire kernel without thinking a little harder about the semantics, the namespace issues, and whether there is or should be room for some appropriate family of similar calls. Just tossing in fetch_or() because it's easy doesn't necessarily seem like what we should be doing.
Quoted 1 line
Also note that
or_return()means that you first do OR and then return the new value.
Yeah, actually fair point. I keep forgetting Linux does this "backwards".
I still think we should use an atomic_xxx() name here rather than just adding things into the namespace willy-nilly.
It's tempting to use atomic_fetch_or() but that actually conflicts with the C11 names, and remarkably, we haven't done that yet in the kernel, so we may want to avoid doing so for now. I can imagine in the not too distant future we detect C11 compilers and allow using <stdatomic.h> if possible. (Obviously some platforms require kernel support or other tricky things for stdatomic.h, so we couldn't use it everywhere.)
We could use something like gcc's old __sync_fetch_and_XXX vs __sync_XXX_and_fetch nomenclature and call it atomic_return_or() to contrast with atomic_or_return(). That avoids clashing with C11 for now and is obviously distinct from atomic_or_return(). I suppose something like atomic_fetch_and_or() isn't terrible either.
There is boilerplate for building generic atomic functions already in include/asm-generic/atomic.h and that could be extended. Unfortunately the atomic64 generic code isn't similarly constructed, so you can't just provide a default atomic64_return_or() based on that stuff, or at least, you only can on platforms that use an array of locks to implement 64-bit atomics. Ugh.
If we did this and then wanted Peter Z's code to take advantage of it, in principle we could just have some macrology which would compare the sizeof(thread_info.flags) to sizeof(int) and sizeof(long) to see which one to use and then cast to the appropriate atomic_t or atomic64_t. That's a little painful but not terrible.
Boy, the whole situation is pretty tangled up, though.
Unless you want to take a big diversion into atomics, I'd be tempted to leave Peter's macro alone and just write it off as necessary evil to handle the fact that thread_info.flags is all kinds of different sizes and types on different platforms, and definitely never an atomic_t. Instead just create an inline function atomic_return_or(), or whatever name you prefer, that operates on an atomic_t, and use the atomic_t type for your structure field. It's clearly a win to mark the data types as being atomic to the extent we can do so, I think.
Signature · 2 lines
Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Re: [PATCH 1/7] atomic: Export fetch_or()
Quoted 52 lines · Chris Metcalf
Quoted 8 lines · Frederic Weisbecker
Quoted 5 lines
Also, I wonder about the nomenclature here: other than cmpxchg
Quoted 4 lines
and xchg, all the atomic ops are named "
atomic_xxx". For something that returns the old value, I'd expect it to beatomic_or_return()and be otherwise like the existingatomic_or()routine, and thus you'd specify "atomic_ttick_dependency".I think Peterz needs it to be type-generic, like cmpxchg, such that he can use it on
tsk->thread_info->flagswhich type can vary. But if we happen to need anatomic_tversion, we can also provide anatomic_fetch_or()version.Yes, I think my point is that Peter Z's version is what is needed for the scheduler, but it may not be the thing we want to start providing to the entire kernel without thinking a little harder about the semantics, the namespace issues, and whether there is or should be room for some appropriate family of similar calls. Just tossing in
fetch_or()because it's easy doesn't necessarily seem like what we should be doing.Quoted 1 line
Also note that
or_return()means that you first do OR and then return the new value.Yeah, actually fair point. I keep forgetting Linux does this "backwards".
I still think we should use an
atomic_xxx()name here rather than just adding things into the namespace willy-nilly.It's tempting to use
atomic_fetch_or()but that actually conflicts with the C11 names, and remarkably, we haven't done that yet in the kernel, so we may want to avoid doing so for now. I can imagine in the not too distant future we detect C11 compilers and allow using <stdatomic.h> if possible. (Obviously some platforms require kernel support or other tricky things forstdatomic.h, so we couldn't use it everywhere.)We could use something like gcc's old
__sync_fetch_and_XXXvs__sync_XXX_and_fetchnomenclature and call itatomic_return_or()to contrast withatomic_or_return(). That avoids clashing with C11 for now and is obviously distinct fromatomic_or_return(). I suppose something likeatomic_fetch_and_or()isn't terrible either.There is boilerplate for building generic atomic functions already in
include/asm-generic/atomic.hand that could be extended. Unfortunately the atomic64 generic code isn't similarly constructed, so you can't just provide a defaultatomic64_return_or()based on that stuff, or at least, you only can on platforms that use an array of locks to implement 64-bit atomics. Ugh.If we did this and then wanted Peter Z's code to take advantage of it, in principle we could just have some macrology which would compare the
sizeof(thread_info.flags)to sizeof(int) and sizeof(long) to see which one to use and then cast to the appropriateatomic_toratomic64_t. That's a little painful but not terrible.Boy, the whole situation is pretty tangled up, though.
Quoted 8 lines
Unless you want to take a big diversion into atomics, I'd be tempted to leave Peter's macro alone and just write it off as necessary evil to handle the fact that thread_info.flags is all kinds of different sizes and types on different platforms, and definitely never an
atomic_t. Instead just create an inline functionatomic_return_or(), or whatever name you prefer, that operates on anatomic_t, and use theatomic_ttype for your structure field. It's clearly a win to mark the data types as being atomic to the extent we can do so, I think.
I agree that cmpxchg, test_and_set_bit, fetch_or... functions with loose namespaces aren't the best layout.
But casting thread_info to atomic_t really worries me, I'm not sure the ending result would be correct at all. I prefer to sacrify correctness over namespace sanity :-)
Re: [PATCH 1/7] atomic: Export fetch_or()
(Resending as plain text. Not sure what Thunderbird was smoking to make this message multipart/alternative originally...)
Quoted 9 lines · Frederic Weisbecker
Quoted 8 lines · Chris Metcalf
Quoted 8 lines
Unless you want to take a big diversion into atomics, I'd be tempted to leave Peter's macro alone and just write it off as necessary evil to handle the fact that thread_info.flags is all kinds of different sizes and types on different platforms, and definitely never an
atomic_t. Instead just create an inline functionatomic_return_or(), or whatever name you prefer, that operates on anatomic_t, and use theatomic_ttype for your structure field. It's clearly a win to mark the data types as being atomic to the extent we can do so, I think.
Quoted 6 lines
I agree that cmpxchg,
test_and_set_bit,fetch_or... functions with loose namespaces aren't the best layout.But casting
thread_infotoatomic_treally worries me, I'm not sure the ending result would be correct at all. I prefer to sacrify correctness over namespace sanity:-)
Just to clear, I wasn't suggesting that that necessarily was the way for you to go. The last four lines of my message quoted above are what I think might be the best way forward, and don't involve messing with thread_info at all.
Signature · 2 lines
Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Peter Zijlstra
maintainerRe: [PATCH 1/7] atomic: Export fetch_or()
In reply to Frederic WeisbeckerQuoted 1 line · Frederic Weisbecker
Also note that
or_return()means that you first do OR and then return the new value.
Yes, that's useless. OR is an irreversible operator, which means or_return() looses data. You can never say if a bit included in the mask was set before.
You really must have fetch_or().
PATCH 2/7 nohz: New tick dependency mask
In reply to Frederic WeisbeckerThe tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
This new dependency engine stays beside the old one until all subsystems having a tick dependency are converted to it.
--- include/linux/sched.h | 8 +++ include/linux/tick.h | 21 ++++++++ kernel/time/tick-sched.c | 130 +++++++++++++++++++++++++++++++++++++++++++++-- kernel/time/tick-sched.h | 1 + 4 files changed, 155 insertions(+), 5 deletions(-)
include/linux/sched.h +8 -0
@@ -706,6 +706,10 @@ struct signal_struct {/* Earliest-expiration cache. */ struct task_cputime cputime_expires; +#ifdef CONFIG_NO_HZ_FULL +unsigned long tick_dependency; +#endif +struct list_head cpu_timers[3 ];struct pid *tty_old_pgrp;
@@ -1528,6 +1532,10 @@ struct task_struct {VTIME_SYS , } vtime_snap_whence;#endif + +#ifdef CONFIG_NO_HZ_FULL +unsigned long tick_dependency; +#endif unsigned long nvcsw, nivcsw;/* context switch counts */ u64 start_time;/* monotonic time in nsec */ u64 real_start_time;/* boot based time in nsec */
include/linux/tick.h +21 -0
@@ -97,6 +97,18 @@ static inline void tick_broadcast_exit(void)tick_broadcast_oneshot_control (TICK_BROADCAST_EXIT ); } +enum tick_dependency_bit { +TICK_POSIX_TIMER_BIT =0 , +TICK_PERF_EVENTS_BIT =1 , +TICK_SCHED_BIT =2 , +TICK_CLOCK_UNSTABLE_BIT =3 +}; + +#define TICK_POSIX_TIMER_MASK (1 << TICK_POSIX_TIMER_BIT) +#define TICK_PERF_EVENTS_MASK (1 << TICK_PERF_EVENTS_BIT) +#define TICK_SCHED_MASK (1 << TICK_SCHED_BIT) +#define TICK_CLOCK_UNSTABLE_MASK (1 << TICK_CLOCK_UNSTABLE_BIT) +#ifdef CONFIG_NO_HZ_COMMON extern int tick_nohz_tick_stopped (void );extern void tick_nohz_idle_enter (void );
@@ -152,6 +164,15 @@ static inline int housekeeping_any_cpu(void)return cpumask_any_and (housekeeping_mask, cpu_online_mask); } +extern void __tick_nohz_set_dep_delayed (enum tick_dependency_bit bit, +unsigned long *dep); +extern void __tick_nohz_clear_dep (enum tick_dependency_bit bit, +unsigned long *dep); +extern void tick_nohz_set_dep (enum tick_dependency_bit bit); +extern void tick_nohz_clear_dep (enum tick_dependency_bit bit); +extern void tick_nohz_set_dep_cpu (enum tick_dependency_bit bit,int cpu); +extern void tick_nohz_clear_dep_cpu (enum tick_dependency_bit bit,int cpu); +extern void tick_nohz_full_kick (void );extern void tick_nohz_full_kick_cpu (int cpu);extern void tick_nohz_full_kick_all (void );
kernel/time/tick-sched.c +125 -5
173 lines not on this page · raw message
kernel/time/tick-sched.h +1 -0
@@ -60,6 +60,7 @@ struct tick_sched {u64 next_timer;ktime_t idle_expires;int do_timer_last; +unsigned long tick_dependency; };extern struct tick_sched *tick_get_tick_sched (int cpu);
Signature · 1 line
2.5.3
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 27 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
This new dependency engine stays beside the old one until all subsystems having a tick dependency are converted to it.
kernel/time/tick-sched.c +6
-0+void tick_nohz_set_dep_cpu (enum tick_dependency_bit bit,int cpu) +{ +unsigned long prev; +struct tick_sched *ts; + + ts =per_cpu_ptr (&tick_cpu_sched, cpu);
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 358
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+ + prev =fetch_or (&ts->tick_dependency,BIT_MASK (bit)); +if (!prev) { +preempt_disable (); +/* Perf needs local kick that is NMI safe */ +if (cpu ==smp_processor_id ()) { +tick_nohz_full_kick (); + }else { +/* Remote irq work not NMI-safe */ +WARN_ON_ONCE (in_nmi ());
Better to say "if (!WARN_ON_ONCE(in_nmi()))" here instead so we don't actually try to kick if we are in an NMI?
Signature · 2 lines
Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 33 lines · Chris Metcalf
Quoted 32 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
This new dependency engine stays beside the old one until all subsystems having a tick dependency are converted to it.
kernel/time/tick-sched.c +11
-0+void tick_nohz_set_dep_cpu (enum tick_dependency_bit bit,int cpu) +{ +unsigned long prev; +struct tick_sched *ts; + + ts =per_cpu_ptr (&tick_cpu_sched, cpu); + + prev =fetch_or (&ts->tick_dependency,BIT_MASK (bit)); +if (!prev) { +preempt_disable (); +/* Perf needs local kick that is NMI safe */
Quoted 8 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 358
Quoted 5 lines
kernel/time/tick-sched.c +5
-0+if (cpu ==smp_processor_id ()) { +tick_nohz_full_kick (); + }else { +/* Remote irq work not NMI-safe */ +WARN_ON_ONCE (in_nmi ());Better to say "if (!WARN_ON_ONCE(in_nmi()))" here instead so we don't actually try to kick if we are in an NMI?
Makes sense yeah. I'll fix that.
Thanks.
Peter Zijlstra
maintainerRe: [PATCH 2/7] nohz: New tick dependency mask
In reply to Frederic WeisbeckerQuoted 10 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Quoted 6 lines
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
Maybe better explain why we need the per task and per signal thingy?
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+static void trace_tick_dependency (unsigned long dep) +{ +if (dep &TICK_POSIX_TIMER_MASK ) { +trace_tick_stop (0 ,"posix timers running\n" ); +return ; + } + +if (dep &TICK_PERF_EVENTS_MASK ) { +trace_tick_stop (0 ,"perf events running\n" ); +return ;
kernel/time/tick-sched.c static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · line 180
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+ } + +if (dep &TICK_SCHED_MASK ) { +trace_tick_stop (0 ,"more than 1 task in runqueue\n" ); +return ; + } + +if (dep &TICK_CLOCK_UNSTABLE_MASK ) +trace_tick_stop (0 ,"unstable sched clock\n" ); +}
I would suggest ditching the strings and using the
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+static void kick_all_work_fn (struct work_struct *work) +{ +tick_nohz_full_kick_all (); +}
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+static DECLARE_WORK(kick_all_work, kick_all_work_fn); + +void __tick_nohz_set_dep_delayed (enum tick_dependency_bit bit,unsigned long *dep) +{ +unsigned long prev; + + prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context.
Why ?
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 9 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled.
This seems to suggest you can call this with IRQs disabled
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 335
Quoted 8 lines
+ */
kernel/time/tick-sched.c +7
-0+void tick_nohz_set_dep (enum tick_dependency_bit bit) +{ +unsigned long prev; + + prev =fetch_or (&tick_dependency,BIT_MASK (bit)); +if (!prev) +tick_nohz_full_kick_all ();
But that function seems implemented using smp_call_function_many() which cannot be called with IRQs disabled.
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Quoted 8 lines
Quoted 6 lines
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
Maybe better explain why we need the per task and per signal thingy?
I'll detail that some more in the changelog. The only user of the per task/per signal tick dependency is posix cpu timer. I've been first proposing a global tick dependency as soon as any posix cpu timer is armed. It simplified everything but some reviewers complained (eg: some users might want to run posix timers on housekeepers without bothering full dynticks CPUs). I could remove the per signal dependency with dispatching it through all threads in the group each time there is an update but that's the best I can think of.
Quoted 15 lines
Quoted 15 lines
kernel/time/tick-sched.c +14
-0+static void trace_tick_dependency (unsigned long dep) +{ +if (dep &TICK_POSIX_TIMER_MASK ) { +trace_tick_stop (0 ,"posix timers running\n" ); +return ; + } + +if (dep &TICK_PERF_EVENTS_MASK ) { +trace_tick_stop (0 ,"perf events running\n" ); +return ; + } + +if (dep &TICK_SCHED_MASK ) { +trace_tick_stop (0 ,"more than 1 task in runqueue\n" );
Quoted 8 lines
kernel/time/tick-sched.c static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · line 180
Quoted 6 lines
kernel/time/tick-sched.c +6
-0+return ; + } + +if (dep &TICK_CLOCK_UNSTABLE_MASK ) +trace_tick_stop (0 ,"unstable sched clock\n" ); +}I would suggest ditching the strings and using the
Using a code value instead?
Quoted 17 lines
Quoted 5 lines
kernel/time/tick-sched.c +4
-0+static void kick_all_work_fn (struct work_struct *work) +{ +tick_nohz_full_kick_all (); +}kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+static DECLARE_WORK(kick_all_work, kick_all_work_fn); + +void __tick_nohz_set_dep_delayed (enum tick_dependency_bit bit,unsigned long *dep) +{ +unsigned long prev; + + prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context. Why ?
Because posix timers code is all called with interrupts disabled and we can't send IPIs then.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 10 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled. This seems to suggest you can call this with IRQs disabled
Ah right, that's a misleading comment. We need to use the _delayed() version when interrupts are disabled.
Thanks.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 335
Quoted 9 lines
+ */
kernel/time/tick-sched.c +7
-0+void tick_nohz_set_dep (enum tick_dependency_bit bit) +{ +unsigned long prev; + + prev =fetch_or (&tick_dependency,BIT_MASK (bit)); +if (!prev) +tick_nohz_full_kick_all ();But that function seems implemented using
smp_call_function_many()which cannot be called with IRQs disabled.
Peter Zijlstra
maintainerRe: [PATCH 2/7] nohz: New tick dependency mask
Quoted 23 lines · Frederic Weisbecker
Quoted 6 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+ prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context. Why ?
Because posix timers code is all called with interrupts disabled and we can't send IPIs then.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 10 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled. This seems to suggest you can call this with IRQs disabled
Quoted 2 lines
Ah right, that's a misleading comment. We need to use the
_delayed()version when interrupts are disabled.
Why can't you use tick_nohz_full_kick_cpu() for all that, which is usable from IRQ context and avoid all that delayed muck?
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 24 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker
Quoted 6 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+ prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context. Why ?
Because posix timers code is all called with interrupts disabled and we can't send IPIs then.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 10 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled. This seems to suggest you can call this with IRQs disabled
Quoted 5 lines
Quoted 2 lines
Ah right, that's a misleading comment. We need to use the
_delayed()version when interrupts are disabled.Why can't you use
tick_nohz_full_kick_cpu()for all that, which is usable from IRQ context and avoid all that delayed muck?
Because I need to kick all the CPUs where the task/signal is running on. That's a bit difficult to do though. I think we had something to try to send an IPI to a task, but I can't retrieve it. Looks easy to do anyway. But in the signal case I'd need to do that for all tasks in the group. That sounds like a costly loop.
So I simplify that with a global IPI.
Peter Zijlstra
maintainerRe: [PATCH 2/7] nohz: New tick dependency mask
Quoted 31 lines · Frederic Weisbecker
Quoted 24 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker
Quoted 6 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+ prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context. Why ?
Because posix timers code is all called with interrupts disabled and we can't send IPIs then.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 10 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled. This seems to suggest you can call this with IRQs disabled
Quoted 5 lines
Quoted 2 lines
Ah right, that's a misleading comment. We need to use the
_delayed()version when interrupts are disabled.Why can't you use
tick_nohz_full_kick_cpu()for all that, which is usable from IRQ context and avoid all that delayed muck?
Quoted 6 lines
Because I need to kick all the CPUs where the task/signal is running on. That's a bit difficult to do though. I think we had something to try to send an IPI to a task, but I can't retrieve it. Looks easy to do anyway. But in the signal case I'd need to do that for all tasks in the group. That sounds like a costly loop.
So I simplify that with a global IPI.
Sure, but what I meant was that:
for_each_cpu(cpu, mask) tick_nohz_full_kick_cpu(cpu);
is IRQ safe, whereas the current thingy is not. Sure, its a wee bit less efficient, but do we really care?
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 39 lines · Peter Zijlstra
Quoted 31 lines · Frederic Weisbecker
Quoted 24 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker
Quoted 6 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 318
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+ prev =fetch_or (dep,BIT_MASK (bit)); +if (!prev) { +/* +* We need the IPIs to be sent from sane process context. Why ?
Because posix timers code is all called with interrupts disabled and we can't send IPIs then.
Quoted 12 lines
kernel/time/tick-sched.c void tick_nohz_full_kick_all(void) · line 327
Quoted 10 lines
kernel/time/tick-sched.c +9
-0+* The posix cpu timers are always set with irqs disabled. +*/ +schedule_work (&kick_all_work); + } +} + +/* +* Set a global tick dependency. Lets do the wide IPI kick asynchronously +* for callers with irqs disabled. This seems to suggest you can call this with IRQs disabled
Quoted 5 lines
Quoted 2 lines
Ah right, that's a misleading comment. We need to use the
_delayed()version when interrupts are disabled.Why can't you use
tick_nohz_full_kick_cpu()for all that, which is usable from IRQ context and avoid all that delayed muck?Quoted 6 lines
Because I need to kick all the CPUs where the task/signal is running on. That's a bit difficult to do though. I think we had something to try to send an IPI to a task, but I can't retrieve it. Looks easy to do anyway. But in the signal case I'd need to do that for all tasks in the group. That sounds like a costly loop.
So I simplify that with a global IPI.
Quoted 7 lines
Sure, but what I meant was that:
for_each_cpu(cpu, mask) tick_nohz_full_kick_cpu(cpu);is IRQ safe, whereas the current thingy is not. Sure, its a wee bit less efficient, but do we really care?
Right, this overhead probably doesn't matter much. I'll do that and we'll see if people complain.
Peter Zijlstra
maintainerRe: [PATCH 2/7] nohz: New tick dependency mask
In reply to Frederic WeisbeckerQuoted 23 lines · Frederic Weisbecker
Quoted 14 lines
Quoted 14 lines
kernel/time/tick-sched.c +14
-0+static void trace_tick_dependency (unsigned long dep) +{ +if (dep &TICK_POSIX_TIMER_MASK ) { +trace_tick_stop (0 ,"posix timers running\n" ); +return ; + } + +if (dep &TICK_PERF_EVENTS_MASK ) { +trace_tick_stop (0 ,"perf events running\n" ); +return ; + } + +if (dep &TICK_SCHED_MASK ) { +trace_tick_stop (0 ,"more than 1 task in runqueue\n" );Quoted 8 lines
kernel/time/tick-sched.c static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · line 180
Quoted 6 lines
kernel/time/tick-sched.c +6
-0+return ; + } + +if (dep &TICK_CLOCK_UNSTABLE_MASK ) +trace_tick_stop (0 ,"unstable sched clock\n" ); +}I would suggest ditching the strings and using the
Quoted 1 line
Using a code value instead?
Duh, it seems I forgot to finish that sentence :/
I meant to say use the ftrace enum stuff. So yes, you encode a value and the ftrace printing muck will generate a string if and when required.
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 25 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker
Quoted 22 lines
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+static void trace_tick_dependency (unsigned long dep) +{ +if (dep &TICK_POSIX_TIMER_MASK ) { +trace_tick_stop (0 ,"posix timers running\n" ); +return ; + } + +if (dep &TICK_PERF_EVENTS_MASK ) { +trace_tick_stop (0 ,"perf events running\n" ); +return ;kernel/time/tick-sched.c static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · line 180
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+ } + +if (dep &TICK_SCHED_MASK ) { +trace_tick_stop (0 ,"more than 1 task in runqueue\n" ); +return ; + } + +if (dep &TICK_CLOCK_UNSTABLE_MASK ) +trace_tick_stop (0 ,"unstable sched clock\n" ); +}I would suggest ditching the strings and using the
Quoted 6 lines
Quoted 1 line
Using a code value instead?
Duh, it seems I forgot to finish that sentence :/
I meant to say use the ftrace enum stuff. So yes, you encode a value and the ftrace printing muck will generate a string if and when required.
Right I first wanted to avoid that because it makes parsing more complicated for tools but probably it works fine now with libtraceevents.
Peter Zijlstra
maintainerRe: [PATCH 2/7] nohz: New tick dependency mask
In reply to Frederic WeisbeckerQuoted 26 lines · Frederic Weisbecker
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Quoted 8 lines
Quoted 6 lines
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
Maybe better explain why we need the per task and per signal thingy?
I'll detail that some more in the changelog. The only user of the per task/per signal tick dependency is posix cpu timer. I've been first proposing a global tick dependency as soon as any posix cpu timer is armed.
Quoted 5 lines
It simplified everything but some reviewers complained (eg:
some users might want to run posix timers on housekeepers without bothering full dynticks CPUs). I could remove the per signal dependency with dispatching it through all threads in the group each time there is an update but that's the best I can think of.
Right, I remember some of that. Seems worth preserving these reasons. Maybe even in code comments near the definition of these things.
Re: [PATCH 2/7] nohz: New tick dependency mask
Quoted 27 lines · Peter Zijlstra
Quoted 25 lines · Frederic Weisbecker
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Frederic Weisbecker
The tick dependency is evaluated on every IRQ. This is a batch of checks which determine whether it is safe to stop the tick or not. These checks are often split in many details: posix cpu timers, scheduler, sched clock, perf events. Each of which are made of smaller details: posix cpu timer involves checking process wide timers then thread wide timers. Perf involves checking freq events then more per cpu details.
Checking these details asynchronously every time we update the full dynticks state bring avoidable overhead and a messy layout.
Quoted 8 lines
Quoted 6 lines
Lets introduce instead tick dependency masks: one for system wide dependency (unstable sched clock), one for CPU wide dependency (sched, perf), and task/signal level dependencies. The subsystems are responsible of setting and clearing their dependency through a set of APIs that will take care of concurrent dependency mask modifications and kick targets to restart the relevant CPU tick whenever needed.
Maybe better explain why we need the per task and per signal thingy?
I'll detail that some more in the changelog. The only user of the per task/per signal tick dependency is posix cpu timer. I've been first proposing a global tick dependency as soon as any posix cpu timer is armed.
Quoted 8 lines
Quoted 5 lines
It simplified everything but some reviewers complained (eg:
some users might want to run posix timers on housekeepers without bothering full dynticks CPUs). I could remove the per signal dependency with dispatching it through all threads in the group each time there is an update but that's the best I can think of.Right, I remember some of that. Seems worth preserving these reasons. Maybe even in code comments near the definition of these things.
Agreed! I'll comment that some more.
Thanks.
PATCH 3/7 perf: Migrate perf to use new tick dependency mask model
In reply to Frederic WeisbeckerInstead of providing asynchronous checks for the nohz subsystem to verify perf event tick dependency, migrate perf to the new mask.
Perf needs the tick for two situations:
1) Freq events. We could set the tick dependency when those are installed on a CPU context. But setting a global dependency on top of the global freq events accounting is much easier. If people want that to be optimized, we can still refine that on the per-CPU tick dependency level. This patch dooesn't change the current behaviour anyway.
2) Throttled events: this is a per-cpu dependency.
--- include/linux/perf_event.h | 6 ------ kernel/events/core.c | 22 +++++++--------------- kernel/time/tick-sched.c | 6 ------ 3 files changed, 7 insertions(+), 27 deletions(-)
include/linux/perf_event.h +0 -6
@@ -1023,12 +1023,6 @@ static inline void perf_event_task_tick(void) { }static inline int perf_event_release_kernel (struct perf_event *event) {return 0 ; }#endif -#if defined(CONFIG_PERF_EVENTS ) && defined(CONFIG_NO_HZ_FULL ) -extern bool perf_event_can_stop_tick (void ); -#else -static inline bool perf_event_can_stop_tick (void ) {return true; } -#endif -#if defined(CONFIG_PERF_EVENTS ) && defined(CONFIG_CPU_SUP_INTEL )extern void perf_restore_debug_store (void );#else
kernel/events/core.c +7 -15
@@ -3099,17 +3099,6 @@ done:return rotate; } -#ifdef CONFIG_NO_HZ_FULL -bool perf_event_can_stop_tick (void ) -{ -if (atomic_read (&nr_freq_events) || -__this_cpu_read (perf_throttled_count)) -return false; -else -return true; -} -#endif -void perf_event_task_tick (void ) {struct list_head *head =this_cpu_ptr (&active_ctx_list);
@@ -3120,6 +3109,7 @@ void perf_event_task_tick(void)__this_cpu_inc (perf_throttled_seq); throttled =__this_cpu_xchg (perf_throttled_count,0 ); +tick_nohz_clear_dep_cpu (TICK_PERF_EVENTS_BIT ,smp_processor_id ());list_for_each_entry_safe (ctx, tmp, head, active_ctx_list)perf_adjust_freq_unthr_context (ctx, throttled);
@@ -3540,8 +3530,10 @@ static void unaccount_event(struct perf_event *event)atomic_dec (&nr_comm_events);if (event->attr.task)atomic_dec (&nr_task_events); -if (event->attr.freq) -atomic_dec (&nr_freq_events); +if (event->attr.freq) { +if (atomic_dec_and_test (&nr_freq_events)) +tick_nohz_clear_dep (TICK_PERF_EVENTS_BIT ); + }if (event->attr.context_switch) {static_key_slow_dec_deferred (&perf_sched_events);atomic_dec (&nr_switch_events);
@@ -6307,9 +6299,9 @@ static int __perf_event_overflow(struct perf_event *event,if (unlikely (throttle && hwc->interrupts >= max_samples_per_tick)) {__this_cpu_inc (perf_throttled_count); +tick_nohz_set_dep_cpu (TICK_PERF_EVENTS_BIT ,smp_processor_id ()); hwc->interrupts =MAX_INTERRUPTS ;perf_log_throttle (event,0 ); -tick_nohz_full_kick (); ret =1 ; } }
@@ -7695,7 +7687,7 @@ static void account_event(struct perf_event *event)atomic_inc (&nr_task_events);if (event->attr.freq) {if (atomic_inc_return (&nr_freq_events) ==1 ) -tick_nohz_full_kick_all (); +tick_nohz_set_dep (TICK_PERF_EVENTS_BIT ); }if (event->attr.context_switch) {atomic_inc (&nr_switch_events);
kernel/time/tick-sched.c +0 -6
@@ -22,7 +22,6 @@#include <linux/module.h> #include <linux/irq_work.h> #include <linux/posix-timers.h> -#include <linux/perf_event.h> #include <linux/context_tracking.h> #include <asm/irq_regs.h>
@@ -213,11 +212,6 @@ static bool can_stop_full_tick(struct tick_sched *ts)return false; } -if (!perf_event_can_stop_tick ()) { -trace_tick_stop (0 ,"perf events running\n" ); -return false; - } -#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK /* * sched_clock_tick() needs us?
Signature · 1 line
2.5.3
Re: [PATCH 3/7] perf: Migrate perf to use new tick dependency mask model
Quoted 29 lines · Frederic Weisbecker
Instead of providing asynchronous checks for the nohz subsystem to verify perf event tick dependency, migrate perf to the new mask.
Perf needs the tick for two situations:
1) Freq events. We could set the tick dependency when those are installed on a CPU context. But setting a global dependency on top of the global freq events accounting is much easier. If people want that to be optimized, we can still refine that on the per-CPU tick dependency level. This patch dooesn't change the current behaviour anyway.
2) Throttled events: this is a per-cpu dependency.
kernel/events/core.c +4
-2@@ -3540,8 +3530,10 @@ static void unaccount_event(struct perf_event *event)atomic_dec (&nr_comm_events);if (event->attr.task)atomic_dec (&nr_task_events); -if (event->attr.freq) -atomic_dec (&nr_freq_events); +if (event->attr.freq) { +if (atomic_dec_and_test (&nr_freq_events)) +tick_nohz_clear_dep (TICK_PERF_EVENTS_BIT ); + }if (event->attr.context_switch) {static_key_slow_dec_deferred (&perf_sched_events);atomic_dec (&nr_switch_events);
kernel/events/core.c static void account_event(struct perf_event *event) · line 7693
Quoted 8 lines
kernel/events/core.c +1
-1@@ -7695,7 +7687,7 @@ static void account_event(struct perf_event *event)atomic_inc (&nr_task_events);if (event->attr.freq) {if (atomic_inc_return (&nr_freq_events) ==1 ) -tick_nohz_full_kick_all (); +tick_nohz_set_dep (TICK_PERF_EVENTS_BIT ); }if (event->attr.context_switch) {atomic_inc (&nr_switch_events);
It would be helpful to have a comment explaining why these two can't race with each other, e.g. this race:
[cpu 1] atomic_dec_and_test
[cpu 2] atomic_inc_return
[cpu 2] tick_nohz_set_dep()
[cpu 1] tick_nohz_clear_dep()
Or perhaps this is a true race condition possibility?
I think we're OK for the sched cases since they're protected under the rq lock, I think. I'm not sure about the POSIX cpu timers.
Signature · 2 lines
Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Re: [PATCH 3/7] perf: Migrate perf to use new tick dependency mask model
Quoted 47 lines · Chris Metcalf
Quoted 29 lines · Frederic Weisbecker
Instead of providing asynchronous checks for the nohz subsystem to verify perf event tick dependency, migrate perf to the new mask.
Perf needs the tick for two situations:
1) Freq events. We could set the tick dependency when those are installed on a CPU context. But setting a global dependency on top of the global freq events accounting is much easier. If people want that to be optimized, we can still refine that on the per-CPU tick dependency level. This patch dooesn't change the current behaviour anyway.
2) Throttled events: this is a per-cpu dependency.
kernel/events/core.c +4
-2@@ -3540,8 +3530,10 @@ static void unaccount_event(struct perf_event *event)atomic_dec (&nr_comm_events);if (event->attr.task)atomic_dec (&nr_task_events); -if (event->attr.freq) -atomic_dec (&nr_freq_events); +if (event->attr.freq) { +if (atomic_dec_and_test (&nr_freq_events)) +tick_nohz_clear_dep (TICK_PERF_EVENTS_BIT ); + }if (event->attr.context_switch) {static_key_slow_dec_deferred (&perf_sched_events);atomic_dec (&nr_switch_events);kernel/events/core.c static void account_event(struct perf_event *event) · line 7693
Quoted 8 lines
kernel/events/core.c +1
-1@@ -7695,7 +7687,7 @@ static void account_event(struct perf_event *event)atomic_inc (&nr_task_events);if (event->attr.freq) {if (atomic_inc_return (&nr_freq_events) ==1 ) -tick_nohz_full_kick_all (); +tick_nohz_set_dep (TICK_PERF_EVENTS_BIT ); }if (event->attr.context_switch) {atomic_inc (&nr_switch_events);It would be helpful to have a comment explaining why these two can't race with each other, e.g. this race:
[cpu 1]
atomic_dec_and_test
[cpu 2]atomic_inc_return
[cpu 2]tick_nohz_set_dep()
[cpu 1]tick_nohz_clear_dep()
Quoted 4 lines
Or perhaps this is a true race condition possibility?
I think we're OK for the sched cases since they're protected under the rq lock, I think. I'm not sure about the POSIX cpu timers.
Hmm, how did I miss that...
So in the case of perf, either we need locking, in which case we may want to use something like tick_nohz_add_dep() which takes care of counting. But perf would be the only user.
Another possibility is to rather set/clear the tick mask on the task level in event_sched_in/event_sched_out using ctx->nr_freq which is protected by ctx->lock. I think I should rather do that.
Concerning the others:
_ sched: we are under the rq lock, like you noticed, we are fine.
_ posix timers: we are under sighand lock, so we are fine too.
_ sched_clock_stable: that one is more obscure. It seems that set_sched_clock_stable()
and clear_sched_clock_stable() can race on static keys if running concurrently, and that would concern tick mask as well.
Thanks.
Quoted 5 lines
Signature · 3 lines
-- Chris Metcalf, EZChip Semiconductor http://www.ezchip.com
Peter Zijlstra
maintainerRe: [PATCH 3/7] perf: Migrate perf to use new tick dependency mask model
Quoted 12 lines · Frederic Weisbecker
Quoted 10 lines · Chris Metcalf
It would be helpful to have a comment explaining why these two can't race with each other, e.g. this race:
[cpu 1]
atomic_dec_and_test
[cpu 2]atomic_inc_return
[cpu 2]tick_nohz_set_dep()
[cpu 1]tick_nohz_clear_dep()Or perhaps this is a true race condition possibility?
Quoted 8 lines
Quoted 2 lines
I think we're OK for the sched cases since they're protected under the rq lock, I think. I'm not sure about the POSIX cpu timers.
Hmm, how did I miss that...
So in the case of perf, either we need locking, in which case we may want to use something like
tick_nohz_add_dep()which takes care of counting. But perf would be the only user.
Right; so you could use something like atomic_dec_and_mutex_lock(), that would only globally serialize the 0<->1 transitions without incurring overhead to any other state transitions.
A possibly even less expensive option might be something along the lines of:
tick_nohz_update_perf_dep()
{
static DEFINE_SPINLOCK(lock);
bool freq;
spin_lock(&lock);
freq = !!atomic_read(&nr_freq_events);
if (freq ^ !!tick_nohz_test_dep(PERF)) {
if (freq)
tick_nohz_set_dep(PERF);
else
tick_nohz_clear_dep(PERF);
}
spin_unlock(&lock);
}if (atomic_inc_return(&nr_freq_events) == 1) tick_nohz_update_perf_dep(); if (atomic_dec_return(&nr_freq_events) == 0) tick_nohz_update_perf_dep();
That avoids the atomic_add_unless() muckery.
Quoted 3 lines
_
sched_clock_stable: that one is more obscure. It seems thatset_sched_clock_stable()
andclear_sched_clock_stable()can race on static keys if running concurrently, and that would concern tick mask as well.
All you need to ensure here is that clear wins. Once we clear sched_clock_stable we should _never_ set it again.
Re: [PATCH 3/7] perf: Migrate perf to use new tick dependency mask model
Quoted 30 lines · Peter Zijlstra
Quoted 12 lines · Frederic Weisbecker
Quoted 10 lines · Chris Metcalf
It would be helpful to have a comment explaining why these two can't race with each other, e.g. this race:
[cpu 1]
atomic_dec_and_test
[cpu 2]atomic_inc_return
[cpu 2]tick_nohz_set_dep()
[cpu 1]tick_nohz_clear_dep()Or perhaps this is a true race condition possibility?
Quoted 8 lines
Quoted 2 lines
I think we're OK for the sched cases since they're protected under the rq lock, I think. I'm not sure about the POSIX cpu timers.
Hmm, how did I miss that...
So in the case of perf, either we need locking, in which case we may want to use something like
tick_nohz_add_dep()which takes care of counting. But perf would be the only user.Right; so you could use something like
atomic_dec_and_mutex_lock(), that would only globally serialize the 0<->1 transitions without incurring overhead to any other state transitions.A possibly even less expensive option might be something along the lines of:
tick_nohz_update_perf_dep()
Quoted 14 lines
{
static DEFINE_SPINLOCK(lock);
bool freq;
spin_lock(&lock);
freq = !!atomic_read(&nr_freq_events);
if (freq ^ !!tick_nohz_test_dep(PERF)) {
if (freq)
tick_nohz_set_dep(PERF);
else
tick_nohz_clear_dep(PERF);
}
spin_unlock(&lock);
}Well, doing the inc/dec inside the lock is easier to read :-)
Quoted 11 lines
if (atomic_inc_return(&nr_freq_events) == 1) tick_nohz_update_perf_dep(); if (atomic_dec_return(&nr_freq_events) == 0) tick_nohz_update_perf_dep();That avoids the
atomic_add_unless()muckery.
Right, I can do either that or I can move the dependency to the CPU level and count nr_freq to the cpu_ctx when any ctx gets scheduled in/out. Then everytime we inc and nr_freq == 1, we set the dependency (all that should be serialized as it only happens locally).
Which way do you prefer? Arguably, the global dependency approach is probably more simple (although the above based on _test() and atomic_read() is tricky) and the CPU dependency is more fine-grained (and we avoid the above tricks). Not sure we need it to be fine-grained in this level though.
Quoted 7 lines
Quoted 4 lines
_
sched_clock_stable: that one is more obscure. It seems thatset_sched_clock_stable()
andclear_sched_clock_stable()can race on static keys if running concurrently, and that would concern tick mask as well.All you need to ensure here is that clear wins. Once we clear
sched_clock_stablewe should _never_ set it again.
Ok, so we better make sure that such sequence:
set_sched_clock_stable();
clear_sched_clock_stable();...is always serialized by callers somewhow. If they can't happen in parallel we are fine because the set is sequential whereas the clear does the static key change asynchronously.
Peter Zijlstra
maintainerRe: [PATCH 3/7] perf: Migrate perf to use new tick dependency mask model
Quoted 4 lines · Frederic Weisbecker
Right, I can do either that or I can move the dependency to the CPU level and count
nr_freqto thecpu_ctxwhen any ctx gets scheduled in/out. Then everytime we inc andnr_freq== 1, we set the dependency (all that should be serialized as it only happens locally).
Doing it per CPU would, as you say, add accounting crap to the context switch path. Now given that context switches with perf enabled are already silly expensive that might just fall away into the noise.
PATCH 4/7 sched: Account rr and fifo tasks separately
In reply to Frederic WeisbeckerIn order to evaluate tick dependency, we need to account SCHED_RR and SCHED_FIFO tasks separately as those policies don't have the same preemption requirements.
We still keep rt_nr_running as a cache to avoid additions between nr_rr and nr_fifo all over the place.
--- kernel/sched/rt.c | 34 ++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 2 files changed, 36 insertions(+)
kernel/sched/rt.c +34 -0
@@ -1152,12 +1152,43 @@ unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) }static inline +unsigned int rt_se_fifo_nr_running (struct sched_rt_entity *rt_se) +{ +struct rt_rq *group_rq =group_rt_rq (rt_se); +struct task_struct *tsk; + +if (group_rq) +return group_rq->fifo_nr_running; + + tsk =rt_task_of (rt_se); + +return (tsk->policy ==SCHED_FIFO ) ?1 :0 ; +} + +static inline +unsigned int rt_se_rr_nr_running (struct sched_rt_entity *rt_se) +{ +struct rt_rq *group_rq =group_rt_rq (rt_se); +struct task_struct *tsk; + +if (group_rq) +return group_rq->rr_nr_running; + + tsk =rt_task_of (rt_se); + +return (tsk->policy ==SCHED_RR ) ?1 :0 ; +} + +static inline void inc_rt_tasks (struct sched_rt_entity *rt_se,struct rt_rq *rt_rq) {int prio =rt_se_prio (rt_se);WARN_ON (!rt_prio (prio)); rt_rq->rt_nr_running +=rt_se_nr_running (rt_se); + rt_rq->fifo_nr_running +=rt_se_fifo_nr_running (rt_se); + rt_rq->rr_nr_running +=rt_se_rr_nr_running (rt_se); +WARN_ON_ONCE (rt_rq->rt_nr_running != rt_rq->fifo_nr_running + rt_rq->rr_nr_running);inc_rt_prio (rt_rq, prio);inc_rt_migration (rt_se, rt_rq);
@@ -1170,6 +1201,9 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)WARN_ON (!rt_prio (rt_se_prio (rt_se)));WARN_ON (!rt_rq->rt_nr_running); rt_rq->rt_nr_running -=rt_se_nr_running (rt_se); + rt_rq->fifo_nr_running -=rt_se_fifo_nr_running (rt_se); + rt_rq->rr_nr_running -=rt_se_rr_nr_running (rt_se); +WARN_ON_ONCE (rt_rq->rt_nr_running != rt_rq->fifo_nr_running + rt_rq->rr_nr_running);dec_rt_prio (rt_rq,rt_se_prio (rt_se));dec_rt_migration (rt_se, rt_rq);
kernel/sched/sched.h +2 -0
@@ -433,6 +433,8 @@ static inline int rt_bandwidth_enabled(void)struct rt_rq {struct rt_prio_array active;unsigned int rt_nr_running; +unsigned int fifo_nr_running; +unsigned int rr_nr_running; #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED struct { int curr; /* highest queued rt task prio */
Signature · 1 line
2.5.3
Peter Zijlstra
maintainerRe: [PATCH 4/7] sched: Account rr and fifo tasks separately
Quoted 6 lines · Frederic Weisbecker
In order to evaluate tick dependency, we need to account
SCHED_RRandSCHED_FIFOtasks separately as those policies don't have the same preemption requirements.We still keep
rt_nr_runningas a cache to avoid additions betweennr_rrandnr_fifoall over the place.
In which case you only need one of nr_fifo/nr_rr. Less accounting is better.
Pick the one you need for the nohz_full condition, and leave the other. A quick look at sched_can_stop_tick() seems to suggest nr_rr is the interesting one. nr_rr < 2 should allow stopping the tick.
Re: [PATCH 4/7] sched: Account rr and fifo tasks separately
Quoted 8 lines · Peter Zijlstra
Quoted 6 lines · Frederic Weisbecker
In order to evaluate tick dependency, we need to account
SCHED_RRandSCHED_FIFOtasks separately as those policies don't have the same preemption requirements.We still keep
rt_nr_runningas a cache to avoid additions betweennr_rrandnr_fifoall over the place.
Quoted 6 lines
In which case you only need one of nr_fifo/nr_rr. Less accounting is better.
Pick the one you need for the
nohz_fullcondition, and leave the other. A quick look atsched_can_stop_tick()seems to suggestnr_rris the interesting one.nr_rr< 2 should allow stopping the tick.
Sounds pretty good! I'm going to do that!
Thanks!
PATCH 5/7 sched: Migrate sched to use new tick dependency mask model
In reply to Frederic WeisbeckerInstead of providing asynchronous checks for the nohz subsystem to verify sched tick dependency, migrate sched to the new mask.
Everytime a task is enqueued or dequeued, we evaluate the state of the tick dependency on top of the policy of the tasks in the runqueue, by order of priority:
SCHED_DEADLINE: Need the tick in order to periodically check for runtimeSCHED_FIFO : Don't need the tick (no round-robin)SCHED_RR : Need the tick if more than 1 task of the same priority
for round robin (simplified with checking if more than one SCHED_RR task no matter what priority).SCHED_NORMAL : Need the tick if more than 1 task for round-robin.
We could optimize that further with one flag per sched policy on the tick dependency mask and perform only the checks relevant to the policy concerned by an enqueue/dequeue operation.
Since the checks aren't based on the current task anymore, we could get rid of the task switch hook but it's still needed for posix cpu timers.
--- include/linux/sched.h | 3 --- kernel/sched/core.c | 33 ++++++++++++++++++--------------- kernel/sched/sched.h | 47 +++++++++++++++++++++++++++++++++-------------- kernel/time/tick-sched.c | 5 ----- 4 files changed, 51 insertions(+), 37 deletions(-)
include/linux/sched.h +0 -3
@@ -2348,10 +2348,7 @@ static inline void wake_up_nohz_cpu(int cpu) { }#endif #ifdef CONFIG_NO_HZ_FULL -extern bool sched_can_stop_tick (void );extern u64 scheduler_tick_max_deferment (void ); -#else -static inline bool sched_can_stop_tick (void ) {return false; }#endif #ifdef CONFIG_SCHED_AUTOGROUP
kernel/sched/core.c +18 -15
@@ -701,31 +701,34 @@ static inline bool got_nohz_idle_kick(void)#endif /* CONFIG_NO_HZ_COMMON */ #ifdef CONFIG_NO_HZ_FULL -bool sched_can_stop_tick (void ) +bool sched_can_stop_tick (struct rq *rq) { +/* Deadline tasks, even if single, need the tick */ +if (rq->dl.dl_nr_running) +return false; +/* -* FIFO realtime policy runs the highest priority task. Other runnable -* tasks are of a lower priority. The scheduler tick does nothing. +* FIFO realtime policy runs the highest priority task (after DEADLINE). +* Other runnable tasks are of a lower priority. The scheduler tick +* isn't needed. */ -if (current->policy ==SCHED_FIFO ) +if (rq->rt.fifo_nr_running)return true;/* * Round-robin realtime tasks time slice with other tasks at the same -* realtime priority. Is this task the only one at this priority? +* realtime priority. */ -if (current->policy ==SCHED_RR ) { -struct sched_rt_entity *rt_se = ¤t->rt; - -return rt_se->run_list.prev == rt_se->run_list.next; +if (rq->rt.rr_nr_running) { +if (rq->rt.rr_nr_running ==1 ) +return true; +else +return false; } -/* -* More than one running task need preemption. -* nr_running update is assumed to be visible -* after IPI is sent from wakers. -*/ -if (this_rq ()->nr_running >1 ) + +/* Normal multitasking need periodic preemption checks */ +if (rq->cfs.nr_running >1 )return false;return true;
kernel/sched/sched.h +33 -14
65 lines not on this page · raw message
kernel/time/tick-sched.c +0 -5
@@ -202,11 +202,6 @@ static bool can_stop_full_tick(struct tick_sched *ts)return false; } -if (!sched_can_stop_tick ()) { -trace_tick_stop (0 ,"more than 1 task in runqueue\n" ); -return false; - } -if (!posix_cpu_timers_can_stop_tick (current)) {trace_tick_stop (0 ,"posix timers running\n" );return false;
Signature · 1 line
2.5.3
PATCH 6/7 posix-cpu-timers: Migrate to use new tick dependency mask model
In reply to Frederic WeisbeckerInstead of providing asynchronous checks for the nohz subsystem to verify posix cpu timers tick dependency, migrate the latter to the new mask.
In order to keep track of the running timers and expose the tick dependency accordingly, we must probe the timers queuing and dequeuing on threads and process lists.
Unfortunately it implies both task and signal level dependencies. We should be able to further optimize this and merge all that on the task level dependency, at the cost of a bit of complexity and may be overhead.
--- include/linux/posix-timers.h | 3 --- kernel/time/posix-cpu-timers.c | 52 +++++++++--------------------------------- kernel/time/tick-sched.c | 5 ---- 3 files changed, 11 insertions(+), 49 deletions(-)
include/linux/posix-timers.h +0 -3
@@ -128,9 +128,6 @@ void posix_cpu_timer_schedule(struct k_itimer *timer);void run_posix_cpu_timers (struct task_struct *task);void posix_cpu_timers_exit (struct task_struct *task);void posix_cpu_timers_exit_group (struct task_struct *task); - -bool posix_cpu_timers_can_stop_tick (struct task_struct *tsk); -void set_process_cpu_timer (struct task_struct *task,unsigned int clock_idx,cputime_t *newval,cputime_t *oldval);
kernel/time/posix-cpu-timers.c +11 -41
119 lines not on this page · raw message
kernel/time/tick-sched.c +0 -5
@@ -202,11 +202,6 @@ static bool can_stop_full_tick(struct tick_sched *ts)return false; } -if (!posix_cpu_timers_can_stop_tick (current)) { -trace_tick_stop (0 ,"posix timers running\n" ); -return false; - } -#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK /* * sched_clock_tick() needs us?
Signature · 1 line
2.5.3
PATCH 7/7 sched-clock: Migrate to use new tick dependency mask model
In reply to Frederic WeisbeckerInstead of checking sched_clock_stable from the nohz subsystem to verify its tick dependency, migrate it to the new mask in order to include it to the all-in-one check.
--- kernel/sched/clock.c | 5 +++++ kernel/time/tick-sched.c | 26 ++++++-------------------- 2 files changed, 11 insertions(+), 20 deletions(-)
kernel/sched/clock.c +5 -0
@@ -61,6 +61,7 @@#include <linux/static_key.h> #include <linux/workqueue.h> #include <linux/compiler.h> +#include <linux/tick.h> /* * Scheduler clock - returns current time in nanosec units.
@@ -89,6 +90,8 @@ static void __set_sched_clock_stable(void) {if (!sched_clock_stable ())static_key_slow_inc (&__sched_clock_stable); + +tick_nohz_clear_dep (TICK_CLOCK_UNSTABLE_BIT ); }void set_sched_clock_stable (void )
@@ -108,6 +111,8 @@ static void __clear_sched_clock_stable(struct work_struct *work)/* XXX worry about clock continuity */ if (sched_clock_stable ())static_key_slow_dec (&__sched_clock_stable); + +tick_nohz_set_dep (TICK_CLOCK_UNSTABLE_BIT ); } static DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable);
kernel/time/tick-sched.c +6 -20
@@ -174,8 +174,13 @@ static void trace_tick_dependency(unsigned long dep)return ; } -if (dep &TICK_CLOCK_UNSTABLE_MASK ) +#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK +if (dep &TICK_CLOCK_UNSTABLE_MASK ) {trace_tick_stop (0 ,"unstable sched clock\n" ); +WARN_ONCE (tick_nohz_full_running, +"NO_HZ FULL will not work with unstable sched clock" ); + } +#endif }static bool can_stop_full_tick (struct tick_sched *ts)
@@ -202,25 +207,6 @@ static bool can_stop_full_tick(struct tick_sched *ts)return false; } -#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK -/* -* sched_clock_tick() needs us? -* -* TODO: kick full dynticks CPUs when -* sched_clock_stable is set. -*/ -if (!sched_clock_stable ()) { -trace_tick_stop (0 ,"unstable sched clock\n" ); -/* -* Don't allow the user to think they can get -* full NO_HZ with this machine. -*/ -WARN_ONCE (tick_nohz_full_running, -"NO_HZ FULL will not work with unstable sched clock" ); -return false; - } -#endif -return true; }
Signature · 1 line
2.5.3