tick: Cleanups and reduce jiffies_seq held times
cover tick: Cleanups and reduce jiffies_seq held times
Yunfeng proposed a change to tick_do_update_jiffies64() to reduce the contention on jiffies_seq sequence counter:
https://lore.kernel.org/r/[email protected]
This made me look deeper and there are lots of other things to optimize, but also things to document which are completely undocumented today.
The lot is based on
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
and also available via git from:
git://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git tick
Thanks,
tglx
PATCH 1/7 tick/broadcast: Serialize access to tick_next_period
tick_broadcast_setup_oneshot() accesses tick_next_period twice without any serialization. This is wrong in two aspects:
- Reading it twice might make the broadcast data inconsistent if the variable is updated concurrently.
- On 32bit systems the access might see an partial update
Protect it with jiffies_lock. That's safe as none of the callchains leading up to this function can create a lock ordering violation:
timer interrupt
run_local_timers()
hrtimer_run_queues()
hrtimer_switch_to_hres()
tick_init_highres()
tick_switch_to_oneshot()
tick_broadcast_switch_to_oneshot()
or
tick_check_oneshot_change()
tick_nohz_switch_to_nohz()
tick_switch_to_oneshot()
tick_broadcast_switch_to_oneshot()
--- kernel/time/tick-broadcast.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-)
kernel/time/tick-broadcast.c +20 -3
@@ -877,6 +877,22 @@ static void tick_broadcast_init_next_eve } } +static inline ktime_t tick_get_next_period (void ) +{ +ktime_t next; + +/* +* Protect against concurrent updates (store /load tearing on +* 32bit). It does not matter if the time is already in the +* past. The broadcast device which is about to be programmed will +* fire in any case. +*/ +raw_spin_lock (&jiffies_lock); + next = tick_next_period; +raw_spin_unlock (&jiffies_lock); +return next; +} +/** * tick_broadcast_setup_oneshot - setup the broadcast device */
@@ -905,10 +921,11 @@ static void tick_broadcast_setup_oneshot tick_broadcast_oneshot_mask, tmpmask);if (was_periodic && !cpumask_empty (tmpmask)) { +ktime_t nextevt =tick_get_next_period (); +clockevents_switch_state (bc,CLOCK_EVT_STATE_ONESHOT ); -tick_broadcast_init_next_event (tmpmask, - tick_next_period); -tick_broadcast_set_event (bc, cpu, tick_next_period); +tick_broadcast_init_next_event (tmpmask, nextevt); +tick_broadcast_set_event (bc, cpu, nextevt); }else bc->next_event =KTIME_MAX ; }else {
[tip: timers/core] tick/broadcast: Serialize access to tick_next_period
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: f73f64d5687192bc8eb7f3d9521ca6256b79f224 Gitweb: https://git.kernel.org/tip/f73f64d5687192bc8eb7f3d9521ca6256b79f224 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:43 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:28 +01:00
tick/broadcast: Serialize access to tick_next_period
tick_broadcast_setup_oneshot() accesses tick_next_period twice without any serialization. This is wrong in two aspects:
- Reading it twice might make the broadcast data inconsistent if the variable is updated concurrently.
- On 32bit systems the access might see an partial update
Protect it with jiffies_lock. That's safe as none of the callchains leading up to this function can create a lock ordering violation:
timer interrupt
run_local_timers()
hrtimer_run_queues()
hrtimer_switch_to_hres()
tick_init_highres()
tick_switch_to_oneshot()
tick_broadcast_switch_to_oneshot()
or
tick_check_oneshot_change()
tick_nohz_switch_to_nohz()
tick_switch_to_oneshot()
tick_broadcast_switch_to_oneshot()
--- kernel/time/tick-broadcast.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-)
kernel/time/tick-broadcast.c +20 -3
@@ -877,6 +877,22 @@ static void tick_broadcast_init_next_event(struct cpumask *mask, } } +static inline ktime_t tick_get_next_period (void ) +{ +ktime_t next; + +/* +* Protect against concurrent updates (store /load tearing on +* 32bit). It does not matter if the time is already in the +* past. The broadcast device which is about to be programmed will +* fire in any case. +*/ +raw_spin_lock (&jiffies_lock); + next = tick_next_period; +raw_spin_unlock (&jiffies_lock); +return next; +} +/** * tick_broadcast_setup_oneshot - setup the broadcast device */
@@ -905,10 +921,11 @@ static void tick_broadcast_setup_oneshot(struct clock_event_device *bc) tick_broadcast_oneshot_mask, tmpmask);if (was_periodic && !cpumask_empty (tmpmask)) { +ktime_t nextevt =tick_get_next_period (); +clockevents_switch_state (bc,CLOCK_EVT_STATE_ONESHOT ); -tick_broadcast_init_next_event (tmpmask, - tick_next_period); -tick_broadcast_set_event (bc, cpu, tick_next_period); +tick_broadcast_init_next_event (tmpmask, nextevt); +tick_broadcast_set_event (bc, cpu, nextevt); }else bc->next_event =KTIME_MAX ; }else {
PATCH 2/7 tick: Document protections for tick related data
In reply to Thomas GleixnerThe protection rules for tick_next_period and last_jiffies_update are blury at best. Clarify this.
--- kernel/time/tick-common.c | 4 +++- kernel/time/tick-sched.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-)
kernel/time/tick-common.c +3 -1
@@ -27,7 +27,9 @@*/ DEFINE_PER_CPU(struct tick_device, tick_cpu_device);/* -* Tick next event: keeps track of the tick time +* Tick next event: keeps track of the tick time. It's updated by the +* CPU which handles the tick and protected by jiffies_lock. There is +* no requirement to write hold the jiffies seqcount for it. */ ktime_t tick_next_period;ktime_t tick_period;
kernel/time/tick-sched.c +3 -1
@@ -44,7 +44,9 @@ struct tick_sched *tick_get_tick_sched(i#if defined(CONFIG_NO_HZ_COMMON ) || defined(CONFIG_HIGH_RES_TIMERS )/* -* The time, when the last jiffy update happened. Protected by jiffies_lock. +* The time, when the last jiffy update happened. Write access must hold +* jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a +* consistent view of jiffies and last_jiffies_update. */ static ktime_t last_jiffies_update;
[tip: timers/core] tick: Document protections for tick related data
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: c398960cd82b233886fbff163986f998b5a5c008 Gitweb: https://git.kernel.org/tip/c398960cd82b233886fbff163986f998b5a5c008 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:44 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:28 +01:00
tick: Document protections for tick related data
The protection rules for tick_next_period and last_jiffies_update are blury at best. Clarify this.
--- kernel/time/tick-common.c | 4 +++- kernel/time/tick-sched.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-)
kernel/time/tick-common.c +3 -1
@@ -27,7 +27,9 @@*/ DEFINE_PER_CPU(struct tick_device, tick_cpu_device);/* -* Tick next event: keeps track of the tick time +* Tick next event: keeps track of the tick time. It's updated by the +* CPU which handles the tick and protected by jiffies_lock. There is +* no requirement to write hold the jiffies seqcount for it. */ ktime_t tick_next_period;ktime_t tick_period;
kernel/time/tick-sched.c +3 -1
@@ -44,7 +44,9 @@ struct tick_sched *tick_get_tick_sched(int cpu)#if defined(CONFIG_NO_HZ_COMMON ) || defined(CONFIG_HIGH_RES_TIMERS )/* -* The time, when the last jiffy update happened. Protected by jiffies_lock. +* The time, when the last jiffy update happened. Write access must hold +* jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a +* consistent view of jiffies and last_jiffies_update. */ static ktime_t last_jiffies_update;
PATCH 3/7 tick/sched: Use tick_next_period for lockless quick check
In reply to Thomas GleixnerNo point in doing calculations.
tick_next_period = last_jiffies_update + tick_period
Just check whether now is before tick_next_period to figure out whether jiffies need an update.
Add a comment why the intentional data race in the quick check is safe or not so safe in a 32bit corner case and why we don't worry about it.
--- kernel/time/tick-sched.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-)
kernel/time/tick-sched.c +33 -13
@@ -59,11 +59,29 @@ static void tick_do_update_jiffies64(ktiktime_t delta;/* -* Do a quick check without holding jiffies_lock: -* The READ_ONCE() pairs with two updates done later in this function. +* Do a quick check without holding jiffies_lock. The READ_ONCE() +* pairs with the update done later in this function. +* +* This is also an intentional data race which is even safe on +* 32bit in theory. If there is a concurrent update then the check +* might give a random answer. It does not matter because if it +* returns then the concurrent update is already taking care, if it +* falls through then it will pointlessly contend on jiffies_lock. +* +* Though there is one nasty case on 32bit due to store tearing of +* the 64bit value. If the first 32bit store makes the quick check +* return on all other CPUs and the writing CPU context gets +* delayed to complete the second store (scheduled out on virt) +* then jiffies can become stale for up to ~2^32 nanoseconds +* without noticing. After that point all CPUs will wait for +* jiffies lock. +* +* OTOH, this is not any different than the situation with NOHZ=off +* where one CPU is responsible for updating jiffies and +* timekeeping. If that CPU goes out for lunch then all other CPUs +* will operate on stale jiffies until it decides to come back. */ - delta =ktime_sub (now,READ_ONCE (last_jiffies_update)); -if (delta < tick_period) +if (ktime_before (now,READ_ONCE (tick_next_period)))return ;/* Reevaluate with jiffies_lock held */
@@ -74,9 +92,8 @@ static void tick_do_update_jiffies64(ktiif (delta >= tick_period) { delta =ktime_sub (delta, tick_period); -/* Pairs with the lockless read in this function. */ -WRITE_ONCE (last_jiffies_update, -ktime_add (last_jiffies_update, tick_period)); + last_jiffies_update =ktime_add (last_jiffies_update, + tick_period);/* Slow path for long timeouts */ if (unlikely (delta >= tick_period)) {
@@ -84,15 +101,18 @@ static void tick_do_update_jiffies64(kti ticks =ktime_divns (delta, incr); -/* Pairs with the lockless read in this function. */ -WRITE_ONCE (last_jiffies_update, -ktime_add_ns (last_jiffies_update, - incr * ticks)); + last_jiffies_update =ktime_add_ns (last_jiffies_update, + incr * ticks); }do_timer (++ticks); -/* Keep the tick_next_period variable up to date */ - tick_next_period =ktime_add (last_jiffies_update, tick_period); +/* +* Keep the tick_next_period variable up to date. +* WRITE_ONCE() pairs with the READ_ONCE() in the lockless +* quick check above. +*/ +WRITE_ONCE (tick_next_period, +ktime_add (last_jiffies_update, tick_period)); }else {write_seqcount_end (&jiffies_seq);raw_spin_unlock (&jiffies_lock);
[tip: timers/core] tick/sched: Use tick_next_period for lockless quick check
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: 372acbbaa80940189593f9d69c7c069955f24f7a Gitweb: https://git.kernel.org/tip/372acbbaa80940189593f9d69c7c069955f24f7a Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:45 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:29 +01:00
tick/sched: Use tick_next_period for lockless quick check
No point in doing calculations.
tick_next_period = last_jiffies_update + tick_period
Just check whether now is before tick_next_period to figure out whether jiffies need an update.
Add a comment why the intentional data race in the quick check is safe or not so safe in a 32bit corner case and why we don't worry about it.
--- kernel/time/tick-sched.c | 46 +++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-)
kernel/time/tick-sched.c +33 -13
@@ -59,11 +59,29 @@ static void tick_do_update_jiffies64(ktime_t now)ktime_t delta;/* -* Do a quick check without holding jiffies_lock: -* The READ_ONCE() pairs with two updates done later in this function. +* Do a quick check without holding jiffies_lock. The READ_ONCE() +* pairs with the update done later in this function. +* +* This is also an intentional data race which is even safe on +* 32bit in theory. If there is a concurrent update then the check +* might give a random answer. It does not matter because if it +* returns then the concurrent update is already taking care, if it +* falls through then it will pointlessly contend on jiffies_lock. +* +* Though there is one nasty case on 32bit due to store tearing of +* the 64bit value. If the first 32bit store makes the quick check +* return on all other CPUs and the writing CPU context gets +* delayed to complete the second store (scheduled out on virt) +* then jiffies can become stale for up to ~2^32 nanoseconds +* without noticing. After that point all CPUs will wait for +* jiffies lock. +* +* OTOH, this is not any different than the situation with NOHZ=off +* where one CPU is responsible for updating jiffies and +* timekeeping. If that CPU goes out for lunch then all other CPUs +* will operate on stale jiffies until it decides to come back. */ - delta =ktime_sub (now,READ_ONCE (last_jiffies_update)); -if (delta < tick_period) +if (ktime_before (now,READ_ONCE (tick_next_period)))return ;/* Reevaluate with jiffies_lock held */
@@ -74,9 +92,8 @@ static void tick_do_update_jiffies64(ktime_t now)if (delta >= tick_period) { delta =ktime_sub (delta, tick_period); -/* Pairs with the lockless read in this function. */ -WRITE_ONCE (last_jiffies_update, -ktime_add (last_jiffies_update, tick_period)); + last_jiffies_update =ktime_add (last_jiffies_update, + tick_period);/* Slow path for long timeouts */ if (unlikely (delta >= tick_period)) {
@@ -84,15 +101,18 @@ static void tick_do_update_jiffies64(ktime_t now) ticks =ktime_divns (delta, incr); -/* Pairs with the lockless read in this function. */ -WRITE_ONCE (last_jiffies_update, -ktime_add_ns (last_jiffies_update, - incr * ticks)); + last_jiffies_update =ktime_add_ns (last_jiffies_update, + incr * ticks); }do_timer (++ticks); -/* Keep the tick_next_period variable up to date */ - tick_next_period =ktime_add (last_jiffies_update, tick_period); +/* +* Keep the tick_next_period variable up to date. +* WRITE_ONCE() pairs with the READ_ONCE() in the lockless +* quick check above. +*/ +WRITE_ONCE (tick_next_period, +ktime_add (last_jiffies_update, tick_period)); }else {write_seqcount_end (&jiffies_seq);raw_spin_unlock (&jiffies_lock);
PATCH 4/7 tick/sched: Reduce seqcount held scope in tick_do_update_jiffies64()
In reply to Thomas GleixnerIf jiffies are up to date already (caller lost the race against another CPU) there is no point to change the sequence count. Doing that just forces other CPUs into the seqcount retry loop in tick_nohz_next_event() for nothing.
Just bail out early.
--- kernel/time/tick-sched.c | 53 ++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 28 deletions(-)
kernel/time/tick-sched.c +25 -28
@@ -86,38 +86,35 @@ static void tick_do_update_jiffies64(kti/* Reevaluate with jiffies_lock held */ raw_spin_lock (&jiffies_lock); +if (ktime_before (now, tick_next_period)) { +raw_spin_unlock (&jiffies_lock); +return ; + } +write_seqcount_begin (&jiffies_seq); - delta =ktime_sub (now, last_jiffies_update); -if (delta >= tick_period) { + last_jiffies_update =ktime_add (last_jiffies_update, tick_period); - delta =ktime_sub (delta, tick_period); - last_jiffies_update =ktime_add (last_jiffies_update, - tick_period); - -/* Slow path for long timeouts */ -if (unlikely (delta >= tick_period)) { -s64 incr =ktime_to_ns (tick_period); - - ticks =ktime_divns (delta, incr); - - last_jiffies_update =ktime_add_ns (last_jiffies_update, - incr * ticks); - } -do_timer (++ticks); - -/* -* Keep the tick_next_period variable up to date. -* WRITE_ONCE() pairs with the READ_ONCE() in the lockless -* quick check above. -*/ -WRITE_ONCE (tick_next_period, -ktime_add (last_jiffies_update, tick_period)); - }else { -write_seqcount_end (&jiffies_seq); -raw_spin_unlock (&jiffies_lock); -return ; + delta =ktime_sub (now, tick_next_period); +if (unlikely (delta >= tick_period)) { +/* Slow path for long idle sleep times */ +s64 incr =ktime_to_ns (tick_period); + + ticks =ktime_divns (delta, incr); + + last_jiffies_update =ktime_add_ns (last_jiffies_update, + incr * ticks); } + +do_timer (++ticks); + +/* +* Keep the tick_next_period variable up to date. WRITE_ONCE() +* pairs with the READ_ONCE() in the lockless quick check above. +*/ +WRITE_ONCE (tick_next_period, +ktime_add (last_jiffies_update, tick_period)); +write_seqcount_end (&jiffies_seq);raw_spin_unlock (&jiffies_lock);update_wall_time ();
[tip: timers/core] tick/sched: Reduce seqcount held scope in tick_do_update_jiffies64()
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: 94ad2e3cedb82af034f6d97c58022f162b669f9b Gitweb: https://git.kernel.org/tip/94ad2e3cedb82af034f6d97c58022f162b669f9b Author: Yunfeng Ye <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:46 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:29 +01:00
tick/sched: Reduce seqcount held scope in tick_do_update_jiffies64()
If jiffies are up to date already (caller lost the race against another CPU) there is no point to change the sequence count. Doing that just forces other CPUs into the seqcount retry loop in tick_nohz_next_event() for nothing.
Just bail out early.
[ tglx: Rewrote most of it ]
--- kernel/time/tick-sched.c | 47 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-)
kernel/time/tick-sched.c +22 -25
@@ -86,38 +86,35 @@ static void tick_do_update_jiffies64(ktime_t now)/* Reevaluate with jiffies_lock held */ raw_spin_lock (&jiffies_lock); +if (ktime_before (now, tick_next_period)) { +raw_spin_unlock (&jiffies_lock); +return ; + } +write_seqcount_begin (&jiffies_seq); - delta =ktime_sub (now, last_jiffies_update); -if (delta >= tick_period) { + last_jiffies_update =ktime_add (last_jiffies_update, tick_period); - delta =ktime_sub (delta, tick_period); - last_jiffies_update =ktime_add (last_jiffies_update, - tick_period); + delta =ktime_sub (now, tick_next_period); +if (unlikely (delta >= tick_period)) { +/* Slow path for long idle sleep times */ +s64 incr =ktime_to_ns (tick_period); -/* Slow path for long timeouts */ -if (unlikely (delta >= tick_period)) { -s64 incr =ktime_to_ns (tick_period); + ticks =ktime_divns (delta, incr); - ticks =ktime_divns (delta, incr); + last_jiffies_update =ktime_add_ns (last_jiffies_update, + incr * ticks); + } - last_jiffies_update =ktime_add_ns (last_jiffies_update, - incr * ticks); - } -do_timer (++ticks); +do_timer (++ticks); + +/* +* Keep the tick_next_period variable up to date. WRITE_ONCE() +* pairs with the READ_ONCE() in the lockless quick check above. +*/ +WRITE_ONCE (tick_next_period, +ktime_add (last_jiffies_update, tick_period)); -/* -* Keep the tick_next_period variable up to date. -* WRITE_ONCE() pairs with the READ_ONCE() in the lockless -* quick check above. -*/ -WRITE_ONCE (tick_next_period, -ktime_add (last_jiffies_update, tick_period)); - }else { -write_seqcount_end (&jiffies_seq); -raw_spin_unlock (&jiffies_lock); -return ; - }write_seqcount_end (&jiffies_seq);raw_spin_unlock (&jiffies_lock);update_wall_time ();
PATCH 5/7 tick/sched: Optimize tick_do_update_jiffies64() further
In reply to Thomas GleixnerNow that it's clear that there is always one tick to account, simplify the calculations some more.
--- kernel/time/tick-sched.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
kernel/time/tick-sched.c +6 -5
@@ -55,7 +55,7 @@ static ktime_t last_jiffies_update;*/ static void tick_do_update_jiffies64 (ktime_t now) { -unsigned long ticks =0 ; +unsigned long ticks =1 ;ktime_t delta;/*
@@ -93,20 +93,21 @@ static void tick_do_update_jiffies64(ktiwrite_seqcount_begin (&jiffies_seq); - last_jiffies_update =ktime_add (last_jiffies_update, tick_period); - delta =ktime_sub (now, tick_next_period);if (unlikely (delta >= tick_period)) {/* Slow path for long idle sleep times */ s64 incr =ktime_to_ns (tick_period); - ticks =ktime_divns (delta, incr); + ticks +=ktime_divns (delta, incr); last_jiffies_update =ktime_add_ns (last_jiffies_update, incr * ticks); + }else { + last_jiffies_update =ktime_add (last_jiffies_update, + tick_period); } -do_timer (++ticks); +do_timer (ticks);/* * Keep the tick_next_period variable up to date. WRITE_ONCE()
[tip: timers/core] tick/sched: Optimize tick_do_update_jiffies64() further
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: 7a35bf2a6a871cd0252cd371d741e7d070b53af9 Gitweb: https://git.kernel.org/tip/7a35bf2a6a871cd0252cd371d741e7d070b53af9 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:47 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:29 +01:00
tick/sched: Optimize tick_do_update_jiffies64() further
Now that it's clear that there is always one tick to account, simplify the calculations some more.
--- kernel/time/tick-sched.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
kernel/time/tick-sched.c +6 -5
@@ -55,7 +55,7 @@ static ktime_t last_jiffies_update;*/ static void tick_do_update_jiffies64 (ktime_t now) { -unsigned long ticks =0 ; +unsigned long ticks =1 ;ktime_t delta;/*
@@ -93,20 +93,21 @@ static void tick_do_update_jiffies64(ktime_t now)write_seqcount_begin (&jiffies_seq); - last_jiffies_update =ktime_add (last_jiffies_update, tick_period); - delta =ktime_sub (now, tick_next_period);if (unlikely (delta >= tick_period)) {/* Slow path for long idle sleep times */ s64 incr =ktime_to_ns (tick_period); - ticks =ktime_divns (delta, incr); + ticks +=ktime_divns (delta, incr); last_jiffies_update =ktime_add_ns (last_jiffies_update, incr * ticks); + }else { + last_jiffies_update =ktime_add (last_jiffies_update, + tick_period); } -do_timer (++ticks); +do_timer (ticks);/* * Keep the tick_next_period variable up to date. WRITE_ONCE()
PATCH 6/7 tick/sched: Release seqcount before invoking calc_load_global()
In reply to Thomas Gleixnercalc_load_global() does not need the sequence count protection.
--- kernel/time/tick-sched.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
kernel/time/tick-sched.c +11 -1
@@ -20,6 +20,7 @@#include <linux/sched/clock.h> #include <linux/sched/stat.h> #include <linux/sched/nohz.h> +#include <linux/sched/loadavg.h> #include <linux/module.h> #include <linux/irq_work.h> #include <linux/posix-timers.h>
@@ -107,7 +108,8 @@ static void tick_do_update_jiffies64(kti tick_period); } -do_timer (ticks); +/* Advance jiffies to complete the jiffies_seq protected job */ + jiffies_64 += ticks;/* * Keep the tick_next_period variable up to date. WRITE_ONCE()
@@ -116,7 +118,15 @@ static void tick_do_update_jiffies64(ktiWRITE_ONCE (tick_next_period,ktime_add (last_jiffies_update, tick_period)); +/* +* Release the sequence count. calc_global_load() below is not +* protected by it, but jiffies_lock needs to be held to prevent +* concurrent invocations. +*/ write_seqcount_end (&jiffies_seq); + +calc_global_load (); +raw_spin_unlock (&jiffies_lock);update_wall_time (); }
[tip: timers/core] tick/sched: Release seqcount before invoking calc_load_global()
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: 896b969e6732b68ee3c12ae4e1aeddf5db99bc46 Gitweb: https://git.kernel.org/tip/896b969e6732b68ee3c12ae4e1aeddf5db99bc46 Author: Yunfeng Ye <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:48 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:29 +01:00
tick/sched: Release seqcount before invoking calc_load_global()
calc_load_global() does not need the sequence count protection.
[ tglx: Split it up properly and added comments ]
--- kernel/time/tick-sched.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
kernel/time/tick-sched.c +11 -1
@@ -20,6 +20,7 @@#include <linux/sched/clock.h> #include <linux/sched/stat.h> #include <linux/sched/nohz.h> +#include <linux/sched/loadavg.h> #include <linux/module.h> #include <linux/irq_work.h> #include <linux/posix-timers.h>
@@ -107,7 +108,8 @@ static void tick_do_update_jiffies64(ktime_t now) tick_period); } -do_timer (ticks); +/* Advance jiffies to complete the jiffies_seq protected job */ + jiffies_64 += ticks;/* * Keep the tick_next_period variable up to date. WRITE_ONCE()
@@ -116,7 +118,15 @@ static void tick_do_update_jiffies64(ktime_t now)WRITE_ONCE (tick_next_period,ktime_add (last_jiffies_update, tick_period)); +/* +* Release the sequence count. calc_global_load() below is not +* protected by it, but jiffies_lock needs to be held to prevent +* concurrent invocations. +*/ write_seqcount_end (&jiffies_seq); + +calc_global_load (); +raw_spin_unlock (&jiffies_lock);update_wall_time (); }
PATCH 7/7 tick: Get rid of tick_period
In reply to Thomas GleixnerThe variable tick_period is initialized to NSEC_PER_TICK / HZ during boot and never updated again.
If NSEC_PER_TICK is not an integer multiple of HZ this computation is less accurate than TICK_NSEC which has proper rounding in place.
Aside of the inaccuracy there is no reason for having this variable at all. It's just a pointless indirection and all usage sites can just use the TICK_NSEC constant.
--- kernel/time/tick-broadcast.c | 2 +- kernel/time/tick-common.c | 8 +++----- kernel/time/tick-internal.h | 1 - kernel/time/tick-sched.c | 22 +++++++++++----------- 4 files changed, 15 insertions(+), 18 deletions(-)
kernel/time/tick-broadcast.c +1 -1
@@ -331,7 +331,7 @@ static void tick_handle_periodic_broadca bc_local =tick_do_periodic_broadcast ();if (clockevent_state_oneshot (dev)) { -ktime_t next =ktime_add (dev->next_event, tick_period); +ktime_t next =ktime_add_ns (dev->next_event,TICK_NSEC );clockevents_program_event (dev, next, true); }
kernel/time/tick-common.c +3 -5
@@ -32,7 +32,6 @@ DEFINE_PER_CPU(struct tick_device, tick_* no requirement to write hold the jiffies seqcount for it. */ ktime_t tick_next_period; -ktime_t tick_period;/* * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
@@ -90,7 +89,7 @@ static void tick_periodic(int cpu)write_seqcount_begin (&jiffies_seq);/* Keep track of the next tick event */ - tick_next_period =ktime_add (tick_next_period, tick_period); + tick_next_period =ktime_add_ns (tick_next_period,TICK_NSEC );do_timer (1 );write_seqcount_end (&jiffies_seq);
@@ -129,7 +128,7 @@ void tick_handle_periodic(struct clock_e* Setup the next period for devices, which do not have * periodic mode: */ - next =ktime_add (next, tick_period); + next =ktime_add_ns (next,TICK_NSEC );if (!clockevents_program_event (dev, next, false))return ;
@@ -175,7 +174,7 @@ void tick_setup_periodic(struct clock_evfor (;;) {if (!clockevents_program_event (dev, next, false))return ; - next =ktime_add (next, tick_period); + next =ktime_add_ns (next,TICK_NSEC ); } } }
@@ -222,7 +221,6 @@ static void tick_setup_device(struct tic tick_do_timer_cpu = cpu; tick_next_period =ktime_get (); - tick_period =NSEC_PER_SEC /HZ ;#ifdef CONFIG_NO_HZ_FULL /* * The boot CPU may be nohz_full, in which case set
kernel/time/tick-internal.h +0 -1
@@ -15,7 +15,6 @@ DECLARE_PER_CPU(struct tick_device, tick_cpu_device);extern ktime_t tick_next_period; -extern ktime_t tick_period;extern int tick_do_timer_cpu__read_mostly ;extern void tick_setup_periodic (struct clock_event_device *dev,int broadcast);
kernel/time/tick-sched.c +11 -11
@@ -95,17 +95,17 @@ static void tick_do_update_jiffies64(ktiwrite_seqcount_begin (&jiffies_seq); delta =ktime_sub (now, tick_next_period); -if (unlikely (delta >= tick_period)) { +if (unlikely (delta >=TICK_NSEC )) {/* Slow path for long idle sleep times */ -s64 incr =ktime_to_ns (tick_period); +s64 incr =TICK_NSEC ; ticks +=ktime_divns (delta, incr); last_jiffies_update =ktime_add_ns (last_jiffies_update, incr * ticks); }else { - last_jiffies_update =ktime_add (last_jiffies_update, - tick_period); + last_jiffies_update =ktime_add_ns (last_jiffies_update, +TICK_NSEC ); }/* Advance jiffies to complete the jiffies_seq protected job */
@@ -116,7 +116,7 @@ static void tick_do_update_jiffies64(kti* pairs with the READ_ONCE() in the lockless quick check above. */ WRITE_ONCE (tick_next_period, -ktime_add (last_jiffies_update, tick_period)); +ktime_add_ns (last_jiffies_update,TICK_NSEC ));/* * Release the sequence count. calc_global_load() below is not
@@ -691,7 +691,7 @@ static void tick_nohz_restart(struct tichrtimer_set_expires (&ts->sched_timer, ts->last_tick);/* Forward the time to expire in the future */ -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );if (ts->nohz_mode ==NOHZ_MODE_HIGHRES ) { hrtimer_start_expires(&ts->sched_timer,
@@ -1260,7 +1260,7 @@ static void tick_nohz_handler(struct cloif (unlikely (ts->tick_stopped))return ; -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );tick_program_event (hrtimer_get_expires (&ts->sched_timer),1 ); }
@@ -1297,7 +1297,7 @@ static void tick_nohz_switch_to_nohz(voi next =tick_init_jiffy_update ();hrtimer_set_expires (&ts->sched_timer, next); -hrtimer_forward_now (&ts->sched_timer, tick_period); +hrtimer_forward_now (&ts->sched_timer,TICK_NSEC );tick_program_event (hrtimer_get_expires (&ts->sched_timer),1 );tick_nohz_activate (ts,NOHZ_MODE_LOWRES ); }
@@ -1363,7 +1363,7 @@ static enum hrtimer_restart tick_sched_tif (unlikely (ts->tick_stopped))return HRTIMER_NORESTART ; -hrtimer_forward (timer, now, tick_period); +hrtimer_forward (timer, now,TICK_NSEC );return HRTIMER_RESTART ; }
@@ -1397,13 +1397,13 @@ void tick_setup_sched_timer(void)/* Offset the tick to avert jiffies_lock contention. */ if (sched_skew_tick) { -u64 offset =ktime_to_ns (tick_period) >>1 ; +u64 offset =TICK_NSEC >>1 ;do_div (offset,num_possible_cpus ()); offset *=smp_processor_id ();hrtimer_add_expires_ns (&ts->sched_timer, offset); } -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );hrtimer_start_expires (&ts->sched_timer,HRTIMER_MODE_ABS_PINNED_HARD );tick_nohz_activate (ts,NOHZ_MODE_HIGHRES ); }
[tip: timers/core] tick: Get rid of tick_period
The following commit has been merged into the timers/core branch of tip:
The following commit has been merged into the timers/core branch of tip:
Commit-ID: b996544916429946bf4934c1c01a306d1690972c Gitweb: https://git.kernel.org/tip/b996544916429946bf4934c1c01a306d1690972c Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Nov 2020 14:19:49 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Thu, 19 Nov 2020 10:48:29 +01:00
tick: Get rid of tick_period
The variable tick_period is initialized to NSEC_PER_TICK / HZ during boot and never updated again.
If NSEC_PER_TICK is not an integer multiple of HZ this computation is less accurate than TICK_NSEC which has proper rounding in place.
Aside of the inaccuracy there is no reason for having this variable at all. It's just a pointless indirection and all usage sites can just use the TICK_NSEC constant.
--- kernel/time/tick-broadcast.c | 2 +- kernel/time/tick-common.c | 8 +++----- kernel/time/tick-internal.h | 1 - kernel/time/tick-sched.c | 22 +++++++++++----------- 4 files changed, 15 insertions(+), 18 deletions(-)
kernel/time/tick-broadcast.c +1 -1
@@ -331,7 +331,7 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev) bc_local =tick_do_periodic_broadcast ();if (clockevent_state_oneshot (dev)) { -ktime_t next =ktime_add (dev->next_event, tick_period); +ktime_t next =ktime_add_ns (dev->next_event,TICK_NSEC );clockevents_program_event (dev, next, true); }
kernel/time/tick-common.c +3 -5
@@ -32,7 +32,6 @@ DEFINE_PER_CPU(struct tick_device, tick_cpu_device);* no requirement to write hold the jiffies seqcount for it. */ ktime_t tick_next_period; -ktime_t tick_period;/* * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
@@ -90,7 +89,7 @@ static void tick_periodic(int cpu)write_seqcount_begin (&jiffies_seq);/* Keep track of the next tick event */ - tick_next_period =ktime_add (tick_next_period, tick_period); + tick_next_period =ktime_add_ns (tick_next_period,TICK_NSEC );do_timer (1 );write_seqcount_end (&jiffies_seq);
@@ -129,7 +128,7 @@ void tick_handle_periodic(struct clock_event_device *dev)* Setup the next period for devices, which do not have * periodic mode: */ - next =ktime_add (next, tick_period); + next =ktime_add_ns (next,TICK_NSEC );if (!clockevents_program_event (dev, next, false))return ;
@@ -175,7 +174,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)for (;;) {if (!clockevents_program_event (dev, next, false))return ; - next =ktime_add (next, tick_period); + next =ktime_add_ns (next,TICK_NSEC ); } } }
@@ -222,7 +221,6 @@ static void tick_setup_device(struct tick_device *td, tick_do_timer_cpu = cpu; tick_next_period =ktime_get (); - tick_period =NSEC_PER_SEC /HZ ;#ifdef CONFIG_NO_HZ_FULL /* * The boot CPU may be nohz_full, in which case set
kernel/time/tick-internal.h +0 -1
@@ -15,7 +15,6 @@ DECLARE_PER_CPU(struct tick_device, tick_cpu_device);extern ktime_t tick_next_period; -extern ktime_t tick_period;extern int tick_do_timer_cpu__read_mostly ;extern void tick_setup_periodic (struct clock_event_device *dev,int broadcast);
kernel/time/tick-sched.c +11 -11
@@ -95,17 +95,17 @@ static void tick_do_update_jiffies64(ktime_t now)write_seqcount_begin (&jiffies_seq); delta =ktime_sub (now, tick_next_period); -if (unlikely (delta >= tick_period)) { +if (unlikely (delta >=TICK_NSEC )) {/* Slow path for long idle sleep times */ -s64 incr =ktime_to_ns (tick_period); +s64 incr =TICK_NSEC ; ticks +=ktime_divns (delta, incr); last_jiffies_update =ktime_add_ns (last_jiffies_update, incr * ticks); }else { - last_jiffies_update =ktime_add (last_jiffies_update, - tick_period); + last_jiffies_update =ktime_add_ns (last_jiffies_update, +TICK_NSEC ); }/* Advance jiffies to complete the jiffies_seq protected job */
@@ -116,7 +116,7 @@ static void tick_do_update_jiffies64(ktime_t now)* pairs with the READ_ONCE() in the lockless quick check above. */ WRITE_ONCE (tick_next_period, -ktime_add (last_jiffies_update, tick_period)); +ktime_add_ns (last_jiffies_update,TICK_NSEC ));/* * Release the sequence count. calc_global_load() below is not
@@ -691,7 +691,7 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)hrtimer_set_expires (&ts->sched_timer, ts->last_tick);/* Forward the time to expire in the future */ -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );if (ts->nohz_mode ==NOHZ_MODE_HIGHRES ) { hrtimer_start_expires(&ts->sched_timer,
@@ -1260,7 +1260,7 @@ static void tick_nohz_handler(struct clock_event_device *dev)if (unlikely (ts->tick_stopped))return ; -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );tick_program_event (hrtimer_get_expires (&ts->sched_timer),1 ); }
@@ -1297,7 +1297,7 @@ static void tick_nohz_switch_to_nohz(void) next =tick_init_jiffy_update ();hrtimer_set_expires (&ts->sched_timer, next); -hrtimer_forward_now (&ts->sched_timer, tick_period); +hrtimer_forward_now (&ts->sched_timer,TICK_NSEC );tick_program_event (hrtimer_get_expires (&ts->sched_timer),1 );tick_nohz_activate (ts,NOHZ_MODE_LOWRES ); }
@@ -1363,7 +1363,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)if (unlikely (ts->tick_stopped))return HRTIMER_NORESTART ; -hrtimer_forward (timer, now, tick_period); +hrtimer_forward (timer, now,TICK_NSEC );return HRTIMER_RESTART ; }
@@ -1397,13 +1397,13 @@ void tick_setup_sched_timer(void)/* Offset the tick to avert jiffies_lock contention. */ if (sched_skew_tick) { -u64 offset =ktime_to_ns (tick_period) >>1 ; +u64 offset =TICK_NSEC >>1 ;do_div (offset,num_possible_cpus ()); offset *=smp_processor_id ();hrtimer_add_expires_ns (&ts->sched_timer, offset); } -hrtimer_forward (&ts->sched_timer, now, tick_period); +hrtimer_forward (&ts->sched_timer, now,TICK_NSEC );hrtimer_start_expires (&ts->sched_timer,HRTIMER_MODE_ABS_PINNED_HARD );tick_nohz_activate (ts,NOHZ_MODE_HIGHRES ); }