Generic sched_clock locking fix for tip/timers/urgent
PATCH 1/1 Generic sched_clock locking fix for tip/timers/urgent
Hey Ingo, Thomas,
Wanted to send along this sched_clock fix from Stephen for
tip/timers/urgent. Let me know if you have any issues or concerns.
thanks
-john
Stephen Boyd (1):
sched_clock: Prevent callers from seeing half-updated data
kernel/time/sched_clock.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-)
Signature · 1 line
1.8.3.2
PATCH 1/1 sched_clock: Prevent callers from seeing half-updated data
From: Stephen Boyd <[email protected]>
The generic sched_clock registration function was previously done lockless, due to the fact that it was expected to be called only once. However, now there are systems that may register multiple sched_clock sources, for which the lack of locking has casued problems:
If two sched_clock sources are registered we may end up in a situation where a call to sched_clock() may be accessing the epoch cycle count for the old counter and the cycle count for the new counter. This can lead to confusing results where sched_clock() values jump and then are reset to 0 (due to the way the registration function forces the epoch_ns to be 0).
Fix this by reorganizing the registration function to hold the seqlock for as short a time as possible while we update the clock_data structure for a new counter. We also put any accumulated time into epoch_ns instead of resetting the time to 0 so that the clock doesn't reset after each successful registration.
--- kernel/time/sched_clock.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-)
kernel/time/sched_clock.c +29 -17
@@ -116,20 +116,42 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)void __init sched_clock_register (u64 (*read)(void ),int bits,unsigned long rate) { +u64 res, wrap, new_mask, new_epoch, cyc, ns; +u32 new_mult, new_shift; +ktime_t new_wrap_kt;unsigned long r; -u64 res, wrap;char r_unit;if (cd.rate > rate)return ;WARN_ON (!irqs_disabled ()); - read_sched_clock = read; - sched_clock_mask =CLOCKSOURCE_MASK (bits); - cd.rate = rate;/* calculate the mult/shift to convert counter ticks to ns. */ -clocks_calc_mult_shift (&cd.mult, &cd.shift, rate,NSEC_PER_SEC ,3600 ); +clocks_calc_mult_shift (&new_mult, &new_shift, rate,NSEC_PER_SEC ,3600 ); + + new_mask =CLOCKSOURCE_MASK (bits); + +/* calculate how many ns until we wrap */ + wrap =clocks_calc_max_nsecs (new_mult, new_shift,0 , new_mask); + new_wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); + +/* update epoch for new counter and update epoch_ns from old counter*/ + new_epoch =read (); + cyc =read_sched_clock (); + ns = cd.epoch_ns +cyc_to_ns ((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + +raw_write_seqcount_begin (&cd.seq); + read_sched_clock = read; + sched_clock_mask = new_mask; + cd.rate = rate; + cd.wrap_kt = new_wrap_kt; + cd.mult = new_mult; + cd.shift = new_shift; + cd.epoch_cyc = new_epoch; + cd.epoch_ns = ns; +raw_write_seqcount_end (&cd.seq); r = rate;if (r >=4000000 ) {
@@ -141,22 +163,12 @@ void __init sched_clock_register(u64 (*read)(void), int bits, }else r_unit =' ' ; -/* calculate how many ns until we wrap */ - wrap =clocks_calc_max_nsecs (cd.mult, cd.shift,0 , sched_clock_mask); - cd.wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); -/* calculate the ns resolution of this counter */ - res =cyc_to_ns (1ULL , cd.mult, cd.shift); + res =cyc_to_ns (1ULL , new_mult, new_shift); +pr_info ("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n" , bits, r, r_unit, res, wrap); -update_sched_clock (); - -/* -* Ensure that sched_clock() starts off at 0ns -*/ - cd.epoch_ns =0 ; -/* Enable IRQ time accounting if we have a fast enough sched_clock */ if (irqtime >0 || (irqtime ==-1 && rate >=1000000 ))enable_sched_clock_irqtime ();
Signature · 1 line
1.8.3.2
Re: [PATCH] sched_clock: Prevent callers from seeing half-updated data
Quoted 15 lines · John Stultz
From: Stephen Boyd <[email protected]>
The generic
sched_clockregistration function was previously done lockless, due to the fact that it was expected to be called only once. However, now there are systems that may register multiplesched_clocksources, for which the lack of locking has casued problems:If two
sched_clocksources are registered we may end up in a situation where a call tosched_clock()may be accessing the epoch cycle count for the old counter and the cycle count for the new counter. This can lead to confusing results wheresched_clock()values jump and then are reset to 0 (due to the way the registration function forces theepoch_nsto be 0).
Quoted 8 lines
Fix this by reorganizing the registration function to hold the seqlock for as short a time as possible while we update the
clock_datastructure for a new counter. We also put any accumulated time intoepoch_nsinstead of resetting the time to 0 so that the clock doesn't reset after each successful registration.Cc Peter Zijlstra <[email protected]>
Peter ???
Quoted 94 lines
Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Stephen Boyd <[email protected]>Cc Will Deacon <[email protected]>Cc Josh Cartwright <[email protected]>Reported-by Will Deacon <[email protected]>Signed-off-by Stephen Boyd <[email protected]>[jstultz: Added extra context to the commit message]Signed-off-by John Stultz <[email protected]>--- kernel/time/sched_clock.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-)kernel/time/sched_clock.c +29
-17@@ -116,20 +116,42 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)void __init sched_clock_register (u64 (*read)(void ),int bits,unsigned long rate) { +u64 res, wrap, new_mask, new_epoch, cyc, ns; +u32 new_mult, new_shift; +ktime_t new_wrap_kt;unsigned long r; -u64 res, wrap;char r_unit;if (cd.rate > rate)return ;WARN_ON (!irqs_disabled ()); - read_sched_clock = read; - sched_clock_mask =CLOCKSOURCE_MASK (bits); - cd.rate = rate;/* calculate the mult/shift to convert counter ticks to ns. */ -clocks_calc_mult_shift (&cd.mult, &cd.shift, rate,NSEC_PER_SEC ,3600 ); +clocks_calc_mult_shift (&new_mult, &new_shift, rate,NSEC_PER_SEC ,3600 ); + + new_mask =CLOCKSOURCE_MASK (bits); + +/* calculate how many ns until we wrap */ + wrap =clocks_calc_max_nsecs (new_mult, new_shift,0 , new_mask); + new_wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); + +/* update epoch for new counter and update epoch_ns from old counter*/ + new_epoch =read (); + cyc =read_sched_clock (); + ns = cd.epoch_ns +cyc_to_ns ((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + +raw_write_seqcount_begin (&cd.seq); + read_sched_clock = read; + sched_clock_mask = new_mask; + cd.rate = rate; + cd.wrap_kt = new_wrap_kt; + cd.mult = new_mult; + cd.shift = new_shift; + cd.epoch_cyc = new_epoch; + cd.epoch_ns = ns; +raw_write_seqcount_end (&cd.seq); r = rate;if (r >=4000000 ) {@@ -141,22 +163,12 @@ void __init sched_clock_register(u64 (*read)(void), int bits, }else r_unit =' ' ; -/* calculate how many ns until we wrap */ - wrap =clocks_calc_max_nsecs (cd.mult, cd.shift,0 , sched_clock_mask); - cd.wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); -/* calculate the ns resolution of this counter */ - res =cyc_to_ns (1ULL , cd.mult, cd.shift); + res =cyc_to_ns (1ULL , new_mult, new_shift); +pr_info ("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n" , bits, r, r_unit, res, wrap); -update_sched_clock (); - -/* -* Ensure that sched_clock() starts off at 0ns -*/ - cd.epoch_ns =0 ; -/* Enable IRQ time accounting if we have a fast enough sched_clock */ if (irqtime >0 || (irqtime ==-1 && rate >=1000000 ))enable_sched_clock_irqtime ();Signature · 1 line
1.8.3.2
Re: [PATCH] sched_clock: Prevent callers from seeing half-updated data
Quoted 24 lines · Thomas Gleixner
Quoted 22 lines · John Stultz
From: Stephen Boyd <[email protected]>
The generic
sched_clockregistration function was previously done lockless, due to the fact that it was expected to be called only once. However, now there are systems that may register multiplesched_clocksources, for which the lack of locking has casued problems:If two
sched_clocksources are registered we may end up in a situation where a call tosched_clock()may be accessing the epoch cycle count for the old counter and the cycle count for the new counter. This can lead to confusing results wheresched_clock()values jump and then are reset to 0 (due to the way the registration function forces theepoch_nsto be 0).Fix this by reorganizing the registration function to hold the seqlock for as short a time as possible while we update the
clock_datastructure for a new counter. We also put any accumulated time intoepoch_nsinstead of resetting the time to 0 so that the clock doesn't reset after each successful registration.
Quoted 3 lines
Quoted 1 line
Cc Peter Zijlstra <[email protected]>Peter ???
So this is the generic sched_clock work, which Peter really hasn't had much involvement with yet (mostly because its not yet generic enough to work with more then a few arches). But I included him in the CC since I think it would be good to have him following along.
thanks
-john
Re: [PATCH] sched_clock: Prevent callers from seeing half-updated data
Quoted 25 lines · John Stultz
Quoted 24 lines · Thomas Gleixner
Quoted 22 lines · John Stultz
From: Stephen Boyd <[email protected]>
The generic
sched_clockregistration function was previously done lockless, due to the fact that it was expected to be called only once. However, now there are systems that may register multiplesched_clocksources, for which the lack of locking has casued problems:If two
sched_clocksources are registered we may end up in a situation where a call tosched_clock()may be accessing the epoch cycle count for the old counter and the cycle count for the new counter. This can lead to confusing results wheresched_clock()values jump and then are reset to 0 (due to the way the registration function forces theepoch_nsto be 0).Fix this by reorganizing the registration function to hold the seqlock for as short a time as possible while we update the
clock_datastructure for a new counter. We also put any accumulated time intoepoch_nsinstead of resetting the time to 0 so that the clock doesn't reset after each successful registration.
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
Cc Peter Zijlstra <[email protected]>Peter ???
So this is the generic
sched_clockwork, which Peter really hasn't had much involvement with yet (mostly because its not yet generic enough to work with more then a few arches). But I included him in the CC since I think it would be good to have him following along.
Fair enough. I'll pick it up.
Thanks,
tglx
tip-bot for Stephen Boyd
merge bot[tip:timers/urgent] sched_clock: Prevent callers from seeing half-updated data
In reply to John Stultzsched_clock: Prevent callers from seeing half-updated data
Commit-ID: 5ae8aabeaec3fe69c4fb21cbe5b17b72b35b5892 Gitweb: http://git.kernel.org/tip/5ae8aabeaec3fe69c4fb21cbe5b17b72b35b5892 Author: Stephen Boyd <[email protected]> AuthorDate: Mon, 17 Feb 2014 10:45:36 -0800 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 19 Feb 2014 17:07:22 +0100
sched_clock: Prevent callers from seeing half-updated data
The generic sched_clock registration function was previously done lockless, due to the fact that it was expected to be called only once. However, now there are systems that may register multiple sched_clock sources, for which the lack of locking has casued problems:
If two sched_clock sources are registered we may end up in a situation where a call to sched_clock() may be accessing the epoch cycle count for the old counter and the cycle count for the new counter. This can lead to confusing results where sched_clock() values jump and then are reset to 0 (due to the way the registration function forces the epoch_ns to be 0).
Fix this by reorganizing the registration function to hold the seqlock for as short a time as possible while we update the clock_data structure for a new counter. We also put any accumulated time into epoch_ns instead of resetting the time to 0 so that the clock doesn't reset after each successful registration.
[jstultz: Added extra context to the commit message]
--- kernel/time/sched_clock.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-)
kernel/time/sched_clock.c +29 -17
@@ -116,20 +116,42 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)void __init sched_clock_register (u64 (*read)(void ),int bits,unsigned long rate) { +u64 res, wrap, new_mask, new_epoch, cyc, ns; +u32 new_mult, new_shift; +ktime_t new_wrap_kt;unsigned long r; -u64 res, wrap;char r_unit;if (cd.rate > rate)return ;WARN_ON (!irqs_disabled ()); - read_sched_clock = read; - sched_clock_mask =CLOCKSOURCE_MASK (bits); - cd.rate = rate;/* calculate the mult/shift to convert counter ticks to ns. */ -clocks_calc_mult_shift (&cd.mult, &cd.shift, rate,NSEC_PER_SEC ,3600 ); +clocks_calc_mult_shift (&new_mult, &new_shift, rate,NSEC_PER_SEC ,3600 ); + + new_mask =CLOCKSOURCE_MASK (bits); + +/* calculate how many ns until we wrap */ + wrap =clocks_calc_max_nsecs (new_mult, new_shift,0 , new_mask); + new_wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); + +/* update epoch for new counter and update epoch_ns from old counter*/ + new_epoch =read (); + cyc =read_sched_clock (); + ns = cd.epoch_ns +cyc_to_ns ((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + +raw_write_seqcount_begin (&cd.seq); + read_sched_clock = read; + sched_clock_mask = new_mask; + cd.rate = rate; + cd.wrap_kt = new_wrap_kt; + cd.mult = new_mult; + cd.shift = new_shift; + cd.epoch_cyc = new_epoch; + cd.epoch_ns = ns; +raw_write_seqcount_end (&cd.seq); r = rate;if (r >=4000000 ) {
@@ -141,22 +163,12 @@ void __init sched_clock_register(u64 (*read)(void), int bits, }else r_unit =' ' ; -/* calculate how many ns until we wrap */ - wrap =clocks_calc_max_nsecs (cd.mult, cd.shift,0 , sched_clock_mask); - cd.wrap_kt =ns_to_ktime (wrap - (wrap >>3 )); -/* calculate the ns resolution of this counter */ - res =cyc_to_ns (1ULL , cd.mult, cd.shift); + res =cyc_to_ns (1ULL , new_mult, new_shift); +pr_info ("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n" , bits, r, r_unit, res, wrap); -update_sched_clock (); - -/* -* Ensure that sched_clock() starts off at 0ns -*/ - cd.epoch_ns =0 ; -/* Enable IRQ time accounting if we have a fast enough sched_clock */ if (irqtime >0 || (irqtime ==-1 && rate >=1000000 ))enable_sched_clock_irqtime ();