sched, x86: Fix cpufreq + sched_clock() TSC scaling
Commit-ID: 702dd8c6624b353eb48dd20a0d36405cfe4f4f99 Gitweb: http://git.kernel.org/tip/702dd8c6624b353eb48dd20a0d36405cfe4f4f99 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 16 Jun 2009 12:34:17 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 16 Jun 2009 21:48:01 +0200
sched, x86: Fix cpufreq + sched_clock() TSC scaling
For freqency dependent TSCs we only scale the cycles, we do not account for the discrepancy in absolute value. Add a constant factor so that:
c1 + f1*cyc == c2 + f2*cyc
[ Impact: fix/reduce sched_clock() jumps across frequency changing events ]
Signed-off-by Peter Zijlstra <[email protected]>
Cc Peter Zijlstra <[email protected]>
LKML-Reference: <new-submission>
Signed-off-by Ingo Molnar <[email protected]>
Chucked-on-by Thomas Gleixner <[email protected]>
Signed-off-by Andrew Morton <[email protected]>
--- arch/x86/include/asm/timer.h | 6 +++++- arch/x86/kernel/tsc.c | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-)
arch/x86/include/asm/timer.h +5 -1
@@ -45,12 +45,16 @@ extern int no_timer_check;*/ DECLARE_PER_CPU(unsigned long, cyc2ns); +DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ static inline unsigned long long __cycles_2_ns (unsigned long long cyc) { -return cyc *per_cpu (cyc2ns,smp_processor_id ()) >>CYC2NS_SCALE_FACTOR ; +int cpu =smp_processor_id (); +unsigned long long ns =per_cpu (cyc2ns_offset, cpu); + ns += cyc *per_cpu (cyc2ns, cpu) >>CYC2NS_SCALE_FACTOR ; +return ns; }static inline unsigned long long cycles_2_ns (unsigned long long cyc)
arch/x86/kernel/tsc.c +6 -2
@@ -589,22 +589,26 @@ EXPORT_SYMBOL(recalibrate_cpu_khz);*/ DEFINE_PER_CPU(unsigned long, cyc2ns); +DEFINE_PER_CPU(unsigned long long, cyc2ns_offset);static void set_cyc2ns_scale (unsigned long cpu_khz,int cpu) { -unsigned long long tsc_now, ns_now; +unsigned long long tsc_now, ns_now, *offset;unsigned long flags, *scale;local_irq_save (flags);sched_clock_idle_sleep_event (); scale = &per_cpu (cyc2ns, cpu); + offset = &per_cpu (cyc2ns_offset, cpu);rdtscll (tsc_now); ns_now =__cycles_2_ns (tsc_now); -if (cpu_khz) +if (cpu_khz) { *scale = (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR )/cpu_khz; + *offset = ns_now - (tsc_now * *scale >>CYC2NS_SCALE_FACTOR ); + }sched_clock_idle_wakeup_event (0 );local_irq_restore (flags);