perf: x86: Improve accuracy of perf/sched clock
PATCH v2 1/1 perf: x86: Improve accuracy of perf/sched clock
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because CYC2NS_SCALE_FACTOR is 10, the accuracy is +/- 1/2048
The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33)
--- arch/x86/kernel/tsc.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)
arch/x86/kernel/tsc.c +20 -13
@@ -167,21 +167,21 @@ static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data)* ns = cycles * cyc2ns_scale / SC * * And since SC is a constant power of two, we can convert the div -* into a shift. +* into a shift. The larger SC is, the more accurate the conversion, but +* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication +* (64-bit result) can be used. So start by trying SC = 2^32, reducing +* until the criteria are met. * -* We can use khz divisor instead of mhz to keep a better precision, since -* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. +* We can use khz divisor instead of mhz to keep a better precision. * ([email protected]) * * [email protected] "math is hard, lets go shopping!" */ -#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static void cyc2ns_data_init (struct cyc2ns_data *data) { data->cyc2ns_mul =0 ; - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + data->cyc2ns_shift =0 ; data->cyc2ns_offset =0 ; data->__count =0 ; }
@@ -215,14 +215,14 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)if (likely (data == tail)) { ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift); }else { data->__count++;barrier (); ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift);barrier ();
@@ -239,6 +239,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)unsigned long long tsc_now, ns_now;struct cyc2ns_data *data;unsigned long flags; +u64 mult; +u32 shft =32 ;local_irq_save (flags);sched_clock_idle_sleep_event ();
@@ -256,12 +258,17 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + mult = (u64 )NSEC_PER_MSEC <<32 ; + mult += cpu_khz /2 ; +do_div (mult, cpu_khz); +while (mult >U32_MAX ) { + mult >>=1 ; + shft -=1 ; + } + data->cyc2ns_mul = mult; + data->cyc2ns_shift = shft; data->cyc2ns_offset = ns_now - -mul_u64_u32_shr (tsc_now, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); +mul_u64_u32_shr (tsc_now, data->cyc2ns_mul, data->cyc2ns_shift);cyc2ns_write_end (cpu, data);
Signature · 1 line
1.9.1
Quoted 5 lines · Adrian Hunter
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because
CYC2NS_SCALE_FACTORis 10, the accuracy is +/- 1/2048
Quoted 7 lines
The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33)
Signed-off-by Adrian Hunter <[email protected]>
Is this OK?
Quoted 86 lines
--- arch/x86/kernel/tsc.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)arch/x86/kernel/tsc.c +20
-13@@ -167,21 +167,21 @@ static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data)* ns = cycles * cyc2ns_scale / SC * * And since SC is a constant power of two, we can convert the div -* into a shift. +* into a shift. The larger SC is, the more accurate the conversion, but +* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication +* (64-bit result) can be used. So start by trying SC = 2^32, reducing +* until the criteria are met. * -* We can use khz divisor instead of mhz to keep a better precision, since -* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. +* We can use khz divisor instead of mhz to keep a better precision. * ([email protected]) * * [email protected] "math is hard, lets go shopping!" */ -#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static void cyc2ns_data_init (struct cyc2ns_data *data) { data->cyc2ns_mul =0 ; - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + data->cyc2ns_shift =0 ; data->cyc2ns_offset =0 ; data->__count =0 ; }@@ -215,14 +215,14 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)if (likely (data == tail)) { ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift); }else { data->__count++;barrier (); ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift);barrier ();@@ -239,6 +239,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)unsigned long long tsc_now, ns_now;struct cyc2ns_data *data;unsigned long flags; +u64 mult; +u32 shft =32 ;local_irq_save (flags);sched_clock_idle_sleep_event ();@@ -256,12 +258,17 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + mult = (u64 )NSEC_PER_MSEC <<32 ; + mult += cpu_khz /2 ; +do_div (mult, cpu_khz); +while (mult >U32_MAX ) { + mult >>=1 ; + shft -=1 ; + } + data->cyc2ns_mul = mult; + data->cyc2ns_shift = shft; data->cyc2ns_offset = ns_now - -mul_u64_u32_shr (tsc_now, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); +mul_u64_u32_shr (tsc_now, data->cyc2ns_mul, data->cyc2ns_shift);cyc2ns_write_end (cpu, data);
Quoted 12 lines · Adrian Hunter
Quoted 11 lines · Adrian Hunter
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because
CYC2NS_SCALE_FACTORis 10, the accuracy is +/- 1/2048The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33)
Quoted 3 lines
Quoted 1 line
Signed-off-by Adrian Hunter <[email protected]>Is this OK?
Yes I think so, sorry I got backlogged with preparation for Seattle.
Thomas, if you see this, could you merge with my ACK on, otherwise I'll queue it the normal route once I'm back.
Quoted 14 lines · Adrian Hunter
arch/x86/kernel/tsc.c +2
-1@@ -239,6 +239,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)unsigned long long tsc_now, ns_now;struct cyc2ns_data *data;unsigned long flags; +u64 mult; +u32 shft =32 ;local_irq_save (flags);sched_clock_idle_sleep_event ();@@ -256,12 +258,17 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul =
arch/x86/kernel/tsc.c static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) · line 267
Quoted 10 lines
arch/x86/kernel/tsc.c +7
-3-DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + mult = (u64 )NSEC_PER_MSEC <<32 ; + mult += cpu_khz /2 ; +do_div (mult, cpu_khz); +while (mult >U32_MAX ) { + mult >>=1 ; + shft -=1 ; + }
This is an open coded variant of clocks_calc_mult_shift(). Can we please use that one?
Thanks,
tglx
Quoted 20 lines · Thomas Gleixner
Quoted 19 lines · Adrian Hunter
arch/x86/kernel/tsc.c +4
-4@@ -239,6 +239,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)unsigned long long tsc_now, ns_now;struct cyc2ns_data *data;unsigned long flags; +u64 mult; +u32 shft =32 ;local_irq_save (flags);sched_clock_idle_sleep_event ();@@ -256,12 +258,17 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + mult = (u64 )NSEC_PER_MSEC <<32 ; + mult += cpu_khz /2 ;
Quoted 8 lines
arch/x86/kernel/tsc.c static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) · line 267
Quoted 5 lines
arch/x86/kernel/tsc.c +5
-0+do_div (mult, cpu_khz); +while (mult >U32_MAX ) { + mult >>=1 ; + shft -=1 ; + }This is an open coded variant of
clocks_calc_mult_shift(). Can we please use that one?
Sure. clocks_calc_mult_shift() does a division on each shift which is a bit slower (avoids a 1-bit rounding error but that will never be more than 1 in 2^32 in this case), and the 'maxsec' functionality is not needed because mul_u64_u32_shr() is used to avoid 64-bit overflow.
I will send V3.
PATCH v3 1/1 perf: x86: Improve accuracy of perf/sched clock
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because CYC2NS_SCALE_FACTOR is 10, the accuracy is +/- 1/2048
The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33). That is achieved by using the 'clocks_calc_mult_shift()' function.
---
Changes in V3:
Use clocks_calc_mult_shift() instead of open-coded calculation
Changes in V2:
Remove definition of CYC2NS_SCALE_FACTOR Amend comments
arch/x86/kernel/tsc.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)
arch/x86/kernel/tsc.c +11 -13
@@ -167,21 +167,20 @@ static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data)* ns = cycles * cyc2ns_scale / SC * * And since SC is a constant power of two, we can convert the div -* into a shift. +* into a shift. The larger SC is, the more accurate the conversion, but +* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication +* (64-bit result) can be used. * -* We can use khz divisor instead of mhz to keep a better precision, since -* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. +* We can use khz divisor instead of mhz to keep a better precision. * ([email protected]) * * [email protected] "math is hard, lets go shopping!" */ -#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static void cyc2ns_data_init (struct cyc2ns_data *data) { data->cyc2ns_mul =0 ; - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + data->cyc2ns_shift =0 ; data->cyc2ns_offset =0 ; data->__count =0 ; }
@@ -215,14 +214,14 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)if (likely (data == tail)) { ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift); }else { data->__count++;barrier (); ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift);barrier ();
@@ -256,12 +255,11 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; +clocks_calc_mult_shift (&data->cyc2ns_mul, &data->cyc2ns_shift, cpu_khz, +NSEC_PER_MSEC ,0 ); + data->cyc2ns_offset = ns_now - -mul_u64_u32_shr (tsc_now, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); +mul_u64_u32_shr (tsc_now, data->cyc2ns_mul, data->cyc2ns_shift);cyc2ns_write_end (cpu, data);
Signature · 1 line
1.9.1
Quoted 5 lines · Adrian Hunter
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because
CYC2NS_SCALE_FACTORis 10, the accuracy is +/- 1/2048
Quoted 8 lines
The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33). That is achieved by using the '
clocks_calc_mult_shift()' function.Signed-off-by Adrian Hunter <[email protected]>
Ping?
Quoted 83 lines
---
Changes in V3:
Use
clocks_calc_mult_shift()instead of open-coded calculationChanges in V2:
Remove definition of
CYC2NS_SCALE_FACTORAmend commentsarch/x86/kernel/tsc.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)arch/x86/kernel/tsc.c +11
-13@@ -167,21 +167,20 @@ static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data)* ns = cycles * cyc2ns_scale / SC * * And since SC is a constant power of two, we can convert the div -* into a shift. +* into a shift. The larger SC is, the more accurate the conversion, but +* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication +* (64-bit result) can be used. * -* We can use khz divisor instead of mhz to keep a better precision, since -* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. +* We can use khz divisor instead of mhz to keep a better precision. * ([email protected]) * * [email protected] "math is hard, lets go shopping!" */ -#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static void cyc2ns_data_init (struct cyc2ns_data *data) { data->cyc2ns_mul =0 ; - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + data->cyc2ns_shift =0 ; data->cyc2ns_offset =0 ; data->__count =0 ; }@@ -215,14 +214,14 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)if (likely (data == tail)) { ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift); }else { data->__count++;barrier (); ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift);barrier ();@@ -256,12 +255,11 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; +clocks_calc_mult_shift (&data->cyc2ns_mul, &data->cyc2ns_shift, cpu_khz, +NSEC_PER_MSEC ,0 ); + data->cyc2ns_offset = ns_now - -mul_u64_u32_shr (tsc_now, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); +mul_u64_u32_shr (tsc_now, data->cyc2ns_mul, data->cyc2ns_shift);cyc2ns_write_end (cpu, data);
Peter Zijlstra
AppliedQuoted 13 lines · Adrian Hunter
Quoted 12 lines · Adrian Hunter
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because
CYC2NS_SCALE_FACTORis 10, the accuracy is +/- 1/2048The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33). That is achieved by using the '
clocks_calc_mult_shift()' function.
Quoted 3 lines
Quoted 1 line
Signed-off-by Adrian Hunter <[email protected]>Ping?
I've queued it. Thanks
tip-bot for Adrian Hunter
merge bot[tip:perf/core] perf/x86: Improve accuracy of perf/sched clock
In reply to Adrian Hunterperf/x86: Improve accuracy of perf/sched clock
Commit-ID: b20112edeadf0b8a1416de061caa4beb11539902 Gitweb: http://git.kernel.org/tip/b20112edeadf0b8a1416de061caa4beb11539902 Author: Adrian Hunter <[email protected]> AuthorDate: Fri, 21 Aug 2015 12:05:18 +0300 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 13 Sep 2015 11:27:20 +0200
perf/x86: Improve accuracy of perf/sched clock
When TSC is stable perf/sched clock is based on it. However the conversion from cycles to nanoseconds is not as accurate as it could be. Because CYC2NS_SCALE_FACTOR is 10, the accuracy is +/- 1/2048
The change is to calculate the maximum shift that results in a multiplier that is still a 32-bit number. For example all frequencies over 1 GHz will have a shift of 32, making the accuracy of the conversion +/- 1/(2^33). That is achieved by using the 'clocks_calc_mult_shift()' function.
--- arch/x86/kernel/tsc.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)
arch/x86/kernel/tsc.c +11 -13
@@ -167,21 +167,20 @@ static void cyc2ns_write_end(int cpu, struct cyc2ns_data *data)* ns = cycles * cyc2ns_scale / SC * * And since SC is a constant power of two, we can convert the div -* into a shift. +* into a shift. The larger SC is, the more accurate the conversion, but +* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication +* (64-bit result) can be used. * -* We can use khz divisor instead of mhz to keep a better precision, since -* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. +* We can use khz divisor instead of mhz to keep a better precision. * ([email protected]) * * [email protected] "math is hard, lets go shopping!" */ -#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static void cyc2ns_data_init (struct cyc2ns_data *data) { data->cyc2ns_mul =0 ; - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; + data->cyc2ns_shift =0 ; data->cyc2ns_offset =0 ; data->__count =0 ; }
@@ -215,14 +214,14 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)if (likely (data == tail)) { ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift); }else { data->__count++;barrier (); ns = data->cyc2ns_offset; - ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); + ns +=mul_u64_u32_shr (cyc, data->cyc2ns_mul, data->cyc2ns_shift);barrier ();
@@ -256,12 +255,11 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)* time function is continuous; see the comment near struct * cyc2ns_data. */ - data->cyc2ns_mul = -DIV_ROUND_CLOSEST (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR , - cpu_khz); - data->cyc2ns_shift =CYC2NS_SCALE_FACTOR ; +clocks_calc_mult_shift (&data->cyc2ns_mul, &data->cyc2ns_shift, cpu_khz, +NSEC_PER_MSEC ,0 ); + data->cyc2ns_offset = ns_now - -mul_u64_u32_shr (tsc_now, data->cyc2ns_mul,CYC2NS_SCALE_FACTOR ); +mul_u64_u32_shr (tsc_now, data->cyc2ns_mul, data->cyc2ns_shift);cyc2ns_write_end (cpu, data);