lib/vdso: Bugfix and consolidation
Subsystems (17)
cover lib/vdso: Bugfix and consolidation
The recent discussions about changes in the VDSO to support PPC triggered this consolidation series which in turn unearthed a subtle bug affecting ARM.
The bug affects only ARM 32bit systems which lack a VDSO capable timer. The absence of the ARM architected timer causes the VDSO update code to terminate early. As a result the parts of the VDSO data which is usable without a VDSO capable clocksource are not updated either, but the VDSO code uses them, i.e. it uses stale data and will serve that forever. The fix is to make only the parts of the update conditional which are related to te high resolution interfaces.
The rest of the series is addressing the following points:
- Allow architectures to compile out the high resolution parts of the VDSO library when they know at compile time that no VDSO capable clocksource will be available.
- Move the storage of the VDSO clockmode into generic code and remove all the redundant copies of the storage and the handling of it from the architectures.
This allows to remove the check for a valid clocksource at runtime from the clock readout function and make it generic by checking for a clock mode != NONE in the generic code. This generates better code on PPC and does not affect x86/arm performance.
Thanks,
tglx
8<----------
arch/arm/Kconfig | 1 arch/arm/include/asm/clocksource.h | 5 - arch/arm/include/asm/vdso/gettimeofday.h | 6 + arch/arm/include/asm/vdso/vsyscall.h | 35 --------- arch/arm64/Kconfig | 1 arch/arm64/include/asm/clocksource.h | 5 - arch/arm64/include/asm/vdso/vsyscall.h | 9 -- arch/mips/Kconfig | 1 arch/mips/include/asm/clocksource.h | 18 ---- arch/mips/include/asm/vdso/vsyscall.h | 9 -- arch/mips/kernel/csrc-r4k.c | 2 arch/x86/Kconfig | 1 arch/x86/entry/vdso/vma.c | 8 +- arch/x86/include/asm/clocksource.h | 23 ++++-- arch/x86/include/asm/mshyperv.h | 4 - arch/x86/include/asm/vdso/gettimeofday.h | 6 - arch/x86/include/asm/vdso/vsyscall.h | 15 ---- arch/x86/include/asm/vgtod.h | 6 - arch/x86/kernel/kvmclock.c | 9 ++ arch/x86/kernel/pvclock.c | 2 arch/x86/kernel/time.c | 12 --- arch/x86/kernel/tsc.c | 32 +++++--- arch/x86/kvm/trace.h | 4 - arch/x86/kvm/x86.c | 22 +++--- arch/x86/xen/time.c | 36 ++++++--- b/arch/mips/include/asm/vdso/gettimeofday.h | 29 +++---- b/include/asm-generic/vdso/vsyscall.h | 14 --- drivers/clocksource/arm_arch_timer.c | 8 +- drivers/clocksource/hyperv_timer.c | 7 + drivers/clocksource/mips-gic-timer.c | 8 +- include/linux/clocksource.h | 102 ++++++++++++++++------------ include/vdso/datapage.h | 2 kernel/time/clocksource.c | 9 ++ kernel/time/namespace.c | 7 + kernel/time/vsyscall.c | 43 +++++------ lib/vdso/gettimeofday.c | 43 +++++++---- 36 files changed, 259 insertions(+), 285 deletions(-)
PATCH 1/15 lib/vdso: Make __arch_update_vdso_data() logic understandable
The function name suggests that this is a boolean checking whether the architecture asks for an update of the VDSO data, but it works the other way round. To spare further confusion invert the logic.
--- arch/arm/include/asm/vdso/vsyscall.h | 4 ++-- include/asm-generic/vdso/vsyscall.h | 4 ++-- kernel/time/vsyscall.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-)
arch/arm/include/asm/vdso/vsyscall.h +2 -2
@@ -34,9 +34,9 @@ struct vdso_data *__arm_get_k_vdso_data(#define __arch_get_k_vdso_data __arm_get_k_vdso_datastatic __always_inline -int __arm_update_vdso_data(void) +bool __arm_update_vdso_data(void) { -return !cntvct_ok; +return cntvct_ok; }#define __arch_update_vdso_data __arm_update_vdso_data
include/asm-generic/vdso/vsyscall.h +2 -2
@@ -12,9 +12,9 @@ static __always_inline struct vdso_data#endif /* __arch_get_k_vdso_data */ #ifndef __arch_update_vdso_data -static __always_inline int __arch_update_vdso_data(void) +static __always_inline bool __arch_update_vdso_data(void) { -return 0 ; +return true; }#endif /* __arch_update_vdso_data */
kernel/time/vsyscall.c +1 -1
@@ -84,7 +84,7 @@ void update_vsyscall(struct timekeeper *struct vdso_timestamp *vdso_ts;u64 nsec; -if (__arch_update_vdso_data ()) { +if (!__arch_update_vdso_data ()) {/* * Some architectures might want to skip the update of the * data page.
[tip: timers/urgent] lib/vdso: Make __arch_update_vdso_data() logic understandable
The following commit has been merged into the timers/urgent branch of tip:
The following commit has been merged into the timers/urgent branch of tip:
Commit-ID: 9a6b55ac4a44060bcb782baf002859b2a2c63267 Gitweb: https://git.kernel.org/tip/9a6b55ac4a44060bcb782baf002859b2a2c63267 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 14 Jan 2020 19:52:38 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Fri, 17 Jan 2020 15:53:50 +01:00
lib/vdso: Make __arch_update_vdso_data() logic understandable
The function name suggests that this is a boolean checking whether the architecture asks for an update of the VDSO data, but it works the other way round. To spare further confusion invert the logic.
--- arch/arm/include/asm/vdso/vsyscall.h | 4 ++-- include/asm-generic/vdso/vsyscall.h | 4 ++-- kernel/time/vsyscall.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-)
arch/arm/include/asm/vdso/vsyscall.h +2 -2
@@ -34,9 +34,9 @@ struct vdso_data *__arm_get_k_vdso_data(void)#define __arch_get_k_vdso_data __arm_get_k_vdso_datastatic __always_inline -int __arm_update_vdso_data(void) +bool __arm_update_vdso_data(void) { -return !cntvct_ok; +return cntvct_ok; }#define __arch_update_vdso_data __arm_update_vdso_data
include/asm-generic/vdso/vsyscall.h +2 -2
@@ -12,9 +12,9 @@ static __always_inline struct vdso_data *__arch_get_k_vdso_data(void)#endif /* __arch_get_k_vdso_data */ #ifndef __arch_update_vdso_data -static __always_inline int __arch_update_vdso_data(void) +static __always_inline bool __arch_update_vdso_data(void) { -return 0 ; +return true; }#endif /* __arch_update_vdso_data */
kernel/time/vsyscall.c +1 -1
@@ -84,7 +84,7 @@ void update_vsyscall(struct timekeeper *tk)struct vdso_timestamp *vdso_ts;u64 nsec; -if (__arch_update_vdso_data ()) { +if (!__arch_update_vdso_data ()) {/* * Some architectures might want to skip the update of the * data page.
PATCH 2/15 lib/vdso: Update coarse timekeeper unconditionally
In reply to Thomas GleixnerThe low resolution parts of the VDSO, i.e.:
clock_gettime(CLOCK_*_COARSE)
clock_getres()
time()
can be used even if there is no VDSO capable clocksource. But if an architecture opts out of the VDSO data update then this information becomes stale. This affects ARM when there is no architected timer available. The lack of update causes userspace to use stale data forever.
Make the update of the low resolution parts unconditional and only prevent the update of the high resolution parts if the architecture requests it.
--- kernel/time/vsyscall.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-)
kernel/time/vsyscall.c +17 -20
@@ -28,11 +28,6 @@ static inline void update_vdso_data(stru vdata[CS_RAW ].mult = tk->tkr_raw.mult; vdata[CS_RAW ].shift = tk->tkr_raw.shift; -/* CLOCK_REALTIME */ - vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ]; - vdso_ts->sec = tk->xtime_sec; - vdso_ts->nsec = tk->tkr_mono.xtime_nsec; -/* CLOCK_MONOTONIC */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_MONOTONIC ]; vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
@@ -70,12 +65,6 @@ static inline void update_vdso_data(stru vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_TAI ]; vdso_ts->sec = tk->xtime_sec + (s64 )tk->tai_offset; vdso_ts->nsec = tk->tkr_mono.xtime_nsec; - -/* -* Read without the seqlock held by clock_getres(). -* Note: No need to have a second copy. -*/ -WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution); }void update_vsyscall (struct timekeeper *tk)
@@ -84,20 +73,17 @@ void update_vsyscall(struct timekeeper *struct vdso_timestamp *vdso_ts;u64 nsec; -if (!__arch_update_vdso_data ()) { -/* -* Some architectures might want to skip the update of the -* data page. -*/ -return ; - } -/* copy vsyscall data */ vdso_write_begin (vdata); vdata[CS_HRES_COARSE ].clock_mode =__arch_get_clock_mode (tk); vdata[CS_RAW ].clock_mode =__arch_get_clock_mode (tk); +/* CLOCK_REALTIME also required for time() */ + vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ]; + vdso_ts->sec = tk->xtime_sec; + vdso_ts->nsec = tk->tkr_mono.xtime_nsec; +/* CLOCK_REALTIME_COARSE */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME_COARSE ]; vdso_ts->sec = tk->xtime_sec;
@@ -110,7 +96,18 @@ void update_vsyscall(struct timekeeper * nsec = nsec + tk->wall_to_monotonic.tv_nsec; vdso_ts->sec +=__iter_div_u64_rem (nsec,NSEC_PER_SEC , &vdso_ts->nsec); -update_vdso_data (vdata, tk); +/* +* Read without the seqlock held by clock_getres(). +* Note: No need to have a second copy. +*/ +WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution); + +/* +* Architectures can opt out of updating the high resolution part +* of the VDSO. +*/ +if (__arch_update_vdso_data ()) +update_vdso_data (vdata, tk);__arch_update_vsyscall (vdata, tk);
[tip: timers/urgent] lib/vdso: Update coarse timekeeper unconditionally
The following commit has been merged into the timers/urgent branch of tip:
The following commit has been merged into the timers/urgent branch of tip:
Commit-ID: 9f24c540f7f8eb3a981528da9a9a636a5bdf5987 Gitweb: https://git.kernel.org/tip/9f24c540f7f8eb3a981528da9a9a636a5bdf5987 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 14 Jan 2020 19:52:39 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Fri, 17 Jan 2020 15:53:50 +01:00
lib/vdso: Update coarse timekeeper unconditionally
The low resolution parts of the VDSO, i.e.:
clock_gettime(CLOCK_*_COARSE), clock_getres(), time()
can be used even if there is no VDSO capable clocksource.
But if an architecture opts out of the VDSO data update then this information becomes stale. This affects ARM when there is no architected timer available. The lack of update causes userspace to use stale data forever.
Make the update of the low resolution parts unconditional and only skip the update of the high resolution parts if the architecture requests it.
--- kernel/time/vsyscall.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-)
kernel/time/vsyscall.c +17 -20
@@ -28,11 +28,6 @@ static inline void update_vdso_data(struct vdso_data *vdata, vdata[CS_RAW ].mult = tk->tkr_raw.mult; vdata[CS_RAW ].shift = tk->tkr_raw.shift; -/* CLOCK_REALTIME */ - vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ]; - vdso_ts->sec = tk->xtime_sec; - vdso_ts->nsec = tk->tkr_mono.xtime_nsec; -/* CLOCK_MONOTONIC */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_MONOTONIC ]; vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
@@ -70,12 +65,6 @@ static inline void update_vdso_data(struct vdso_data *vdata, vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_TAI ]; vdso_ts->sec = tk->xtime_sec + (s64 )tk->tai_offset; vdso_ts->nsec = tk->tkr_mono.xtime_nsec; - -/* -* Read without the seqlock held by clock_getres(). -* Note: No need to have a second copy. -*/ -WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution); }void update_vsyscall (struct timekeeper *tk)
@@ -84,20 +73,17 @@ void update_vsyscall(struct timekeeper *tk)struct vdso_timestamp *vdso_ts;u64 nsec; -if (!__arch_update_vdso_data ()) { -/* -* Some architectures might want to skip the update of the -* data page. -*/ -return ; - } -/* copy vsyscall data */ vdso_write_begin (vdata); vdata[CS_HRES_COARSE ].clock_mode =__arch_get_clock_mode (tk); vdata[CS_RAW ].clock_mode =__arch_get_clock_mode (tk); +/* CLOCK_REALTIME also required for time() */ + vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ]; + vdso_ts->sec = tk->xtime_sec; + vdso_ts->nsec = tk->tkr_mono.xtime_nsec; +/* CLOCK_REALTIME_COARSE */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME_COARSE ]; vdso_ts->sec = tk->xtime_sec;
@@ -110,7 +96,18 @@ void update_vsyscall(struct timekeeper *tk) nsec = nsec + tk->wall_to_monotonic.tv_nsec; vdso_ts->sec +=__iter_div_u64_rem (nsec,NSEC_PER_SEC , &vdso_ts->nsec); -update_vdso_data (vdata, tk); +/* +* Read without the seqlock held by clock_getres(). +* Note: No need to have a second copy. +*/ +WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution); + +/* +* Architectures can opt out of updating the high resolution part +* of the VDSO. +*/ +if (__arch_update_vdso_data ()) +update_vdso_data (vdata, tk);__arch_update_vsyscall (vdata, tk);
PATCH 3/15 ARM: vdso: Remove unused function
In reply to Thomas GleixnerThe function is nowhere used. Aside of that this check should only cover the high resolution parts of the VDSO which require a VDSO capable clocksource and not the complete functionality as the name suggests. Will be replaced with something more useful.
--- arch/arm/include/asm/vdso/vsyscall.h | 7 ------- 1 file changed, 7 deletions(-)
arch/arm/include/asm/vdso/vsyscall.h +0 -7
@@ -50,13 +50,6 @@ int __arm_get_clock_mode(struct timekeep#define __arch_get_clock_mode __arm_get_clock_modestatic __always_inline -int __arm_use_vsyscall(struct vdso_data *vdata) -{ -return vdata[CS_HRES_COARSE ].clock_mode; -} -#define __arch_use_vsyscall __arm_use_vsyscall - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) {flush_dcache_page (virt_to_page (vdata));
PATCH 4/15 lib/vdso: Allow the high resolution parts to be compiled out
In reply to Thomas GleixnerIf the architecture knows at compile time that there is no VDSO capable clocksource supported it makes sense to optimize the guts of the high resolution parts of the VDSO out at build time. Add a helper function to check whether the VDSO should be high resolution capable and provide a stub which can be overridden by an architecture.
--- lib/vdso/gettimeofday.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
lib/vdso/gettimeofday.c +11 -0
@@ -38,6 +38,13 @@ u64 vdso_calc_delta(u64 cycles, u64 last }#endif +#ifndef __arch_vdso_hres_capable +static inline bool __arch_vdso_hres_capable (void ) +{ +return true; +} +#endif +#ifdef CONFIG_TIME_NS static int do_hres_timens (const struct vdso_data *vdns,clockid_t clk,struct __kernel_timespec *ts)
@@ -101,6 +108,10 @@ static __always_inline int do_hres(constu64 cycles, last, sec, ns;u32 seq; +/* Allows to compile the high resilution parts out */ +if (!__arch_vdso_hres_capable ()) +return -1 ; + do { /* * Open coded to handle VCLOCK_TIMENS. Time namespace
PATCH 5/15 ARM: vdso: Compile high resolution parts conditionally
In reply to Thomas GleixnerIf the architected timer is disabled in the kernel configuration then let the core VDSO code drop the high resolution parts at compile time.
--- arch/arm/include/asm/vdso/gettimeofday.h | 6 ++++++ 1 file changed, 6 insertions(+)
arch/arm/include/asm/vdso/gettimeofday.h +6 -0
@@ -106,6 +106,12 @@ static __always_inline int clock_getres3return ret; } +static inline bool arm_vdso_hres_capable (void ) +{ +return IS_ENABLED (CONFIG_ARM_ARCH_TIMER ); +} +#define __arch_vdso_hres_capable arm_vdso_hres_capable + static __always_inline u64 __arch_get_hw_counter(int clock_mode) {#ifdef CONFIG_ARM_ARCH_TIMER
PATCH 6/15 MIPS: vdso: Compile high resolution parts conditionally
In reply to Thomas GleixnerIf neither the R4K nor the GIC timer is enabled in the kernel configuration then let the core VDSO code drop the high resolution parts at compile time.
--- arch/mips/include/asm/vdso/gettimeofday.h | 7 +++++++ 1 file changed, 7 insertions(+)
arch/mips/include/asm/vdso/gettimeofday.h +7 -0
@@ -199,6 +199,13 @@ static __always_inline u64 __arch_get_hwreturn cycle_now; } +static inline bool mips_vdso_hres_capable (void ) +{ +return IS_ENABLED (CONFIG_CSRC_R4K ) || +IS_ENABLED (CONFIG_CLKSRC_MIPS_GIC ); +} +#define __arch_vdso_hres_capable mips_vdso_hres_capable + static __always_inline const struct vdso_data *__arch_get_vdso_data(void) {return get_vdso_data ();
PATCH 7/15 clocksource: Cleanup struct clocksource and documentation
In reply to Thomas GleixnerReformat the struct definition, add missing member documentation. No functional change.
--- include/linux/clocksource.h | 87 +++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 40 deletions(-)
include/linux/clocksource.h +47 -40
@@ -32,9 +32,19 @@ struct module;* Provides mostly state-free accessors to the underlying hardware. * This is the structure used for system time. * -* @name: ptr to clocksource name -* @list: list head for registration -* @rating: rating value for selection (higher is better) +* @read: Returns a cycle value, passes clocksource as argument +* @mask: Bitmask for two's complement +* subtraction of non 64 bit counters +* @mult: Cycle to nanosecond multiplier +* @shift: Cycle to nanosecond divisor (power of two) +* @max_idle_ns: Maximum idle time permitted by the clocksource (nsecs) +* @maxadj: Maximum adjustment value to mult (~11%) +* @archdata: Optional arch-specific data +* @max_cycles: Maximum safe cycle value which won't overflow on +* multiplication +* @name: Pointer to clocksource name +* @list: List head for registration (internal) +* @rating: Rating value for selection (higher is better) * To avoid rating inflation the following * list should give you a guide as to how * to assign your clocksource a rating
@@ -49,27 +59,23 @@ struct module;* 400-499: Perfect * The ideal clocksource. A must-use where * available. -* @read: returns a cycle value, passes clocksource as argument -* @enable: optional function to enable the clocksource -* @disable: optional function to disable the clocksource -* @mask: bitmask for two's complement -* subtraction of non 64 bit counters -* @mult: cycle to nanosecond multiplier -* @shift: cycle to nanosecond divisor (power of two) -* @max_idle_ns: max idle time permitted by the clocksource (nsecs) -* @maxadj: maximum adjustment value to mult (~11%) -* @max_cycles: maximum safe cycle value which won't overflow on multiplication -* @flags: flags describing special properties -* @archdata: arch-specific data -* @suspend: suspend function for the clocksource, if necessary -* @resume: resume function for the clocksource, if necessary +* @flags: Flags describing special properties +* @enable: Optional function to enable the clocksource +* @disable: Optional function to disable the clocksource +* @suspend: Optional suspend function for the clocksource +* @resume: Optional resume function for the clocksource * @mark_unstable: Optional function to inform the clocksource driver that * the watchdog marked the clocksource unstable -* @owner: module reference, must be set by clocksource in modules +* @tick_stable: Optional function called periodically from the watchdog +* code to provide stable syncrhonization points +* @wd_list: List head to enqueue into the watchdog list (internal) +* @cs_last: Last clocksource value for clocksource watchdog +* @wd_last: Last watchdog value corresponding to @cs_last +* @owner: Module reference, must be set by clocksource in modules * * Note: This struct is not used in hotpathes of the timekeeping code * because the timekeeper caches the hot path fields in its own data -* structure, so no line cache alignment is required, +* structure, so no cache line alignment is required, * * The pointer to the clocksource itself is handed to the read * callback. If you need extra information there you can wrap struct
@@ -78,35 +84,36 @@ struct module;* structure. */ struct clocksource { -u64 (*read)(struct clocksource *cs); -u64 mask; -u32 mult; -u32 shift; -u64 max_idle_ns; -u32 maxadj; +u64 (*read)(struct clocksource *cs); +u64 mask; +u32 mult; +u32 shift; +u64 max_idle_ns; +u32 maxadj;#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA struct arch_clocksource_data archdata;#endif -u64 max_cycles; -const char *name; -struct list_head list; -int rating; -int (*enable)(struct clocksource *cs); -void (*disable)(struct clocksource *cs); -unsigned long flags; -void (*suspend)(struct clocksource *cs); -void (*resume)(struct clocksource *cs); -void (*mark_unstable)(struct clocksource *cs); -void (*tick_stable)(struct clocksource *cs); +u64 max_cycles; +const char *name; +struct list_head list; +int rating; +unsigned long flags; + +int (*enable)(struct clocksource *cs); +void (*disable)(struct clocksource *cs); +void (*suspend)(struct clocksource *cs); +void (*resume)(struct clocksource *cs); +void (*mark_unstable)(struct clocksource *cs); +void (*tick_stable)(struct clocksource *cs);/* private: */ #ifdef CONFIG_CLOCKSOURCE_WATCHDOG /* Watchdog related data, used by the framework */ -struct list_head wd_list; -u64 cs_last; -u64 wd_last; +struct list_head wd_list; +u64 cs_last; +u64 wd_last;#endif -struct module *owner; +struct module *owner; };/*
PATCH 8/15 x86/vdso: Move VDSO clocksource state tracking to callback
In reply to Thomas GleixnerAll architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code.
X86 abuses the function which retrieves the architecture specific clock mode storage to mark the clocksource as used in the VDSO. That's silly because this is invoked on every tick when the VDSO data is updated.
Move this functionality to the clocksource::enable() callback so it gets invoked once when the clocksource is installed. This allows to make the clock mode storage generic.
--- arch/x86/entry/vdso/vma.c | 4 ++++ arch/x86/include/asm/clocksource.h | 12 ++++++++++++ arch/x86/include/asm/mshyperv.h | 2 ++ arch/x86/include/asm/vdso/vsyscall.h | 10 +--------- arch/x86/include/asm/vgtod.h | 6 ------ arch/x86/kernel/kvmclock.c | 7 +++++++ arch/x86/kernel/tsc.c | 32 ++++++++++++++++++++------------ arch/x86/xen/time.c | 17 ++++++++++++----- drivers/clocksource/hyperv_timer.c | 7 +++++++ 9 files changed, 65 insertions(+), 32 deletions(-)
arch/x86/entry/vdso/vma.c +4 -0
@@ -38,6 +38,8 @@ struct vdso_data *arch_get_vdso_data(voi }#undef EMIT_VVAR +unsigned int vclocks_used__read_mostly ; +#if defined(CONFIG_X86_64 )unsigned int __read_mostly vdso64_enabled =1 ;#endif
@@ -445,6 +447,8 @@ static __init int vdso_setup(char *s)static int __init init_vdso (void ) { +BUILD_BUG_ON (VCLOCK_MAX >=32 ); +init_vdso_image (&vdso_image_64);#ifdef CONFIG_X86_X32_ABI
arch/x86/include/asm/clocksource.h +12 -0
@@ -14,4 +14,16 @@ struct arch_clocksource_data { int vclock_mode; }; +extern unsigned int vclocks_used; + +static inline bool vclock_was_used (int vclock) +{ +return READ_ONCE (vclocks_used) & (1U << vclock); +} + +static inline void vclocks_set_used (unsigned int which) +{ +WRITE_ONCE (vclocks_used,READ_ONCE (vclocks_used) | (1 << which)); +} +#endif /* _ASM_X86_CLOCKSOURCE_H */
arch/x86/include/asm/mshyperv.h +2 -0
@@ -47,6 +47,8 @@ typedef int (*hyperv_fill_flush_list_fun wrmsrl(HV_X64_MSR_REFERENCE_TSC, val)#define hv_set_clocksource_vdso (val) \ ((val).archdata.vclock_mode = VCLOCK_HVCLOCK) +#define hv_enable_vdso_clocksource () \ + vclocks_set_used(VCLOCK_HVCLOCK);#define hv_get_raw_timer () rdtsc_ordered()void hyperv_callback_vector (void );
arch/x86/include/asm/vdso/vsyscall.h +1 -9
@@ -10,8 +10,6 @@#include <asm/vgtod.h> #include <asm/vvar.h> -int vclocks_used__read_mostly ; - DEFINE_VVAR(struct vdso_data, _vdso_data);/* * Update the vDSO data page to keep in sync with kernel timekeeping.
@@ -26,13 +24,7 @@ struct vdso_data *__x86_get_k_vdso_data(static __always_inline int __x86_get_clock_mode(struct timekeeper *tk) { -int vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; - -/* Mark the new vclock used. */ -BUILD_BUG_ON (VCLOCK_MAX >=32 ); -WRITE_ONCE (vclocks_used,READ_ONCE (vclocks_used) | (1 << vclock_mode)); - -return vclock_mode; +return tk->tkr_mono.clock->archdata.vclock_mode; }#define __arch_get_clock_mode __x86_get_clock_mode
arch/x86/include/asm/vgtod.h +0 -6
@@ -15,10 +15,4 @@ typedef u64 gtod_long_t;typedef unsigned long gtod_long_t ;#endif -extern int vclocks_used; -static inline bool vclock_was_used (int vclock) -{ -return READ_ONCE (vclocks_used) & (1 << vclock); -} -#endif /* _ASM_X86_VGTOD_H */
arch/x86/kernel/kvmclock.c +7 -0
@@ -159,12 +159,19 @@ bool kvm_check_and_clear_guest_paused(voreturn ret; } +static int kvm_cs_enable (struct clocksource *cs) +{ +vclocks_set_used (VCLOCK_PVCLOCK ); +return 0 ; +} +struct clocksource kvm_clock = { .name ="kvm-clock" , .read = kvm_clock_get_cycles, .rating =400 , .mask =CLOCKSOURCE_MASK (64 ), .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .enable = kvm_cs_enable, };EXPORT_SYMBOL_GPL (kvm_clock);
arch/x86/kernel/tsc.c +20 -12
@@ -1108,17 +1108,24 @@ static void tsc_cs_tick_stable(struct clsched_clock_tick_stable (); } +static int tsc_cs_enable (struct clocksource *cs) +{ +vclocks_set_used (VCLOCK_TSC ); +return 0 ; +} +/* * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc() */ static struct clocksource clocksource_tsc_early = { - .name ="tsc-early" , - .rating =299 , - .read = read_tsc, - .mask =CLOCKSOURCE_MASK (64 ), - .flags =CLOCK_SOURCE_IS_CONTINUOUS | + .name ="tsc-early" , + .rating =299 , + .read = read_tsc, + .mask =CLOCKSOURCE_MASK (64 ), + .flags =CLOCK_SOURCE_IS_CONTINUOUS |CLOCK_SOURCE_MUST_VERIFY , - .archdata = { .vclock_mode =VCLOCK_TSC }, + .archdata = { .vclock_mode =VCLOCK_TSC }, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable,
@@ -1131,14 +1138,15 @@ static struct clocksource clocksource_ts* been found good. */ static struct clocksource clocksource_tsc = { - .name ="tsc" , - .rating =300 , - .read = read_tsc, - .mask =CLOCKSOURCE_MASK (64 ), - .flags =CLOCK_SOURCE_IS_CONTINUOUS | + .name ="tsc" , + .rating =300 , + .read = read_tsc, + .mask =CLOCKSOURCE_MASK (64 ), + .flags =CLOCK_SOURCE_IS_CONTINUOUS |CLOCK_SOURCE_VALID_FOR_HRES |CLOCK_SOURCE_MUST_VERIFY , - .archdata = { .vclock_mode =VCLOCK_TSC }, + .archdata = { .vclock_mode =VCLOCK_TSC }, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable,
arch/x86/xen/time.c +12 -5
@@ -145,12 +145,19 @@ static struct notifier_block xen_pvclock .notifier_call = xen_pvclock_gtod_notify, }; +static int xen_cs_enable (struct clocksource *cs) +{ +vclocks_set_used (VCLOCK_PVCLOCK ); +return 0 ; +} +static struct clocksource xen_clocksource__read_mostly = { - .name ="xen" , - .rating =400 , - .read = xen_clocksource_get_cycles, - .mask = ~0 , - .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .name ="xen" , + .rating =400 , + .read = xen_clocksource_get_cycles, + .mask =CLOCKSOURCE_MASK (64 ), + .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .enable = xen_cs_enable, };/*
drivers/clocksource/hyperv_timer.c +7 -0
@@ -330,12 +330,19 @@ static u64 read_hv_sched_clock_tsc(void)return read_hv_clock_tsc (NULL ) - hv_sched_clock_offset; } +static int hv_cs_enable (struct clocksource *cs) +{ +hv_enable_vdso_clocksource (); +return 0 ; +} +static struct clocksource hyperv_cs_tsc = { .name ="hyperv_clocksource_tsc_page" , .rating =400 , .read = read_hv_clock_tsc, .mask =CLOCKSOURCE_MASK (64 ), .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .enable = hv_cs_enable, }; static u64 notrace read_hv_clock_msr(struct clocksource *arg)
Re: [patch 08/15] x86/vdso: Move VDSO clocksource state tracking to callback
Quoted 7 lines · Thomas Gleixner
All architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code.
X86 abuses the function which retrieves the architecture specific clock mode storage to mark the clocksource as used in the VDSO. That's silly because this is invoked on every tick when the VDSO data is updated.
Quoted 5 lines
Move this functionality to the
clocksource::enable()callback so it gets invoked once when the clocksource is installed. This allows to make the clock mode storage generic.Signed-off-by Thomas Gleixner <[email protected]>
Juergen
RE: [patch 08/15] x86/vdso: Move VDSO clocksource state tracking to callback
In reply to Thomas GleixnerFrom: Thomas Gleixner <[email protected]> Sent: Tuesday, January 14, 2020 10:53 AM
Quoted 8 lines
All architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code.
X86 abuses the function which retrieves the architecture specific clock mode storage to mark the clocksource as used in the VDSO. That's silly because this is invoked on every tick when the VDSO data is updated.
Quoted 5 lines
Move this functionality to the
clocksource::enable()callback so it gets invoked once when the clocksource is installed. This allows to make the clock mode storage generic.Signed-off-by Thomas Gleixner <[email protected]>
PATCH 9/15 clocksource: Add common vdso clock mode storage
In reply to Thomas GleixnerAll architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code.
Provide generic storage for it. The new Kconfig symbol is intermediate and will be removed once all architectures are converted over.
--- include/linux/clocksource.h | 12 +++++++++++- kernel/time/clocksource.c | 9 +++++++++ kernel/time/vsyscall.c | 7 +++++++ lib/vdso/Kconfig | 3 +++ lib/vdso/gettimeofday.c | 13 +++++++++++-- 5 files changed, 41 insertions(+), 3 deletions(-)
include/linux/clocksource.h +11 -1
@@ -23,10 +23,19 @@struct clocksource ;struct module ; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA +#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA ) || \ + defined(CONFIG_GENERIC_VDSO_CLOCK_MODE )#include <asm/clocksource.h> #endif +enum vdso_clock_mode { +VDSO_CLOCKMODE_NONE , +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +VDSO_ARCH_CLOCKMODES , +#endif +VDSO_CLOCKMODE_MAX , +}; +/** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware.
@@ -97,6 +106,7 @@ struct clocksource {const char *name;struct list_head list;int rating; +enum vdso_clock_mode vdso_clock_mode;unsigned long flags;int (*enable)(struct clocksource *cs);
kernel/time/clocksource.c +9 -0
@@ -921,6 +921,15 @@ int __clocksource_register_scale(structclocksource_arch_init (cs); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +if (cs->vdso_clock_mode <0 || + cs->vdso_clock_mode >=VDSO_CLOCKMODE_MAX ) { +pr_warn ("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n" , + cs->name, cs->vdso_clock_mode); + cs->vdso_clock_mode =VDSO_CLOCKMODE_NONE ; + } +#endif +/* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale (cs, scale, freq);
kernel/time/vsyscall.c +7 -0
@@ -72,12 +72,19 @@ void update_vsyscall(struct timekeeper *struct vdso_data *vdata =__arch_get_k_vdso_data ();struct vdso_timestamp *vdso_ts;u64 nsec; +s32 mode;/* copy vsyscall data */ vdso_write_begin (vdata); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + mode = tk->tkr_mono.clock->vdso_clock_mode; + vdata[CS_HRES_COARSE ].clock_mode = mode; + vdata[CS_RAW ].clock_mode = mode; +#else vdata[CS_HRES_COARSE ].clock_mode =__arch_get_clock_mode (tk); vdata[CS_RAW ].clock_mode =__arch_get_clock_mode (tk); +#endif /* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ];
lib/vdso/Kconfig +3 -0
@@ -30,4 +30,7 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO +config GENERIC_VDSO_CLOCK_MODE + bool + endif
lib/vdso/gettimeofday.c +11 -2
@@ -7,6 +7,7 @@#include <linux/time.h> #include <linux/kernel.h> #include <linux/hrtimer_defs.h> +#include <linux/clocksource.h> #include <vdso/datapage.h> #include <vdso/helpers.h>
@@ -64,10 +65,14 @@ static int do_hres_timens(const struct v do { seq = vdso_read_begin(vd); + if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + vd->clock_mode == VDSO_CLOCKMODE_NONE) + return -1; cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) + if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + unlikely((s64)cycles < 0)) return -1; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
@@ -132,10 +137,14 @@ static __always_inline int do_hres(const }smp_rmb (); +if (IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && + vd->clock_mode ==VDSO_CLOCKMODE_NONE ) +return -1 ; cycles =__arch_get_hw_counter (vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; -if (unlikely ((s64 )cycles <0 )) +if (!IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && +unlikely ((s64 )cycles <0 ))return -1 ; ns +=vdso_calc_delta (cycles, last, vd->mask, vd->mult);
Re: [patch 09/15] clocksource: Add common vdso clock mode storage
Le 14/01/2020 à 19:52, Thomas Gleixner a écrit :
Quoted 67 lines
All architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code.
Provide generic storage for it. The new
Kconfigsymbol is intermediate and will be removed once all architectures are converted over.Signed-off-by Thomas Gleixner <[email protected]>--- include/linux/clocksource.h | 12 +++++++++++- kernel/time/clocksource.c | 9 +++++++++ kernel/time/vsyscall.c | 7 +++++++ lib/vdso/Kconfig | 3 +++ lib/vdso/gettimeofday.c | 13 +++++++++++-- 5 files changed, 41 insertions(+), 3 deletions(-)include/linux/clocksource.h +11
-1@@ -23,10 +23,19 @@struct clocksource ;struct module ; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA +#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA ) || \ + defined(CONFIG_GENERIC_VDSO_CLOCK_MODE )#include <asm/clocksource.h> #endif +enum vdso_clock_mode { +VDSO_CLOCKMODE_NONE , +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +VDSO_ARCH_CLOCKMODES , +#endif +VDSO_CLOCKMODE_MAX , +}; +/** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. @@ -97,6 +106,7 @@ struct clocksource {const char *name;struct list_head list;int rating; +enum vdso_clock_mode vdso_clock_mode;unsigned long flags;int (*enable)(struct clocksource *cs);kernel/time/clocksource.c +9
-0@@ -921,6 +921,15 @@ int __clocksource_register_scale(structclocksource_arch_init (cs); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +if (cs->vdso_clock_mode <0 || + cs->vdso_clock_mode >=VDSO_CLOCKMODE_MAX ) { +pr_warn ("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n" , + cs->name, cs->vdso_clock_mode); + cs->vdso_clock_mode =VDSO_CLOCKMODE_NONE ; + } +#endif +/* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale (cs, scale, freq);
kernel/time/vsyscall.c void update_vsyscall(struct timekeeper * · line 75
Quoted 4 lines
kernel/time/vsyscall.c +1
-0@@ -72,12 +72,19 @@ void update_vsyscall(struct timekeeper *struct vdso_data *vdata =__arch_get_k_vdso_data ();struct vdso_timestamp *vdso_ts;u64 nsec; +s32 mode;
gcc will claim 'mode' is unused when CONFIG_GENERIC_VDSO_CLOCK_MODE is not defined.
kernel/time/vsyscall.c void update_vsyscall(struct timekeeper * · line 87
Quoted 12 lines
kernel/time/vsyscall.c +6
-0/* copy vsyscall data */ vdso_write_begin (vdata); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + mode = tk->tkr_mono.clock->vdso_clock_mode; + vdata[CS_HRES_COARSE ].clock_mode = mode; + vdata[CS_RAW ].clock_mode = mode; +#else vdata[CS_HRES_COARSE ].clock_mode =__arch_get_clock_mode (tk); vdata[CS_RAW ].clock_mode =__arch_get_clock_mode (tk); +#endif
Can we do :
#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE mode = tk->tkr_mono.clock->vdso_clock_mode; #else mode = __arch_get_clock_mode(tk); #endif vdata[CS_HRES_COARSE].clock_mode = mode; vdata[CS_RAW].clock_mode = mode;
Christophe
Quoted 56 lines
kernel/time/vsyscall.c +0
-0/* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ];lib/vdso/Kconfig +3
-0@@ -30,4 +30,7 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO +config GENERIC_VDSO_CLOCK_MODE + bool + endiflib/vdso/gettimeofday.c +11
-2@@ -7,6 +7,7 @@#include <linux/time.h> #include <linux/kernel.h> #include <linux/hrtimer_defs.h> +#include <linux/clocksource.h> #include <vdso/datapage.h> #include <vdso/helpers.h> @@ -64,10 +65,14 @@ static int do_hres_timens(const struct v do { seq = vdso_read_begin(vd); + if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + vd->clock_mode == VDSO_CLOCKMODE_NONE) + return -1; cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) + if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + unlikely((s64)cycles < 0)) return -1; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);@@ -132,10 +137,14 @@ static __always_inline int do_hres(const }smp_rmb (); +if (IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && + vd->clock_mode ==VDSO_CLOCKMODE_NONE ) +return -1 ; cycles =__arch_get_hw_counter (vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; -if (unlikely ((s64 )cycles <0 )) +if (!IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && +unlikely ((s64 )cycles <0 ))return -1 ; ns +=vdso_calc_delta (cycles, last, vd->mask, vd->mult);
Re: [patch 09/15] clocksource: Add common vdso clock mode storage
Quoted 10 lines · Christophe Leroy
kernel/time/vsyscall.c void update_vsyscall(struct timekeeper * · line 75
Quoted 7 lines
kernel/time/vsyscall.c +1
-0@@ -72,12 +72,19 @@ void update_vsyscall(struct timekeeper *struct vdso_data *vdata =__arch_get_k_vdso_data ();struct vdso_timestamp *vdso_ts;u64 nsec; +s32 mode;gcc will claim 'mode' is unused when
CONFIG_GENERIC_VDSO_CLOCK_MODEis not defined.
I know. It's intermediate and goes away a few patches later, but yes I can fix it up.
Thanks,
tglx
PATCH 10/15 x86/vdso: Use generic VDSO clock mode storage
In reply to Thomas GleixnerSwitch to the generic VDSO clock mode storage.
--- arch/x86/Kconfig | 2 +- arch/x86/entry/vdso/vma.c | 6 +++--- arch/x86/include/asm/clocksource.h | 13 ++++--------- arch/x86/include/asm/mshyperv.h | 4 ++-- arch/x86/include/asm/vdso/gettimeofday.h | 6 +++--- arch/x86/include/asm/vdso/vsyscall.h | 7 ------- arch/x86/kernel/kvmclock.c | 4 ++-- arch/x86/kernel/pvclock.c | 2 +- arch/x86/kernel/time.c | 12 +++--------- arch/x86/kernel/tsc.c | 6 +++--- arch/x86/kvm/trace.h | 4 ++-- arch/x86/kvm/x86.c | 22 +++++++++++----------- arch/x86/xen/time.c | 21 +++++++++++---------- 13 files changed, 46 insertions(+), 63 deletions(-)
arch/x86/Kconfig +1 -1
@@ -57,7 +57,6 @@ config X86 select ACPI_LEGACY_TABLES_LOOKUP if ACPI select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_32BIT_OFF_T if X86_32 - select ARCH_CLOCKSOURCE_DATA select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_DEBUG_VIRTUAL
@@ -124,6 +123,7 @@ config X86 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY + select GENERIC_VDSO_CLOCK_MODE select GENERIC_VDSO_TIME_NS select GUP_GET_PTE_LOW_HIGH if X86_PAE select HARDLOCKUP_CHECK_TIMESTAMP if X86_64
arch/x86/entry/vdso/vma.c +3 -3
@@ -221,7 +221,7 @@ static vm_fault_t vvar_fault(const struc }else if (sym_offset == image->sym_pvclock_page) {struct pvclock_vsyscall_time_info *pvti =pvclock_get_pvti_cpu0_va (); -if (pvti &&vclock_was_used (VCLOCK_PVCLOCK )) { +if (pvti &&vclock_was_used (VDSO_CLOCKMODE_PVCLOCK )) {return vmf_insert_pfn_prot (vma, vmf->address,__pa (pvti) >>PAGE_SHIFT ,pgprot_decrypted (vma->vm_page_prot));
@@ -229,7 +229,7 @@ static vm_fault_t vvar_fault(const struc }else if (sym_offset == image->sym_hvclock_page) {struct ms_hyperv_tsc_page *tsc_pg =hv_get_tsc_page (); -if (tsc_pg &&vclock_was_used (VCLOCK_HVCLOCK )) +if (tsc_pg &&vclock_was_used (VDSO_CLOCKMODE_HVCLOCK ))return vmf_insert_pfn (vma, vmf->address,virt_to_phys (tsc_pg) >>PAGE_SHIFT ); }else if (sym_offset == image->sym_timens_page) {
@@ -447,7 +447,7 @@ static __init int vdso_setup(char *s)static int __init init_vdso (void ) { -BUILD_BUG_ON (VCLOCK_MAX >=32 ); +BUILD_BUG_ON (VDSO_CLOCKMODE_MAX >=32 );init_vdso_image (&vdso_image_64);
arch/x86/include/asm/clocksource.h +4 -9
@@ -4,15 +4,10 @@#ifndef _ASM_X86_CLOCKSOURCE_H#define _ASM_X86_CLOCKSOURCE_H -#define VCLOCK_NONE 0/* No vDSO clock available. */ -#define VCLOCK_TSC 1/* vDSO should use vread_tsc. */ -#define VCLOCK_PVCLOCK 2/* vDSO should use vread_pvclock. */ -#define VCLOCK_HVCLOCK 3/* vDSO should use vread_hvclock. */ -#define VCLOCK_MAX 3 - -struct arch_clocksource_data { -int vclock_mode; -}; +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_TSC, \ + VDSO_CLOCKMODE_PVCLOCK, \ + VDSO_CLOCKMODE_HVCLOCKextern unsigned int vclocks_used;
arch/x86/include/asm/mshyperv.h +2 -2
@@ -46,9 +46,9 @@ typedef int (*hyperv_fill_flush_list_fun#define hv_set_reference_tsc (val) \ wrmsrl(HV_X64_MSR_REFERENCE_TSC, val)#define hv_set_clocksource_vdso (val) \ - ((val).archdata.vclock_mode = VCLOCK_HVCLOCK) + ((val).vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK)#define hv_enable_vdso_clocksource () \ - vclocks_set_used(VCLOCK_HVCLOCK); + vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK);#define hv_get_raw_timer () rdtsc_ordered()void hyperv_callback_vector (void );
arch/x86/include/asm/vdso/gettimeofday.h +3 -3
@@ -243,7 +243,7 @@ static u64 vread_hvclock(void)static inline u64 __arch_get_hw_counter (s32 clock_mode) { -if (clock_mode ==VCLOCK_TSC ) +if (clock_mode ==VDSO_CLOCKMODE_TSC )return (u64 )rdtsc_ordered ();/* * For any memory-mapped vclock type, we need to make sure that gcc
@@ -252,13 +252,13 @@ static inline u64 __arch_get_hw_counter(* question isn't enabled, which will segfault. Hence the barriers. */ #ifdef CONFIG_PARAVIRT_CLOCK -if (clock_mode ==VCLOCK_PVCLOCK ) { +if (clock_mode ==VDSO_CLOCKMODE_PVCLOCK ) {barrier ();return vread_pvclock (); }#endif #ifdef CONFIG_HYPERV_TIMER -if (clock_mode ==VCLOCK_HVCLOCK ) { +if (clock_mode ==VDSO_CLOCKMODE_HVCLOCK ) {barrier ();return vread_hvclock (); }
arch/x86/include/asm/vdso/vsyscall.h +0 -7
@@ -21,13 +21,6 @@ struct vdso_data *__x86_get_k_vdso_data( }#define __arch_get_k_vdso_data __x86_get_k_vdso_data -static __always_inline -int __x86_get_clock_mode(struct timekeeper *tk) -{ -return tk->tkr_mono.clock->archdata.vclock_mode; -} -#define __arch_get_clock_mode __x86_get_clock_mode -/* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h>
arch/x86/kernel/kvmclock.c +2 -2
@@ -161,7 +161,7 @@ bool kvm_check_and_clear_guest_paused(vostatic int kvm_cs_enable (struct clocksource *cs) { -vclocks_set_used (VCLOCK_PVCLOCK ); +vclocks_set_used (VDSO_CLOCKMODE_PVCLOCK );return 0 ; }
@@ -279,7 +279,7 @@ static int __init kvm_setup_vsyscall_timif (!(flags &PVCLOCK_TSC_STABLE_BIT ))return 0 ; - kvm_clock.archdata.vclock_mode =VCLOCK_PVCLOCK ; + kvm_clock.vdso_clock_mode =VDSO_CLOCKMODE_PVCLOCK ;#endif kvmclock_init_mem ();
arch/x86/kernel/pvclock.c +1 -1
@@ -145,7 +145,7 @@ void pvclock_read_wallclock(struct pvclovoid pvclock_set_pvti_cpu0_va (struct pvclock_vsyscall_time_info *pvti) { -WARN_ON (vclock_was_used (VCLOCK_PVCLOCK )); +WARN_ON (vclock_was_used (VDSO_CLOCKMODE_PVCLOCK )); pvti_cpu0_va = pvti; }
arch/x86/kernel/time.c +3 -9
@@ -114,18 +114,12 @@ void __init time_init(void)*/ void clocksource_arch_init (struct clocksource *cs) { -if (cs->archdata.vclock_mode ==VCLOCK_NONE ) +if (cs->vdso_clock_mode ==VDSO_CLOCKMODE_NONE )return ; -if (cs->archdata.vclock_mode >VCLOCK_MAX ) { -pr_warn ("clocksource %s registered with invalid vclock_mode %d. Disabling vclock.\n" , - cs->name, cs->archdata.vclock_mode); - cs->archdata.vclock_mode =VCLOCK_NONE ; - } -if (cs->mask !=CLOCKSOURCE_MASK (64 )) { -pr_warn ("clocksource %s registered with invalid mask %016llx. Disabling vclock.\n" , +pr_warn ("clocksource %s registered with invalid mask %016llx for VDSO. Disabling VDSO support.\n" , cs->name, cs->mask); - cs->archdata.vclock_mode =VCLOCK_NONE ; + cs->vdso_clock_mode =VDSO_CLOCKMODE_NONE ; } }
arch/x86/kernel/tsc.c +3 -3
@@ -1110,7 +1110,7 @@ static void tsc_cs_tick_stable(struct clstatic int tsc_cs_enable (struct clocksource *cs) { -vclocks_set_used (VCLOCK_TSC ); +vclocks_set_used (VDSO_CLOCKMODE_TSC );return 0 ; }
@@ -1124,7 +1124,7 @@ static struct clocksource clocksource_ts .mask = CLOCKSOURCE_MASK(64), .flags =CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable,
@@ -1145,7 +1145,7 @@ static struct clocksource clocksource_ts .flags = CLOCK_SOURCE_IS_CONTINUOUS |CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable,
arch/x86/kvm/trace.h +2 -2
@@ -815,8 +815,8 @@ TRACE_EVENT(kvm_write_tsc_offset,#ifdef CONFIG_X86_64 #define host_clocks \ - {VCLOCK_NONE, "none"}, \ - {VCLOCK_TSC, "tsc"} \ + {VDSO_CLOCKMODE_NONE, "none"}, \ + {VDSO_CLOCKMODE_TSC, "tsc"} \ TRACE_EVENT(kvm_update_master_clock, TP_PROTO(bool use_master_clock, unsigned int host_clock, bool offset_matched),
arch/x86/kvm/x86.c +11 -11
@@ -1573,13 +1573,13 @@ static void update_pvclock_gtod(struct twrite_seqcount_begin (&vdata->seq);/* copy pvclock gtod data */ - vdata->clock.vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; + vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; vdata->clock.cycle_last = tk->tkr_mono.cycle_last; vdata->clock.mask = tk->tkr_mono.mask; vdata->clock.mult = tk->tkr_mono.mult; vdata->clock.shift = tk->tkr_mono.shift; - vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->archdata.vclock_mode; + vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; vdata->raw_clock.mask = tk->tkr_raw.mask; vdata->raw_clock.mult = tk->tkr_raw.mult;
@@ -1775,7 +1775,7 @@ static u64 compute_guest_tsc(struct kvm_static inline int gtod_is_based_on_tsc (int mode) { -return mode ==VCLOCK_TSC || mode ==VCLOCK_HVCLOCK ; +return mode ==VDSO_CLOCKMODE_TSC || mode ==VDSO_CLOCKMODE_HVCLOCK ; }static void kvm_track_tsc_matching (struct kvm_vcpu *vcpu)
@@ -1868,7 +1868,7 @@ static inline bool kvm_check_tsc_unstabl* TSC is marked unstable when we're running on Hyper-V, * 'TSC page' clocksource is good. */ -if (pvclock_gtod_data.clock.vclock_mode ==VCLOCK_HVCLOCK ) +if (pvclock_gtod_data.clock.vclock_mode ==VDSO_CLOCKMODE_HVCLOCK )return false;#endif return check_tsc_unstable ();
@@ -2023,30 +2023,30 @@ static inline u64 vgettsc(struct pvclocku64 tsc_pg_val;switch (clock->vclock_mode) { -case VCLOCK_HVCLOCK : +case VDSO_CLOCKMODE_HVCLOCK : tsc_pg_val =hv_read_tsc_page_tsc (hv_get_tsc_page (), tsc_timestamp);if (tsc_pg_val !=U64_MAX ) {/* TSC page valid */ - *mode =VCLOCK_HVCLOCK ; + *mode =VDSO_CLOCKMODE_HVCLOCK ; v = (tsc_pg_val - clock->cycle_last) & clock->mask; }else {/* TSC page invalid */ - *mode =VCLOCK_NONE ; + *mode =VDSO_CLOCKMODE_NONE ; }break ; -case VCLOCK_TSC : - *mode =VCLOCK_TSC ; +case VDSO_CLOCKMODE_TSC : + *mode =VDSO_CLOCKMODE_TSC ; *tsc_timestamp =read_tsc (); v = (*tsc_timestamp - clock->cycle_last) & clock->mask;break ;default : - *mode =VCLOCK_NONE ; + *mode =VDSO_CLOCKMODE_NONE ; } -if (*mode ==VCLOCK_NONE ) +if (*mode ==VDSO_CLOCKMODE_NONE ) *tsc_timestamp = v =0 ;return v * clock->mult;
arch/x86/xen/time.c +11 -10
@@ -147,7 +147,7 @@ static struct notifier_block xen_pvclockstatic int xen_cs_enable (struct clocksource *cs) { -vclocks_set_used (VCLOCK_PVCLOCK ); +vclocks_set_used (VDSO_CLOCKMODE_PVCLOCK );return 0 ; }
@@ -419,12 +419,13 @@ void xen_restore_time_memory_area(void) ret =HYPERVISOR_vcpu_op (VCPUOP_register_vcpu_time_memory_area,0 , &t);/* -* We don't disable VCLOCK_PVCLOCK entirely if it fails to register the -* secondary time info with Xen or if we migrated to a host without the -* necessary flags. On both of these cases what happens is either -* process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT -* bit set. Userspace checks the latter and if 0, it discards the data -* in pvti and fallbacks to a system call for a reliable timestamp. +* We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to +* register the secondary time info with Xen or if we migrated to a +* host without the necessary flags. On both of these cases what +* happens is either process seeing a zeroed out pvti or seeing no +* PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and +* if 0, it discards the data in pvti and fallbacks to a system +* call for a reliable timestamp. */ if (ret !=0 ) pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
@@ -450,7 +451,7 @@ static void xen_setup_vsyscall_time_info ret =HYPERVISOR_vcpu_op (VCPUOP_register_vcpu_time_memory_area,0 , &t);if (ret) { -pr_notice ("xen: VCLOCK_PVCLOCK not supported (err %d)\n" , ret); +pr_notice ("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n" , ret);free_page ((unsigned long )ti);return ; }
@@ -467,14 +468,14 @@ static void xen_setup_vsyscall_time_infoif (!ret)free_page ((unsigned long )ti); -pr_notice ("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n" ); +pr_notice ("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n" );return ; } xen_clock = ti;pvclock_set_pvti_cpu0_va (xen_clock); - xen_clocksource.archdata.vclock_mode =VCLOCK_PVCLOCK ; + xen_clocksource.vdso_clock_mode =VDSO_CLOCKMODE_PVCLOCK ; }static void __init xen_time_init (void )
Re: [patch 10/15] x86/vdso: Use generic VDSO clock mode storage
Quoted 3 lines · Thomas Gleixner
Switch to the generic VDSO clock mode storage.
Signed-off-by Thomas Gleixner <[email protected]>
Juergen
PATCH 11/15 mips: vdso: Use generic VDSO clock mode storage
In reply to Thomas GleixnerSwitch to the generic VDSO clock mode storage.
--- arch/mips/Kconfig | 2 +- arch/mips/include/asm/clocksource.h | 18 +++--------------- arch/mips/include/asm/vdso/gettimeofday.h | 24 ++++++------------------ arch/mips/include/asm/vdso/vsyscall.h | 9 --------- arch/mips/kernel/csrc-r4k.c | 2 +- drivers/clocksource/mips-gic-timer.c | 8 ++++---- 6 files changed, 15 insertions(+), 48 deletions(-)
arch/mips/Kconfig +1 -1
@@ -4,7 +4,6 @@ config MIPS default y select ARCH_32BIT_OFF_T if !64BIT select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAS_FORTIFY_SOURCE
@@ -36,6 +35,7 @@ config MIPS select GENERIC_SCHED_CLOCK if !CAVIUM_OCTEON_SOC select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_CLOCK_MODE select GUP_GET_PTE_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_COMPILER_H
arch/mips/include/asm/clocksource.h +3 -15
@@ -3,23 +3,11 @@* Copyright (C) 2015 Imagination Technologies * Author: Alex Smith <[email protected]> */ -#ifndef __ASM_CLOCKSOURCE_H#define __ASM_CLOCKSOURCE_H -#include <linux/types.h> - -/* VDSO clocksources. */ -#define VDSO_CLOCK_NONE 0/* No suitable clocksource. */ -#define VDSO_CLOCK_R4K 1/* Use the coprocessor 0 count. */ -#define VDSO_CLOCK_GIC 2/* Use the GIC. */ - -/** -* struct arch_clocksource_data - Architecture-specific clocksource information. -* @vdso_clock_mode: Method the VDSO should use to access the clocksource. -*/ -struct arch_clocksource_data { -u8 vdso_clock_mode; -}; +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMDOE_R4K, \ + VDSO_CLOCKMODE_GIC#endif /* __ASM_CLOCKSOURCE_H */
arch/mips/include/asm/vdso/gettimeofday.h +6 -18
@@ -175,28 +175,16 @@ static __always_inline u64 read_gic_coun static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) { -#ifdef CONFIG_CLKSRC_MIPS_GIC -const struct vdso_data *data =get_vdso_data (); -#endif -u64 cycle_now; - -switch (clock_mode) {#ifdef CONFIG_CSRC_R4K -case VDSO_CLOCK_R4K : - cycle_now =read_r4k_count (); -break ; +if (clock_mode ==VDSO_CLOCKMODE_R4K ) +return read_r4k_count ();#endif #ifdef CONFIG_CLKSRC_MIPS_GIC -case VDSO_CLOCK_GIC : - cycle_now =read_gic_count (data); -break ; +if (clock_mode ==VDSO_CLOCKMODE_GIC ) +return read_gic_count (get_vdso_data ());#endif -default : - cycle_now = __VDSO_USE_SYSCALL; -break ; - } - -return cycle_now; +/* Keep GCC happy */ +return U64_MAX ; }static inline bool mips_vdso_hres_capable (void )
arch/mips/include/asm/vdso/vsyscall.h +0 -9
@@ -19,15 +19,6 @@ struct vdso_data *__mips_get_k_vdso_data }#define __arch_get_k_vdso_data __mips_get_k_vdso_data -static __always_inline -int __mips_get_clock_mode(struct timekeeper *tk) -{ -u32 clock_mode = tk->tkr_mono.clock->archdata.vdso_clock_mode; - -return clock_mode; -} -#define __arch_get_clock_mode __mips_get_clock_mode -/* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h>
arch/mips/kernel/csrc-r4k.c +1 -1
@@ -78,7 +78,7 @@ int __init init_r4k_clocksource(void)* by the VDSO (HWREna is configured by configure_hwrena()). */ if (cpu_has_mips_r2_r6 &&rdhwr_count_usable ()) - clocksource_mips.archdata.vdso_clock_mode =VDSO_CLOCK_R4K ; + clocksource_mips.vdso_clock_mode =VDSO_CLOCKMODE_R4K ;clocksource_register_hz (&clocksource_mips, mips_hpt_frequency);
drivers/clocksource/mips-gic-timer.c +4 -4
@@ -155,10 +155,10 @@ static u64 gic_hpt_read(struct clocksour }static struct clocksource gic_clocksource = { - .name ="GIC" , - .read = gic_hpt_read, - .flags =CLOCK_SOURCE_IS_CONTINUOUS , - .archdata = { .vdso_clock_mode =VDSO_CLOCK_GIC }, + .name ="GIC" , + .read = gic_hpt_read, + .flags =CLOCK_SOURCE_IS_CONTINUOUS , + .vdso_clock_mode =VDSO_CLOCKMODE_GIC , };static int __init __gic_clocksource_init (void )
PATCH 12/15 ARM/arm64: vdso: Use common vdso clock mode storage
In reply to Thomas GleixnerConvert ARM/ARM64 to the generic VDSO clock mode storage. This needs to happen in one go as they share the clocksource driver.
--- arch/arm/Kconfig | 1 - arch/arm/include/asm/clocksource.h | 5 ++--- arch/arm/include/asm/vdso/vsyscall.h | 21 --------------------- arch/arm/mm/Kconfig | 1 + arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/clocksource.h | 5 ++--- arch/arm64/include/asm/vdso/vsyscall.h | 9 --------- drivers/clocksource/arm_arch_timer.c | 8 ++++---- 8 files changed, 10 insertions(+), 42 deletions(-)
arch/arm/Kconfig +0 -1
@@ -3,7 +3,6 @@ config ARM bool default y select ARCH_32BIT_OFF_T - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEVMEM_IS_ALLOWED
arch/arm/include/asm/clocksource.h +2 -3
@@ -1,8 +1,7 @@#ifndef _ASM_CLOCKSOURCE_H#define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { -bool vdso_direct;/* Usable for direct VDSO access? */ -}; +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER#endif
arch/arm/include/asm/vdso/vsyscall.h +0 -21
@@ -11,18 +11,6 @@extern struct vdso_data *vdso_data;extern bool cntvct_ok; -static __always_inline -bool tk_is_cntvct(const struct timekeeper *tk) -{ -if (!IS_ENABLED (CONFIG_ARM_ARCH_TIMER )) -return false; - -if (!tk->tkr_mono.clock->archdata.vdso_direct) -return false; - -return true; -} -/* * Update the vDSO data page to keep in sync with kernel timekeeping. */
@@ -41,15 +29,6 @@ bool __arm_update_vdso_data(void)#define __arch_update_vdso_data __arm_update_vdso_datastatic __always_inline -int __arm_get_clock_mode(struct timekeeper *tk) -{ -u32 __tk_is_cntvct =tk_is_cntvct (tk); - -return __tk_is_cntvct; -} -#define __arch_get_clock_mode __arm_get_clock_mode - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) {flush_dcache_page (virt_to_page (vdata));
arch/arm/mm/Kconfig +1 -0
@@ -900,6 +900,7 @@ config VDSO select GENERIC_TIME_VSYSCALL select GENERIC_VDSO_32 select GENERIC_GETTIMEOFDAY + select GENERIC_VDSO_CLOCK_MODE help Place in the process address space an ELF shared object providing fast implementations of gettimeofday and
arch/arm64/Kconfig +1 -1
@@ -9,7 +9,6 @@ config ARM64 select ACPI_MCFG if (ACPI && PCI) select ACPI_SPCR_TABLE if ACPI select ACPI_PPTT if ACPI - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_HAS_DMA_PREP_COHERENT
@@ -109,6 +108,7 @@ config ARM64 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY + select GENERIC_VDSO_CLOCK_MODE select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND select HAVE_PCI
arch/arm64/include/asm/clocksource.h +2 -3
@@ -2,8 +2,7 @@#ifndef _ASM_CLOCKSOURCE_H#define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { -bool vdso_direct;/* Usable for direct VDSO access? */ -}; +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER#endif
arch/arm64/include/asm/vdso/vsyscall.h +0 -9
@@ -22,15 +22,6 @@ struct vdso_data *__arm64_get_k_vdso_dat#define __arch_get_k_vdso_data __arm64_get_k_vdso_datastatic __always_inline -int __arm64_get_clock_mode(struct timekeeper *tk) -{ -u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; - -return use_syscall; -} -#define __arch_get_clock_mode __arm64_get_clock_mode - -static __always_inline void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) { vdata[CS_HRES_COARSE ].mask =VDSO_PRECISION_MASK ;
drivers/clocksource/arm_arch_timer.c +4 -4
@@ -69,7 +69,7 @@ static enum arch_timer_ppi_nr arch_timerstatic bool arch_timer_c3stop;static bool arch_timer_mem_use_virtual;static bool arch_counter_suspend_stop; -static bool vdso_default = true; +static enum vdso_clock_mode vdso_default =VDSO_CLOCKMODE_ARCHTIMER ;static cpumask_t evtstrm_available =CPU_MASK_NONE ;static bool evtstrm_enable =IS_ENABLED (CONFIG_ARM_ARCH_TIMER_EVTSTREAM );
@@ -560,8 +560,8 @@ void arch_timer_enable_workaround(const* change both the default value and the vdso itself. */ if (wa->read_cntvct_el0) { - clocksource_counter.archdata.vdso_direct = false; - vdso_default = false; + clocksource_counter.vdso_clock_mode =VDSO_CLOCKMODE_NONE ; + vdso_default =VDSO_CLOCKMODE_NONE ; } }
@@ -979,7 +979,7 @@ static void __init arch_counter_register } arch_timer_read_counter = rd; - clocksource_counter.archdata.vdso_direct = vdso_default; + clocksource_counter.vdso_clock_mode = vdso_default; }else { arch_timer_read_counter = arch_counter_get_cntvct_mem; }
PATCH 13/15 lib/vdso: Cleanup clock mode storage leftovers
In reply to Thomas GleixnerNow that all architectures are converted to use the generic storage the helpers and conditionals can be removed.
--- arch/arm/mm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/x86/Kconfig | 1 - include/asm-generic/vdso/vsyscall.h | 7 ------- include/linux/clocksource.h | 4 ++-- kernel/time/vsyscall.c | 5 ----- lib/vdso/Kconfig | 3 --- lib/vdso/gettimeofday.c | 17 +++++------------ 9 files changed, 7 insertions(+), 33 deletions(-)
arch/arm/mm/Kconfig +0 -1
@@ -900,7 +900,6 @@ config VDSO select GENERIC_TIME_VSYSCALL select GENERIC_VDSO_32 select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE help Place in the process address space an ELF shared object providing fast implementations of gettimeofday and
arch/arm64/Kconfig +0 -1
@@ -108,7 +108,6 @@ config ARM64 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND select HAVE_PCI
arch/mips/Kconfig +0 -1
@@ -35,7 +35,6 @@ config MIPS select GENERIC_SCHED_CLOCK if !CAVIUM_OCTEON_SOC select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL - select GENERIC_VDSO_CLOCK_MODE select GUP_GET_PTE_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_COMPILER_H
arch/x86/Kconfig +0 -1
@@ -123,7 +123,6 @@ config X86 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE select GENERIC_VDSO_TIME_NS select GUP_GET_PTE_LOW_HIGH if X86_PAE select HARDLOCKUP_CHECK_TIMESTAMP if X86_64
include/asm-generic/vdso/vsyscall.h +0 -7
@@ -18,13 +18,6 @@ static __always_inline bool __arch_updat }#endif /* __arch_update_vdso_data */ -#ifndef __arch_get_clock_mode -static __always_inline int __arch_get_clock_mode(struct timekeeper *tk) -{ -return 0 ; -} -#endif /* __arch_get_clock_mode */ -#ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata,struct timekeeper *tk)
include/linux/clocksource.h +2 -2
@@ -24,13 +24,13 @@ struct clocksource;struct module ;#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA ) || \ - defined(CONFIG_GENERIC_VDSO_CLOCK_MODE ) + defined(CONFIG_GENERIC_GETTIMEOFDAY )#include <asm/clocksource.h> #endif enum vdso_clock_mode {VDSO_CLOCKMODE_NONE , -#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +#ifdef CONFIG_GENERIC_GETTIMEOFDAY VDSO_ARCH_CLOCKMODES ,#endif VDSO_CLOCKMODE_MAX ,
kernel/time/vsyscall.c +0 -5
@@ -77,14 +77,9 @@ void update_vsyscall(struct timekeeper */* copy vsyscall data */ vdso_write_begin (vdata); -#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE mode = tk->tkr_mono.clock->vdso_clock_mode; vdata[CS_HRES_COARSE ].clock_mode = mode; vdata[CS_RAW ].clock_mode = mode; -#else - vdata[CS_HRES_COARSE ].clock_mode =__arch_get_clock_mode (tk); - vdata[CS_RAW ].clock_mode =__arch_get_clock_mode (tk); -#endif /* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ];
lib/vdso/Kconfig +0 -3
@@ -30,7 +30,4 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO -config GENERIC_VDSO_CLOCK_MODE - bool - endif
lib/vdso/gettimeofday.c +5 -12
@@ -65,16 +65,13 @@ static int do_hres_timens(const struct v do { seq = vdso_read_begin(vd); - if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - vd->clock_mode == VDSO_CLOCKMODE_NONE) + + if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) return -1; + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - unlikely((s64)cycles < 0)) - return -1; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); ns >>= vd->shift; sec = vdso_ts->sec;
@@ -137,16 +134,12 @@ static __always_inline int do_hres(const }smp_rmb (); -if (IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && - vd->clock_mode ==VDSO_CLOCKMODE_NONE ) +if (unlikely (vd->clock_mode ==VDSO_CLOCKMODE_NONE ))return -1 ; + cycles =__arch_get_hw_counter (vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; -if (!IS_ENABLED (CONFIG_GENERIC_VDSO_CLOCK_MODE ) && -unlikely ((s64 )cycles <0 )) -return -1 ; - ns +=vdso_calc_delta (cycles, last, vd->mask, vd->mult); ns >>= vd->shift; sec = vdso_ts->sec;
PATCH 14/15 lib/vdso: Avoid highres update if clocksource is not VDSO capable
In reply to Thomas GleixnerIf the current clocksource is not VDSO capable there is no point in updating the high resolution parts of the VDSO data.
Replace the architecture specific check with a check for a VDSO capable clocksource and skip the update if there is none.
--- arch/arm/include/asm/vdso/vsyscall.h | 7 ------- include/asm-generic/vdso/vsyscall.h | 7 ------- kernel/time/vsyscall.c | 14 +++++++------- 3 files changed, 7 insertions(+), 21 deletions(-)
arch/arm/include/asm/vdso/vsyscall.h +0 -7
@@ -22,13 +22,6 @@ struct vdso_data *__arm_get_k_vdso_data(#define __arch_get_k_vdso_data __arm_get_k_vdso_datastatic __always_inline -bool __arm_update_vdso_data(void) -{ -return cntvct_ok; -} -#define __arch_update_vdso_data __arm_update_vdso_data - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) {flush_dcache_page (virt_to_page (vdata));
include/asm-generic/vdso/vsyscall.h +0 -7
@@ -11,13 +11,6 @@ static __always_inline struct vdso_data }#endif /* __arch_get_k_vdso_data */ -#ifndef __arch_update_vdso_data -static __always_inline bool __arch_update_vdso_data(void) -{ -return true; -} -#endif /* __arch_update_vdso_data */ -#ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata,struct timekeeper *tk)
kernel/time/vsyscall.c +7 -7
@@ -71,15 +71,15 @@ void update_vsyscall(struct timekeeper * {struct vdso_data *vdata =__arch_get_k_vdso_data ();struct vdso_timestamp *vdso_ts; +s32 clock_mode;u64 nsec; -s32 mode;/* copy vsyscall data */ vdso_write_begin (vdata); - mode = tk->tkr_mono.clock->vdso_clock_mode; - vdata[CS_HRES_COARSE ].clock_mode = mode; - vdata[CS_RAW ].clock_mode = mode; + clock_mode = tk->tkr_mono.clock->vdso_clock_mode; + vdata[CS_HRES_COARSE ].clock_mode = clock_mode; + vdata[CS_RAW ].clock_mode = clock_mode;/* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ];
@@ -105,10 +105,10 @@ void update_vsyscall(struct timekeeper *WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution);/* -* Architectures can opt out of updating the high resolution part -* of the VDSO. +* If the current clocksource is not VDSO capable, then spare the +* update of the high reolution parts. */ -if (__arch_update_vdso_data ()) +if (clock_mode !=VDSO_CLOCKMODE_NONE )update_vdso_data (vdata, tk);__arch_update_vsyscall (vdata, tk);
Re: [patch 14/15] lib/vdso: Avoid highres update if clocksource is not VDSO capable
Le 14/01/2020 à 19:52, Thomas Gleixner a écrit :
Quoted 48 lines
If the current clocksource is not VDSO capable there is no point in updating the high resolution parts of the VDSO data.
Replace the architecture specific check with a check for a VDSO capable clocksource and skip the update if there is none.
Signed-off-by Thomas Gleixner <[email protected]>--- arch/arm/include/asm/vdso/vsyscall.h | 7 ------- include/asm-generic/vdso/vsyscall.h | 7 ------- kernel/time/vsyscall.c | 14 +++++++------- 3 files changed, 7 insertions(+), 21 deletions(-)arch/arm/include/asm/vdso/vsyscall.h +0
-7@@ -22,13 +22,6 @@ struct vdso_data *__arm_get_k_vdso_data(#define __arch_get_k_vdso_data __arm_get_k_vdso_datastatic __always_inline -bool __arm_update_vdso_data(void) -{ -return cntvct_ok; -} -#define __arch_update_vdso_data __arm_update_vdso_data - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) {flush_dcache_page (virt_to_page (vdata));include/asm-generic/vdso/vsyscall.h +0
-7@@ -11,13 +11,6 @@ static __always_inline struct vdso_data }#endif /* __arch_get_k_vdso_data */ -#ifndef __arch_update_vdso_data -static __always_inline bool __arch_update_vdso_data(void) -{ -return true; -} -#endif /* __arch_update_vdso_data */ -#ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata,struct timekeeper *tk)
kernel/time/vsyscall.c void update_vsyscall(struct timekeeper * · line 75
Quoted 6 lines
kernel/time/vsyscall.c +1
-1@@ -71,15 +71,15 @@ void update_vsyscall(struct timekeeper * {struct vdso_data *vdata =__arch_get_k_vdso_data ();struct vdso_timestamp *vdso_ts; +s32 clock_mode;u64 nsec; -s32 mode;
Can we name is clock_mode in patch 9 instead of changing name here ?
Christophe
Quoted 28 lines
kernel/time/vsyscall.c +6
-6/* copy vsyscall data */ vdso_write_begin (vdata); - mode = tk->tkr_mono.clock->vdso_clock_mode; - vdata[CS_HRES_COARSE ].clock_mode = mode; - vdata[CS_RAW ].clock_mode = mode; + clock_mode = tk->tkr_mono.clock->vdso_clock_mode; + vdata[CS_HRES_COARSE ].clock_mode = clock_mode; + vdata[CS_RAW ].clock_mode = clock_mode;/* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ];@@ -105,10 +105,10 @@ void update_vsyscall(struct timekeeper *WRITE_ONCE (vdata[CS_HRES_COARSE ].hrtimer_res, hrtimer_resolution);/* -* Architectures can opt out of updating the high resolution part -* of the VDSO. +* If the current clocksource is not VDSO capable, then spare the +* update of the high reolution parts. */ -if (__arch_update_vdso_data ()) +if (clock_mode !=VDSO_CLOCKMODE_NONE )update_vdso_data (vdata, tk);__arch_update_vsyscall (vdata, tk);
PATCH 15/15 lib/vdso: Move VCLOCK_TIMENS to vdso_clock_modes
In reply to Thomas GleixnerMove the time namespace indicator clock mode to the other ones for consistency sake.
--- include/linux/clocksource.h | 3 +++ include/vdso/datapage.h | 2 -- kernel/time/namespace.c | 7 ++++--- lib/vdso/gettimeofday.c | 18 ++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-)
include/linux/clocksource.h +3 -0
@@ -34,6 +34,9 @@ enum vdso_clock_mode {VDSO_ARCH_CLOCKMODES , #endifVDSO_CLOCKMODE_MAX , + +/* Indicator for time namespace VDSO */ +VDSO_CLOCKMODE_TIMENS =INT_MAX };/**
include/vdso/datapage.h +0 -2
@@ -21,8 +21,6 @@#define CS_RAW 1#define CS_BASES (CS_RAW + 1) -#define VCLOCK_TIMENS UINT_MAX -/** * struct vdso_timestamp - basetime per clock_id * @sec: seconds
kernel/time/namespace.c +4 -3
@@ -8,6 +8,7 @@#include <linux/user_namespace.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> +#include <linux/clocksource.h> #include <linux/seq_file.h> #include <linux/proc_ns.h> #include <linux/export.h>
@@ -172,8 +173,8 @@ static struct timens_offset offset_from_* for vdso_data->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * -* Timens page has vdso_data->clock_mode set to VCLOCK_TIMENS which enforces -* the time namespace handling path. +* Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which +* enforces the time namespace handling path. */ static void timens_setup_vdso_data (struct vdso_data *vdata,struct time_namespace *ns)
@@ -183,7 +184,7 @@ static void timens_setup_vdso_data(strucstruct timens_offset boottime =offset_from_ts (ns->offsets.boottime); vdata->seq =1 ; - vdata->clock_mode =VCLOCK_TIMENS ; + vdata->clock_mode =VDSO_CLOCKMODE_TIMENS ; offset[CLOCK_MONOTONIC ] = monotonic; offset[CLOCK_MONOTONIC_RAW ] = monotonic; offset[CLOCK_MONOTONIC_COARSE ] = monotonic;
lib/vdso/gettimeofday.c +10 -8
@@ -116,10 +116,10 @@ static __always_inline int do_hres(const do { /* - * Open coded to handle VCLOCK_TIMENS. Time namespace + * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace * enabled tasks have a special VVAR page installed which * has vd->seq set to 1 and vd->clock_mode set to - * VCLOCK_TIMENS. For non time namespace affected tasks + * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks * this does not affect performance because if vd->seq is * odd, i.e. a concurrent update is in progress the extra * check for vd->clock_mode is just a few extra
@@ -128,7 +128,7 @@ static __always_inline int do_hres(const*/ while (unlikely ((seq =READ_ONCE (vd->seq)) &1 )) {if (IS_ENABLED (CONFIG_TIME_NS ) && - vd->clock_mode ==VCLOCK_TIMENS ) + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS )return do_hres_timens (vd, clk, ts);cpu_relax (); }
@@ -200,12 +200,12 @@ static __always_inline int do_coarse(con do { /* - * Open coded to handle VCLOCK_TIMENS. See comment in + * Open coded to handle VDSO_CLOCK_TIMENS. See comment in * do_hres(). */ while ((seq = READ_ONCE(vd->seq)) & 1) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_coarse_timens(vd, clk, ts); cpu_relax(); }
@@ -292,7 +292,7 @@ static __maybe_unused intif (unlikely (tz !=NULL )) {if (IS_ENABLED (CONFIG_TIME_NS ) && - vd->clock_mode ==VCLOCK_TIMENS ) + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); tz->tz_minuteswest = vd[CS_HRES_COARSE ].tz_minuteswest;
@@ -308,7 +308,8 @@ static __maybe_unused __kernel_old_time_const struct vdso_data *vd =__arch_get_vdso_data ();__kernel_old_time_t t; -if (IS_ENABLED (CONFIG_TIME_NS ) && vd->clock_mode ==VCLOCK_TIMENS ) +if (IS_ENABLED (CONFIG_TIME_NS ) && + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); t =READ_ONCE (vd[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ].sec);
@@ -333,7 +334,8 @@ int __cvdso_clock_getres_common(clockid_if (unlikely ((u32 ) clock >=MAX_CLOCKS ))return -1 ; -if (IS_ENABLED (CONFIG_TIME_NS ) && vd->clock_mode ==VCLOCK_TIMENS ) +if (IS_ENABLED (CONFIG_TIME_NS ) && + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); hrtimer_res =READ_ONCE (vd[CS_HRES_COARSE ].hrtimer_res);
Re: [patch 15/15] lib/vdso: Move VCLOCK_TIMENS to vdso_clock_modes
Le 14/01/2020 à 19:52, Thomas Gleixner a écrit :
Quoted 14 lines
Move the time namespace indicator clock mode to the other ones for consistency sake.
Signed-off-by Thomas Gleixner <[email protected]>--- include/linux/clocksource.h | 3 +++ include/vdso/datapage.h | 2 -- kernel/time/namespace.c | 7 ++++--- lib/vdso/gettimeofday.c | 18 ++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-)
include/linux/clocksource.h enum vdso_clock_mode { · line 38
Quoted 5 lines
include/linux/clocksource.h +2
-0@@ -34,6 +34,9 @@ enum vdso_clock_mode {VDSO_ARCH_CLOCKMODES , #endifVDSO_CLOCKMODE_MAX , + + /* Indicator for time namespace VDSO */ > + VDSO_CLOCKMODE_TIMENS = INT_MAX
Can we shorten the names a bit in order to avoid code line splits ? WOuldn't something like VDSO_CMODE_XXX be explicit enough ?
Christophe
Quoted 113 lines
include/linux/clocksource.h +0
-0};/** include/vdso/datapage.h +0
-2@@ -21,8 +21,6 @@#define CS_RAW 1#define CS_BASES (CS_RAW + 1) -#define VCLOCK_TIMENS UINT_MAX -/** * struct vdso_timestamp - basetime per clock_id * @sec: seconds kernel/time/namespace.c +4
-3@@ -8,6 +8,7 @@#include <linux/user_namespace.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> +#include <linux/clocksource.h> #include <linux/seq_file.h> #include <linux/proc_ns.h> #include <linux/export.h> @@ -172,8 +173,8 @@ static struct timens_offset offset_from_* for vdso_data->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * -* Timens page has vdso_data->clock_mode set to VCLOCK_TIMENS which enforces -* the time namespace handling path. +* Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which +* enforces the time namespace handling path. */ static void timens_setup_vdso_data (struct vdso_data *vdata,struct time_namespace *ns)@@ -183,7 +184,7 @@ static void timens_setup_vdso_data(strucstruct timens_offset boottime =offset_from_ts (ns->offsets.boottime); vdata->seq =1 ; - vdata->clock_mode =VCLOCK_TIMENS ; + vdata->clock_mode =VDSO_CLOCKMODE_TIMENS ; offset[CLOCK_MONOTONIC ] = monotonic; offset[CLOCK_MONOTONIC_RAW ] = monotonic; offset[CLOCK_MONOTONIC_COARSE ] = monotonic;lib/vdso/gettimeofday.c +10
-8@@ -116,10 +116,10 @@ static __always_inline int do_hres(const do { /* - * Open coded to handle VCLOCK_TIMENS. Time namespace + * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace * enabled tasks have a special VVAR page installed which * has vd->seq set to 1 and vd->clock_mode set to - * VCLOCK_TIMENS. For non time namespace affected tasks + * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks * this does not affect performance because if vd->seq is * odd, i.e. a concurrent update is in progress the extra * check for vd->clock_mode is just a few extra@@ -128,7 +128,7 @@ static __always_inline int do_hres(const*/ while (unlikely ((seq =READ_ONCE (vd->seq)) &1 )) {if (IS_ENABLED (CONFIG_TIME_NS ) && - vd->clock_mode ==VCLOCK_TIMENS ) + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS )return do_hres_timens (vd, clk, ts);cpu_relax (); }@@ -200,12 +200,12 @@ static __always_inline int do_coarse(con do { /* - * Open coded to handle VCLOCK_TIMENS. See comment in + * Open coded to handle VDSO_CLOCK_TIMENS. See comment in * do_hres(). */ while ((seq = READ_ONCE(vd->seq)) & 1) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_coarse_timens(vd, clk, ts); cpu_relax(); }@@ -292,7 +292,7 @@ static __maybe_unused intif (unlikely (tz !=NULL )) {if (IS_ENABLED (CONFIG_TIME_NS ) && - vd->clock_mode ==VCLOCK_TIMENS ) + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); tz->tz_minuteswest = vd[CS_HRES_COARSE ].tz_minuteswest;@@ -308,7 +308,8 @@ static __maybe_unused __kernel_old_time_const struct vdso_data *vd =__arch_get_vdso_data ();__kernel_old_time_t t; -if (IS_ENABLED (CONFIG_TIME_NS ) && vd->clock_mode ==VCLOCK_TIMENS ) +if (IS_ENABLED (CONFIG_TIME_NS ) && + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); t =READ_ONCE (vd[CS_HRES_COARSE ].basetime[CLOCK_REALTIME ].sec);@@ -333,7 +334,8 @@ int __cvdso_clock_getres_common(clockid_if (unlikely ((u32 ) clock >=MAX_CLOCKS ))return -1 ; -if (IS_ENABLED (CONFIG_TIME_NS ) && vd->clock_mode ==VCLOCK_TIMENS ) +if (IS_ENABLED (CONFIG_TIME_NS ) && + vd->clock_mode ==VDSO_CLOCKMODE_TIMENS ) vd =__arch_get_timens_vdso_data (); hrtimer_res =READ_ONCE (vd[CS_HRES_COARSE ].hrtimer_res);