kvm/x86: rename kvm's read_tsc() as kvm_read_host_tsc()
Subsystems (6)
PATCH 1/2 kvm/x86: rename kvm's read_tsc() as kvm_read_host_tsc()
Avoid clash with host driver's INDIRECT_CALLABLE_SCOPE read_tsc()
--- arch/x86/kvm/x86.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kvm/x86.c +2 -2
@@ -2632,7 +2632,7 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)#ifdef CONFIG_X86_64 -static u64 read_tsc (void ) +static u64 kvm_read_host_tsc (void ) {u64 ret = (u64 )rdtsc_ordered ();u64 last = pvclock_gtod_data.clock.cycle_last;
@@ -2674,7 +2674,7 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,break ;case VDSO_CLOCKMODE_TSC : *mode =VDSO_CLOCKMODE_TSC ; - *tsc_timestamp =read_tsc (); + *tsc_timestamp =kvm_read_host_tsc (); v = (*tsc_timestamp - clock->cycle_last) & clock->mask;break ;
Signature · 1 line
2.35.1.473.g83b2b277ed-goog
PATCH 2/2 timers: retpoline mitigation for time funcs
Adds indirect call exports for clock reads from tsc, apic, hrtimer, via clock-event, timekeeper & posix interfaces.
--- arch/x86/kernel/apic/apic.c | 8 +++++--- arch/x86/kernel/tsc.c | 3 ++- include/linux/hrtimer.h | 19 ++++++++++++++++--- kernel/time/clockevents.c | 9 ++++++--- kernel/time/hrtimer.c | 3 ++- kernel/time/posix-cpu-timers.c | 4 ++-- kernel/time/posix-timers.c | 3 ++- kernel/time/timekeeping.c | 2 +- 8 files changed, 36 insertions(+), 15 deletions(-)
arch/x86/kernel/apic/apic.c +5 -3
@@ -463,15 +463,17 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);/* * Program the next event, relative to now */ -static int lapic_next_event (unsigned long delta, +INDIRECT_CALLABLE_SCOPE +int lapic_next_event(unsigned long delta,struct clock_event_device *evt) {apic_write (APIC_TMICT , delta);return 0 ; } -static int lapic_next_deadline (unsigned long delta, -struct clock_event_device *evt) +INDIRECT_CALLABLE_SCOPE +int lapic_next_deadline(unsigned long delta, +struct clock_event_device *evt) {u64 tsc;
arch/x86/kernel/tsc.c +2 -1
@@ -1090,7 +1090,8 @@ static void tsc_resume(struct clocksource *cs)* checking the result of read_tsc() - cycle_last for being negative. * That works because CLOCKSOURCE_MASK(64) does not mask out any bit. */ -static u64 read_tsc (struct clocksource *cs) +INDIRECT_CALLABLE_SCOPE +u64 read_tsc(struct clocksource *cs) {return (u64 )rdtsc_ordered (); }
include/linux/hrtimer.h +16 -3
@@ -20,6 +20,7 @@#include <linux/seqlock.h> #include <linux/timer.h> #include <linux/timerqueue.h> +#include <linux/indirect_call_wrapper.h> struct hrtimer_clock_base ;struct hrtimer_cpu_base ;
@@ -297,14 +298,17 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)return ktime_to_ns (timer->node.expires); } +INDIRECT_CALLABLE_DECLARE(extern ktime_t ktime_get(void)); +static inline ktime_t hrtimer_expires_remaining (const struct hrtimer *timer) { -return ktime_sub (timer->node.expires, timer->base->get_time ()); +return ktime_sub (timer->node.expires, +INDIRECT_CALL_1 (timer->base->get_time, ktime_get)); }static inline ktime_t hrtimer_cb_get_time (struct hrtimer *timer) { -return timer->base->get_time (); +return INDIRECT_CALL_1 (timer->base->get_time, ktime_get); }static inline int hrtimer_is_hres_active (struct hrtimer *timer)
@@ -503,7 +507,9 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);static inline u64 hrtimer_forward_now (struct hrtimer *timer,ktime_t interval) { -return hrtimer_forward (timer, timer->base->get_time (), interval); +return hrtimer_forward (timer, +INDIRECT_CALL_1 (timer->base->get_time, ktime_get), + interval); }/* Precise sleep: */
@@ -536,4 +542,11 @@ int hrtimers_dead_cpu(unsigned int cpu);#define hrtimers_dead_cpu NULL#endif +struct clock_event_device ; +INDIRECT_CALLABLE_DECLARE(extern __weak u64 read_tsc(struct clocksource *cs)); +INDIRECT_CALLABLE_DECLARE(extern int thread_cpu_clock_get( +const clockid_t which_clock,struct timespec64 *tp)); +INDIRECT_CALLABLE_DECLARE(extern __weak int lapic_next_deadline( +unsigned long delta,struct clock_event_device *evt)); +#endif
kernel/time/clockevents.c +6 -3
@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) dev->retries++; clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; -if (dev->set_next_event ((unsigned long ) clc, dev) ==0 ) +if (INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev) ==0 )return 0 ;if (++i >2 ) {
@@ -284,7 +285,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) dev->retries++; clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; -if (dev->set_next_event ((unsigned long ) clc, dev) ==0 ) +if (INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev) ==0 )return 0 ; }return -ETIME ;
@@ -331,7 +333,8 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, delta =max (delta, (int64_t ) dev->min_delta_ns); clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; - rc = dev->set_next_event ((unsigned long ) clc, dev); + rc =INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev);return (rc && force) ?clockevents_program_min_delta (dev) : rc; }
kernel/time/hrtimer.c +2 -1
@@ -1241,7 +1241,8 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,remove_hrtimer (timer, base, true, force_local);if (mode &HRTIMER_MODE_REL ) - tim =ktime_add_safe (tim, base->get_time ()); + tim =ktime_add_safe (tim, +INDIRECT_CALL_1 (base->get_time, ktime_get)); tim =hrtimer_update_lowres (timer, tim, mode);
kernel/time/posix-cpu-timers.c +2 -2
@@ -1596,8 +1596,8 @@ static int thread_cpu_clock_getres(const clockid_t which_clock, {return posix_cpu_clock_getres (THREAD_CLOCK , tp); } -static int thread_cpu_clock_get (const clockid_t which_clock, -struct timespec64 *tp) +INDIRECT_CALLABLE_SCOPE +int thread_cpu_clock_get(const clockid_t which_clock, struct timespec64 *tp) {return posix_cpu_clock_get (THREAD_CLOCK , tp); }
kernel/time/posix-timers.c +2 -1
@@ -1089,7 +1089,8 @@ SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,if (!kc)return -EINVAL ; - error = kc->clock_get_timespec (which_clock, &kernel_tp); + error =INDIRECT_CALL_1 (kc->clock_get_timespec, thread_cpu_clock_get, + which_clock, &kernel_tp);if (!error &&put_timespec64 (&kernel_tp, tp)) error = -EFAULT ;
kernel/time/timekeeping.c +1 -1
@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr) {struct clocksource *clock =READ_ONCE (tkr->clock); -return clock->read (clock); +return INDIRECT_CALL_1 (clock->read, read_tsc, clock); }#ifdef CONFIG_DEBUG_TIMEKEEPING
Signature · 1 line
2.35.1.473.g83b2b277ed-goog
Re: [PATCH 2/2] timers: retpoline mitigation for time funcs
Quoted 2 lines · Pete Swain
Adds indirect call exports for clock reads from tsc, apic, hrtimer, via clock-event, timekeeper & posix interfaces.
Sure, but why?
Quoted 11 lines
Signed-off-by Pete Swain <[email protected]>--- arch/x86/kernel/apic/apic.c | 8 +++++--- arch/x86/kernel/tsc.c | 3 ++- include/linux/hrtimer.h | 19 ++++++++++++++++--- kernel/time/clockevents.c | 9 ++++++--- kernel/time/hrtimer.c | 3 ++- kernel/time/posix-cpu-timers.c | 4 ++-- kernel/time/posix-timers.c | 3 ++- kernel/time/timekeeping.c | 2 +- 8 files changed, 36 insertions(+), 15 deletions(-)
Can this please be split up properly?
arch/x86/kernel/apic/apic.c EXPORT_SYMBOL_GPL(setup_APIC_eilvt); · line 468
Quoted 12 lines
arch/x86/kernel/apic/apic.c +2
-1@@ -463,15 +463,17 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);/* * Program the next event, relative to now */ -static int lapic_next_event (unsigned long delta, +INDIRECT_CALLABLE_SCOPE +int lapic_next_event(unsigned long delta,struct clock_event_device *evt)
So this was formatted properly:
static int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)
Now it turned into garbage:
INDIRECT_CALLABLE_SCOPE
int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)
while:
INDIRECT_CALLABLE_SCOPE int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)
or
INDIRECT_CALLABLE_SCOPE
int lapic_next_event(unsigned long delta, struct clock_event_device *evt)
are both what a reader would expect.
Similar issues all over the place.
arch/x86/kernel/tsc.c static void tsc_resume(struct clocksource *cs) · line 1094
Quoted 11 lines
arch/x86/kernel/tsc.c +2
-1@@ -1090,7 +1090,8 @@ static void tsc_resume(struct clocksource *cs)* checking the result of read_tsc() - cycle_last for being negative. * That works because CLOCKSOURCE_MASK(64) does not mask out any bit. */ -static u64 read_tsc (struct clocksource *cs) +INDIRECT_CALLABLE_SCOPE +u64 read_tsc(struct clocksource *cs)
What's the extra line for?
Quoted 16 lines
arch/x86/kernel/tsc.c +0
-0{return (u64 )rdtsc_ordered (); }include/linux/hrtimer.h +1
-0@@ -20,6 +20,7 @@#include <linux/seqlock.h> #include <linux/timer.h> #include <linux/timerqueue.h> +#include <linux/indirect_call_wrapper.h> struct hrtimer_clock_base ;struct hrtimer_cpu_base ;
include/linux/hrtimer.h static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) · line 306
Quoted 10 lines
include/linux/hrtimer.h +4
-1@@ -297,14 +298,17 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)return ktime_to_ns (timer->node.expires); } +INDIRECT_CALLABLE_DECLARE(extern ktime_t ktime_get(void)); +static inline ktime_t hrtimer_expires_remaining (const struct hrtimer *timer) { -return ktime_sub (timer->node.expires, timer->base->get_time ()); +return ktime_sub (timer->node.expires, +INDIRECT_CALL_1 (timer->base->get_time, ktime_get));
This wants a proper explanation in a changelog for the hrtimer check, why this is optimized for ktime_get().
Quoted 21 lines
include/linux/hrtimer.h +4
-2}static inline ktime_t hrtimer_cb_get_time (struct hrtimer *timer) { -return timer->base->get_time (); +return INDIRECT_CALL_1 (timer->base->get_time, ktime_get); }static inline int hrtimer_is_hres_active (struct hrtimer *timer)@@ -503,7 +507,9 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);static inline u64 hrtimer_forward_now (struct hrtimer *timer,ktime_t interval) { -return hrtimer_forward (timer, timer->base->get_time (), interval); +return hrtimer_forward (timer, +INDIRECT_CALL_1 (timer->base->get_time, ktime_get), + interval); }/* Precise sleep: */
include/linux/hrtimer.h int hrtimers_dead_cpu(unsigned int cpu); · line 551
Quoted 10 lines
include/linux/hrtimer.h +7
-0@@ -536,4 +542,11 @@ int hrtimers_dead_cpu(unsigned int cpu);#define hrtimers_dead_cpu NULL#endif +struct clock_event_device ; +INDIRECT_CALLABLE_DECLARE(extern __weak u64 read_tsc(struct clocksource *cs)); +INDIRECT_CALLABLE_DECLARE(extern int thread_cpu_clock_get( +const clockid_t which_clock,struct timespec64 *tp)); +INDIRECT_CALLABLE_DECLARE(extern __weak int lapic_next_deadline( +unsigned long delta,struct clock_event_device *evt)); +
No. This is not how it works. This is a generic header and x86 specific muck has no place here.
kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249
Quoted 12 lines
include/linux/hrtimer.h +0
-0#endif kernel/time/clockevents.c +2
-1@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) dev->retries++; clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; -if (dev->set_next_event ((unsigned long ) clc, dev) ==0 ) +if (INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev) ==0 )
No. We are not sprinkling x86'isms into generic code.
Quoted 20 lines
kernel/time/posix-cpu-timers.c +2
-2@@ -1596,8 +1596,8 @@ static int thread_cpu_clock_getres(const clockid_t which_clock, {return posix_cpu_clock_getres (THREAD_CLOCK , tp); } -static int thread_cpu_clock_get (const clockid_t which_clock, -struct timespec64 *tp) +INDIRECT_CALLABLE_SCOPE +int thread_cpu_clock_get(const clockid_t which_clock, struct timespec64 *tp) {return posix_cpu_clock_get (THREAD_CLOCK , tp); }kernel/time/posix-timers.c +0
-0
kernel/time/posix-timers.c SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock, · line 1093
Quoted 6 lines
kernel/time/posix-timers.c +2
-1@@ -1089,7 +1089,8 @@ SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,if (!kc)return -EINVAL ; - error = kc->clock_get_timespec (which_clock, &kernel_tp); + error =INDIRECT_CALL_1 (kc->clock_get_timespec, thread_cpu_clock_get, + which_clock, &kernel_tp);
The argument for selecting thread_cpu_clock_get() as optimization target is?
kernel/time/timekeeping.c static inline u64 tk_clock_read(const struct tk_read_base *tkr) · line 193
Quoted 13 lines
kernel/time/posix-timers.c +0
-0if (!error &&put_timespec64 (&kernel_tp, tp)) error = -EFAULT ;kernel/time/timekeeping.c +1
-1@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr) {struct clocksource *clock =READ_ONCE (tkr->clock); -return clock->read (clock); +return INDIRECT_CALL_1 (clock->read, read_tsc, clock);
Again. No X86 muck here.
Thanks,
tglx
Re: [PATCH 2/2] timers: retpoline mitigation for time funcs
Pete,
Quoted 21 lines · Thomas Gleixner
kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249
Quoted 11 lines · Pete Swain
kernel/time/clockevents.c +2
-1@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) dev->retries++; clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; -if (dev->set_next_event ((unsigned long ) clc, dev) ==0 ) +if (INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev) ==0 )No. We are not sprinkling x86'isms into generic code.
Quoted 6 lines
kernel/time/timekeeping.c +0
-0@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr) {struct clocksource *clock =READ_ONCE (tkr->clock);
Quoted 4 lines
kernel/time/timekeeping.c static inline u64 tk_clock_read(const struct tk_read_base *tkr) · line 193
Quoted 2 lines
kernel/time/timekeeping.c +1
-1-return clock->read (clock); +return INDIRECT_CALL_1 (clock->read, read_tsc, clock);Again. No X86 muck here.
Just for clarification. I have absolutely no interest in opening this can of worms. The indirect call is in general more expensive on some architectures, so when we grant this for x86, then the next architecture comes around the corner and wants the same treatment. Guess how well that works and how maintainable this is.
And no, you can't just go there and have a
#define arch_read_favourite_clocksource read_foo
because we have multiplatform kernels where the clocksource is selected at runtime which means every platform wants to be the one defining this.
You can do that in your own frankenkernels, but for mainline this is not going to fly. You have to come up with something smarter than just taking your profiling results and slapping INDIRECT_CALL_*() all over the place. INDIRECT_CALL is a hack as it leaves the conditional around even if retpoline gets patched out.
The kernel has mechanisms to do better than that.
Let's look at tk_clock_read(). tkr->clock changes usually once maybe twice during boot. Until shutdown it might change when e.g. TSC becomes unstable and there are a few other cases like suspend/resume. But none of these events are hotpath.
While we have several instances of tk_read_base, all of them have the same clock pointer, except for the rare case of suspend/resume. It's redundant storage for various reasons.
So with some thought this can be implemented with static_call() which is currently supported by x86 and powerpc32, but there is no reason why it can't be supported by other architectures. INDIRECT_CALL is a x86'ism with a dependency on RETPOLINE, which will never gain traction outside of x86.
Thanks,
tglx
Re: [PATCH 2/2] timers: retpoline mitigation for time funcs
In reply to Thomas GleixnerQuoted 10 lines · Thomas Gleixner
kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249
Quoted 7 lines · Pete Swain
kernel/time/clockevents.c +2
-1@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) dev->retries++; clc = ((unsigned long long ) delta * dev->mult) >> dev->shift; -if (dev->set_next_event ((unsigned long ) clc, dev) ==0 ) +if (INDIRECT_CALL_1 (dev->set_next_event, lapic_next_deadline, + (unsigned long ) clc, dev) ==0 )No. We are not sprinkling x86'isms into generic code.
Even if we would do that, then this is completely useless.
The hotpath function is clockevents_program_event() and not clockevents_program_min_delta().
The latter is invoked from the former if the to be programmed event is in the past _and_ the force argument is set, i.e. in 0.1% of the invocations of clockevents_program_event(), which itself has an indirect call to dev->set_next_event().
If your profiles show clockevents_program_min_delta() as dominant, then there is something massively wrong with your kernel.
Thanks,
tglx
Quoted 4 lines · Pete Swain
Avoid clash with host driver's
INDIRECT_CALLABLE_SCOPEread_tsc()Signed-off-by Pete Swain <[email protected]>
Quoted 21 lines · Pete Swain
Avoid clash with host driver's
INDIRECT_CALLABLE_SCOPEread_tsc()Signed-off-by Pete Swain <[email protected]>--- arch/x86/kvm/x86.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)arch/x86/kvm/x86.c +1
-1@@ -2632,7 +2632,7 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)#ifdef CONFIG_X86_64 -static u64 read_tsc (void ) +static u64 kvm_read_host_tsc (void ) {u64 ret = (u64 )rdtsc_ordered ();u64 last = pvclock_gtod_data.clock.cycle_last;
arch/x86/kvm/x86.c static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, · line 2680
Quoted 8 lines
arch/x86/kvm/x86.c +1
-1@@ -2674,7 +2674,7 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,break ;case VDSO_CLOCKMODE_TSC : *mode =VDSO_CLOCKMODE_TSC ; - *tsc_timestamp =read_tsc (); + *tsc_timestamp =kvm_read_host_tsc (); v = (*tsc_timestamp - clock->cycle_last) & clock->mask;break ;
Paolo