Skip to contentSkip to conversation

kvm/x86: rename kvm's read_tsc() as kvm_read_host_tsc()

PATCH In review Pete Swain 4 participants 7 messages Feb 18, 2022 – Apr 11, 2022

Subsystems (6)
lore
Share & tools
lore

Pete Swain

author
#0 lore raw

PATCH 1/2 kvm/x86: rename kvm's read_tsc() as kvm_read_host_tsc()

Avoid clash with host driver's INDIRECT_CALLABLE_SCOPE read_tsc()

---
 arch/x86/kvm/x86.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kvm/x86.c +2 -2
index 641044db415d..0424d77cd214 100644
@@ -2632,7 +2632,7 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
 
 #ifdef CONFIG_X86_64
 
-static u64 read_tsc(void)
+static u64 kvm_read_host_tsc(void)
 {
 	u64 ret = (u64)rdtsc_ordered();
 	u64 last = pvclock_gtod_data.clock.cycle_last;
@@ -2674,7 +2674,7 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
 		break;
 	case VDSO_CLOCKMODE_TSC:
 		*mode = VDSO_CLOCKMODE_TSC;
-		*tsc_timestamp = read_tsc();
+		*tsc_timestamp = kvm_read_host_tsc();
 		v = (*tsc_timestamp - clock->cycle_last) &
 			clock->mask;
 		break;
Signature · 1 line
2.35.1.473.g83b2b277ed-goog
author
#1 lore raw

PATCH 2/2 timers: retpoline mitigation for time funcs

Adds indirect call exports for clock reads from tsc, apic, hrtimer, via clock-event, timekeeper & posix interfaces.

---
 arch/x86/kernel/apic/apic.c    |  8 +++++---
 arch/x86/kernel/tsc.c          |  3 ++-
 include/linux/hrtimer.h        | 19 ++++++++++++++++---
 kernel/time/clockevents.c      |  9 ++++++---
 kernel/time/hrtimer.c          |  3 ++-
 kernel/time/posix-cpu-timers.c |  4 ++--
 kernel/time/posix-timers.c     |  3 ++-
 kernel/time/timekeeping.c      |  2 +-
 8 files changed, 36 insertions(+), 15 deletions(-)
arch/x86/kernel/apic/apic.c +5 -3
index b70344bf6600..523a569dd35e 100644
@@ -463,15 +463,17 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
 /*
  * Program the next event, relative to now
  */
-static int lapic_next_event(unsigned long delta,
+INDIRECT_CALLABLE_SCOPE
+int lapic_next_event(unsigned long delta,
 			    struct clock_event_device *evt)
 {
 	apic_write(APIC_TMICT, delta);
 	return 0;
 }
 
-static int lapic_next_deadline(unsigned long delta,
-			       struct clock_event_device *evt)
+INDIRECT_CALLABLE_SCOPE
+int lapic_next_deadline(unsigned long delta,
+		       struct clock_event_device *evt)
 {
 	u64 tsc;
 
arch/x86/kernel/tsc.c +2 -1
index a698196377be..ff2868d5ddea 100644
@@ -1090,7 +1090,8 @@ static void tsc_resume(struct clocksource *cs)
  * checking the result of read_tsc() - cycle_last for being negative.
  * That works because CLOCKSOURCE_MASK(64) does not mask out any bit.
  */
-static u64 read_tsc(struct clocksource *cs)
+INDIRECT_CALLABLE_SCOPE
+u64 read_tsc(struct clocksource *cs)
 {
 	return (u64)rdtsc_ordered();
 }
include/linux/hrtimer.h +16 -3
index 0ee140176f10..9d2d110f0b8c 100644
@@ -20,6 +20,7 @@
 #include <linux/seqlock.h>
 #include <linux/timer.h>
 #include <linux/timerqueue.h>
+#include <linux/indirect_call_wrapper.h>
 
 struct hrtimer_clock_base;
 struct hrtimer_cpu_base;
@@ -297,14 +298,17 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
 	return ktime_to_ns(timer->node.expires);
 }
 
+INDIRECT_CALLABLE_DECLARE(extern ktime_t ktime_get(void));
+
 static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
 {
-	return ktime_sub(timer->node.expires, timer->base->get_time());
+	return ktime_sub(timer->node.expires,
+			INDIRECT_CALL_1(timer->base->get_time, ktime_get));
 }
 
 static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
 {
-	return timer->base->get_time();
+	return INDIRECT_CALL_1(timer->base->get_time, ktime_get);
 }
 
 static inline int hrtimer_is_hres_active(struct hrtimer *timer)
@@ -503,7 +507,9 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
 static inline u64 hrtimer_forward_now(struct hrtimer *timer,
 				      ktime_t interval)
 {
-	return hrtimer_forward(timer, timer->base->get_time(), interval);
+	return hrtimer_forward(timer,
+			INDIRECT_CALL_1(timer->base->get_time, ktime_get),
+			interval);
 }
 
 /* Precise sleep: */
@@ -536,4 +542,11 @@ int hrtimers_dead_cpu(unsigned int cpu);
 #define hrtimers_dead_cpu	NULL
 #endif
 
+struct clock_event_device;
+INDIRECT_CALLABLE_DECLARE(extern __weak u64 read_tsc(struct clocksource *cs));
+INDIRECT_CALLABLE_DECLARE(extern int thread_cpu_clock_get(
+		const clockid_t which_clock, struct timespec64 *tp));
+INDIRECT_CALLABLE_DECLARE(extern __weak int lapic_next_deadline(
+		unsigned long delta, struct clock_event_device *evt));
+
 #endif
kernel/time/clockevents.c +6 -3
index 003ccf338d20..ac15412e87c4 100644
@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 		dev->retries++;
 		clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-		if (dev->set_next_event((unsigned long) clc, dev) == 0)
+		if (INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+				  (unsigned long) clc, dev) == 0)
 			return 0;
 
 		if (++i > 2) {
@@ -284,7 +285,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 		dev->retries++;
 		clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-		if (dev->set_next_event((unsigned long) clc, dev) == 0)
+		if (INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+				  (unsigned long) clc, dev) == 0)
 			return 0;
 	}
 	return -ETIME;
@@ -331,7 +333,8 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
 	delta = max(delta, (int64_t) dev->min_delta_ns);
 
 	clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-	rc = dev->set_next_event((unsigned long) clc, dev);
+	rc = INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+			   (unsigned long) clc, dev);
 
 	return (rc && force) ? clockevents_program_min_delta(dev) : rc;
 }
kernel/time/hrtimer.c +2 -1
index 0ea8702eb516..e1e17fdfcd18 100644
@@ -1241,7 +1241,8 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
 	remove_hrtimer(timer, base, true, force_local);
 
 	if (mode & HRTIMER_MODE_REL)
-		tim = ktime_add_safe(tim, base->get_time());
+		tim = ktime_add_safe(tim,
+			INDIRECT_CALL_1(base->get_time, ktime_get));
 
 	tim = hrtimer_update_lowres(timer, tim, mode);
 
kernel/time/posix-cpu-timers.c +2 -2
index 96b4e7810426..d8bf325fa84e 100644
@@ -1596,8 +1596,8 @@ static int thread_cpu_clock_getres(const clockid_t which_clock,
 {
 	return posix_cpu_clock_getres(THREAD_CLOCK, tp);
 }
-static int thread_cpu_clock_get(const clockid_t which_clock,
-				struct timespec64 *tp)
+INDIRECT_CALLABLE_SCOPE
+int thread_cpu_clock_get(const clockid_t which_clock, struct timespec64 *tp)
 {
 	return posix_cpu_clock_get(THREAD_CLOCK, tp);
 }
kernel/time/posix-timers.c +2 -1
index 1cd10b102c51..35eac10ee796 100644
@@ -1089,7 +1089,8 @@ SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,
 	if (!kc)
 		return -EINVAL;
 
-	error = kc->clock_get_timespec(which_clock, &kernel_tp);
+	error = INDIRECT_CALL_1(kc->clock_get_timespec, thread_cpu_clock_get,
+				which_clock, &kernel_tp);
 
 	if (!error && put_timespec64(&kernel_tp, tp))
 		error = -EFAULT;
kernel/time/timekeeping.c +1 -1
index dcdcb85121e4..2b1a3b146614 100644
@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr)
 {
 	struct clocksource *clock = READ_ONCE(tkr->clock);
 
-	return clock->read(clock);
+	return INDIRECT_CALL_1(clock->read, read_tsc, clock);
 }
 
 #ifdef CONFIG_DEBUG_TIMEKEEPING
Signature · 1 line
2.35.1.473.g83b2b277ed-goog
#2 lore raw

Re: [PATCH 2/2] timers: retpoline mitigation for time funcs

Quoted 2 lines · Pete Swain

Adds indirect call exports for clock reads from tsc, apic, hrtimer, via clock-event, timekeeper & posix interfaces.

Sure, but why?

Quoted 11 lines
---
 arch/x86/kernel/apic/apic.c    |  8 +++++---
 arch/x86/kernel/tsc.c          |  3 ++-
 include/linux/hrtimer.h        | 19 ++++++++++++++++---
 kernel/time/clockevents.c      |  9 ++++++---
 kernel/time/hrtimer.c          |  3 ++-
 kernel/time/posix-cpu-timers.c |  4 ++--
 kernel/time/posix-timers.c     |  3 ++-
 kernel/time/timekeeping.c      |  2 +-
 8 files changed, 36 insertions(+), 15 deletions(-)

Can this please be split up properly?

arch/x86/kernel/apic/apic.c EXPORT_SYMBOL_GPL(setup_APIC_eilvt); · line 468

Quoted 12 lines
arch/x86/kernel/apic/apic.c +2 -1
index b70344bf6600..523a569dd35e 100644
@@ -463,15 +463,17 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
 /*
  * Program the next event, relative to now
  */
-static int lapic_next_event(unsigned long delta,
+INDIRECT_CALLABLE_SCOPE
+int lapic_next_event(unsigned long delta,
 			    struct clock_event_device *evt)

So this was formatted properly:

static int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)

Now it turned into garbage:

INDIRECT_CALLABLE_SCOPE
int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)

while:

INDIRECT_CALLABLE_SCOPE int lapic_next_event(unsigned long delta,
struct clock_event_device *evt)
or

INDIRECT_CALLABLE_SCOPE
int lapic_next_event(unsigned long delta, struct clock_event_device *evt)

are both what a reader would expect.

Similar issues all over the place.

arch/x86/kernel/tsc.c static void tsc_resume(struct clocksource *cs) · line 1094

Quoted 11 lines
arch/x86/kernel/tsc.c +2 -1
index a698196377be..ff2868d5ddea 100644
@@ -1090,7 +1090,8 @@ static void tsc_resume(struct clocksource *cs)
  * checking the result of read_tsc() - cycle_last for being negative.
  * That works because CLOCKSOURCE_MASK(64) does not mask out any bit.
  */
-static u64 read_tsc(struct clocksource *cs)
+INDIRECT_CALLABLE_SCOPE
+u64 read_tsc(struct clocksource *cs)

What's the extra line for?

Quoted 16 lines
arch/x86/kernel/tsc.c +0 -0
 {
 	return (u64)rdtsc_ordered();
 }
include/linux/hrtimer.h +1 -0
index 0ee140176f10..9d2d110f0b8c 100644
@@ -20,6 +20,7 @@
 #include <linux/seqlock.h>
 #include <linux/timer.h>
 #include <linux/timerqueue.h>
+#include <linux/indirect_call_wrapper.h>
 
 struct hrtimer_clock_base;
 struct hrtimer_cpu_base;

include/linux/hrtimer.h static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) · line 306

Quoted 10 lines
include/linux/hrtimer.h +4 -1
@@ -297,14 +298,17 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
 	return ktime_to_ns(timer->node.expires);
 }
 
+INDIRECT_CALLABLE_DECLARE(extern ktime_t ktime_get(void));
+
 static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
 {
-	return ktime_sub(timer->node.expires, timer->base->get_time());
+	return ktime_sub(timer->node.expires,
+			INDIRECT_CALL_1(timer->base->get_time, ktime_get));

This wants a proper explanation in a changelog for the hrtimer check, why this is optimized for ktime_get().

Quoted 21 lines
include/linux/hrtimer.h +4 -2
 }
 
 static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
 {
-	return timer->base->get_time();
+	return INDIRECT_CALL_1(timer->base->get_time, ktime_get);
 }
 
 static inline int hrtimer_is_hres_active(struct hrtimer *timer)
@@ -503,7 +507,9 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
 static inline u64 hrtimer_forward_now(struct hrtimer *timer,
 				      ktime_t interval)
 {
-	return hrtimer_forward(timer, timer->base->get_time(), interval);
+	return hrtimer_forward(timer,
+			INDIRECT_CALL_1(timer->base->get_time, ktime_get),
+			interval);
 }
 
 /* Precise sleep: */

include/linux/hrtimer.h int hrtimers_dead_cpu(unsigned int cpu); · line 551

Quoted 10 lines
include/linux/hrtimer.h +7 -0
@@ -536,4 +542,11 @@ int hrtimers_dead_cpu(unsigned int cpu);
 #define hrtimers_dead_cpu	NULL
 #endif
 
+struct clock_event_device;
+INDIRECT_CALLABLE_DECLARE(extern __weak u64 read_tsc(struct clocksource *cs));
+INDIRECT_CALLABLE_DECLARE(extern int thread_cpu_clock_get(
+		const clockid_t which_clock, struct timespec64 *tp));
+INDIRECT_CALLABLE_DECLARE(extern __weak int lapic_next_deadline(
+		unsigned long delta, struct clock_event_device *evt));
+

No. This is not how it works. This is a generic header and x86 specific muck has no place here.

kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249

Quoted 12 lines
include/linux/hrtimer.h +0 -0
 #endif
kernel/time/clockevents.c +2 -1
index 003ccf338d20..ac15412e87c4 100644
@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 		dev->retries++;
 		clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-		if (dev->set_next_event((unsigned long) clc, dev) == 0)
+		if (INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+				  (unsigned long) clc, dev) == 0)

No. We are not sprinkling x86'isms into generic code.

Quoted 20 lines
kernel/time/posix-cpu-timers.c +2 -2
index 96b4e7810426..d8bf325fa84e 100644
@@ -1596,8 +1596,8 @@ static int thread_cpu_clock_getres(const clockid_t which_clock,
 {
 	return posix_cpu_clock_getres(THREAD_CLOCK, tp);
 }
-static int thread_cpu_clock_get(const clockid_t which_clock,
-				struct timespec64 *tp)
+INDIRECT_CALLABLE_SCOPE
+int thread_cpu_clock_get(const clockid_t which_clock, struct timespec64 *tp)
 {
 	return posix_cpu_clock_get(THREAD_CLOCK, tp);
 }
kernel/time/posix-timers.c +0 -0
index 1cd10b102c51..35eac10ee796 100644

kernel/time/posix-timers.c SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock, · line 1093

Quoted 6 lines
kernel/time/posix-timers.c +2 -1
@@ -1089,7 +1089,8 @@ SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,
 	if (!kc)
 		return -EINVAL;
 
-	error = kc->clock_get_timespec(which_clock, &kernel_tp);
+	error = INDIRECT_CALL_1(kc->clock_get_timespec, thread_cpu_clock_get,
+				which_clock, &kernel_tp);

The argument for selecting thread_cpu_clock_get() as optimization target is?

kernel/time/timekeeping.c static inline u64 tk_clock_read(const struct tk_read_base *tkr) · line 193

Quoted 13 lines
kernel/time/posix-timers.c +0 -0
 	if (!error && put_timespec64(&kernel_tp, tp))
 		error = -EFAULT;
kernel/time/timekeeping.c +1 -1
index dcdcb85121e4..2b1a3b146614 100644
@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr)
 {
 	struct clocksource *clock = READ_ONCE(tkr->clock);
 
-	return clock->read(clock);
+	return INDIRECT_CALL_1(clock->read, read_tsc, clock);

Again. No X86 muck here.

Thanks,

tglx

#3 lore raw

Re: [PATCH 2/2] timers: retpoline mitigation for time funcs

Pete,

Quoted 21 lines · Thomas Gleixner

kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249

Quoted 11 lines · Pete Swain
kernel/time/clockevents.c +2 -1
index 003ccf338d20..ac15412e87c4 100644
@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 		dev->retries++;
 		clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-		if (dev->set_next_event((unsigned long) clc, dev) == 0)
+		if (INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+				  (unsigned long) clc, dev) == 0)

No. We are not sprinkling x86'isms into generic code.

Quoted 6 lines
kernel/time/timekeeping.c +0 -0
@@ -190,7 +190,7 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr)
 {
 	struct clocksource *clock = READ_ONCE(tkr->clock);
 
Quoted 4 lines

kernel/time/timekeeping.c static inline u64 tk_clock_read(const struct tk_read_base *tkr) · line 193

Quoted 2 lines
kernel/time/timekeeping.c +1 -1
-	return clock->read(clock);
+	return INDIRECT_CALL_1(clock->read, read_tsc, clock);

Again. No X86 muck here.

Just for clarification. I have absolutely no interest in opening this can of worms. The indirect call is in general more expensive on some architectures, so when we grant this for x86, then the next architecture comes around the corner and wants the same treatment. Guess how well that works and how maintainable this is.

And no, you can't just go there and have a

#define arch_read_favourite_clocksource read_foo

because we have multiplatform kernels where the clocksource is selected at runtime which means every platform wants to be the one defining this.

You can do that in your own frankenkernels, but for mainline this is not going to fly. You have to come up with something smarter than just taking your profiling results and slapping INDIRECT_CALL_*() all over the place. INDIRECT_CALL is a hack as it leaves the conditional around even if retpoline gets patched out.

The kernel has mechanisms to do better than that.

Let's look at tk_clock_read(). tkr->clock changes usually once maybe twice during boot. Until shutdown it might change when e.g. TSC becomes unstable and there are a few other cases like suspend/resume. But none of these events are hotpath.

While we have several instances of tk_read_base, all of them have the same clock pointer, except for the rare case of suspend/resume. It's redundant storage for various reasons.

So with some thought this can be implemented with static_call() which is currently supported by x86 and powerpc32, but there is no reason why it can't be supported by other architectures. INDIRECT_CALL is a x86'ism with a dependency on RETPOLINE, which will never gain traction outside of x86.

Thanks,

tglx

#4 lore raw

Re: [PATCH 2/2] timers: retpoline mitigation for time funcs

In reply to Thomas Gleixner
Quoted 10 lines · Thomas Gleixner

kernel/time/clockevents.c static int clockevents_program_min_delta(struct clock_event_device *dev) · line 249

Quoted 7 lines · Pete Swain
kernel/time/clockevents.c +2 -1
@@ -245,7 +245,8 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 		dev->retries++;
 		clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-		if (dev->set_next_event((unsigned long) clc, dev) == 0)
+		if (INDIRECT_CALL_1(dev->set_next_event, lapic_next_deadline,
+				  (unsigned long) clc, dev) == 0)

No. We are not sprinkling x86'isms into generic code.

Even if we would do that, then this is completely useless.

The hotpath function is clockevents_program_event() and not clockevents_program_min_delta().

The latter is invoked from the former if the to be programmed event is in the past _and_ the force argument is set, i.e. in 0.1% of the invocations of clockevents_program_event(), which itself has an indirect call to dev->set_next_event().

If your profiles show clockevents_program_min_delta() as dominant, then there is something massively wrong with your kernel.

Thanks,

tglx

maintainer Acked-by on v1 1/2
#6 lore raw
In reply to Pete Swain
Quoted 21 lines · Pete Swain

Avoid clash with host driver's INDIRECT_CALLABLE_SCOPE read_tsc()

---
  arch/x86/kvm/x86.c | 4 ++--
  1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kvm/x86.c +1 -1
index 641044db415d..0424d77cd214 100644
@@ -2632,7 +2632,7 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
 
  #ifdef CONFIG_X86_64
 
-static u64 read_tsc(void)
+static u64 kvm_read_host_tsc(void)
  {
  	u64 ret = (u64)rdtsc_ordered();
  	u64 last = pvclock_gtod_data.clock.cycle_last;

arch/x86/kvm/x86.c static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, · line 2680

Quoted 8 lines
arch/x86/kvm/x86.c +1 -1
@@ -2674,7 +2674,7 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
  		break;
  	case VDSO_CLOCKMODE_TSC:
  		*mode = VDSO_CLOCKMODE_TSC;
-		*tsc_timestamp = read_tsc();
+		*tsc_timestamp = kvm_read_host_tsc();
  		v = (*tsc_timestamp - clock->cycle_last) &
  			clock->mask;
  		break;

Paolo