Skip to contentSkip to conversation

lib/vdso: Use __arch_use_vsyscall() to indicate fallback

PATCH In review Huacai Chen 4 participants 7 messages Oct 18, 2019 – Oct 21, 2019

Subsystems (6)
Stable branch x86 MIPS arm64 +2 more subsystems
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 lib/vdso: Use __arch_use_vsyscall() to indicate fallback

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

To solve this problem, we use U64_MAX as the only "invalid" return value -- this is still not fully correct, but has no problem in most cases. Moreover, all vdso time-related functions should rely on the return value of __arch_use_vsyscall(), because update_vdso_data() and update_vsyscall_tz() also rely on it. So, in the core functions of __cvdso_gettimeofday(), __cvdso_clock_gettime() and __cvdso_clock_ getres(), if __arch_use_vsyscall() returns false, we use the fallback functions directly.

Fixes 00b26474c2f1613d7ab894c5 ("lib/vdso: Provide generic VDSO implementation")
Cc Arnd Bergmann <[email protected]>
Cc Paul Burton <[email protected]>
---
 arch/arm64/include/asm/vdso/vsyscall.h |  2 +-
 arch/mips/include/asm/vdso/vsyscall.h  |  2 +-
 include/asm-generic/vdso/vsyscall.h    |  2 +-
 lib/vdso/gettimeofday.c                | 12 +++++++++++-
 4 files changed, 14 insertions(+), 4 deletions(-)
arch/arm64/include/asm/vdso/vsyscall.h +1 -1
index 0c731bf..406e6de 100644
@@ -31,7 +31,7 @@ int __arm64_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __arm64_get_clock_mode
 
 static __always_inline
-int __arm64_use_vsyscall(struct vdso_data *vdata)
+int __arm64_use_vsyscall(const struct vdso_data *vdata)
 {
 	return !vdata[CS_HRES_COARSE].clock_mode;
 }
arch/mips/include/asm/vdso/vsyscall.h +1 -1
index 1953147..8b10dd7 100644
@@ -29,7 +29,7 @@ int __mips_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __mips_get_clock_mode
 
 static __always_inline
-int __mips_use_vsyscall(struct vdso_data *vdata)
+int __mips_use_vsyscall(const struct vdso_data *vdata)
 {
 	return (vdata[CS_HRES_COARSE].clock_mode != VDSO_CLOCK_NONE);
 }
include/asm-generic/vdso/vsyscall.h +1 -1
index e94b1978..ac05a625 100644
@@ -26,7 +26,7 @@ static __always_inline int __arch_get_clock_mode(struct timekeeper *tk)
 #endif /* __arch_get_clock_mode */
 
 #ifndef __arch_use_vsyscall
-static __always_inline int __arch_use_vsyscall(struct vdso_data *vdata)
+static __always_inline int __arch_use_vsyscall(const struct vdso_data *vdata)
 {
 	return 1;
 }
lib/vdso/gettimeofday.c +11 -1
index e630e7f..4ad062e 100644
@@ -9,6 +9,7 @@
 #include <linux/hrtimer_defs.h>
 #include <vdso/datapage.h>
 #include <vdso/helpers.h>
+#include <vdso/vsyscall.h>
 
 /*
  * The generic vDSO implementation requires that gettimeofday.h
@@ -50,7 +51,7 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk,
 		cycles = __arch_get_hw_counter(vd->clock_mode);
 		ns = vdso_ts->nsec;
 		last = vd->cycle_last;
-		if (unlikely((s64)cycles < 0))
+		if (unlikely(cycles == U64_MAX))
 			return -1;
 
 		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
@@ -91,6 +92,9 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
 	if (unlikely((u32) clock >= MAX_CLOCKS))
 		return -1;
 
+	if (!__arch_use_vsyscall(vd))
+		return -1;
+
 	/*
 	 * Convert the clockid to a bitmask and use it to check which
 	 * clocks are handled in the VDSO directly.
@@ -145,6 +149,9 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
 	const struct vdso_data *vd = __arch_get_vdso_data();
 
+	if (!__arch_use_vsyscall(vd))
+		return gettimeofday_fallback(tv, tz);
+
 	if (likely(tv != NULL)) {
 		struct __kernel_timespec ts;
 
@@ -189,6 +196,9 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
 	if (unlikely((u32) clock >= MAX_CLOCKS))
 		return -1;
 
+	if (!__arch_use_vsyscall(vd))
+		return -1;
+
 	hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 	/*
 	 * Convert the clockid to a bitmask and use it to check which
Signature · 7 lines
2.7.0


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
maintainer
#1 lore raw
Quoted 7 lines · Huacai Chen

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

s/negtive/negative

What's the actual bug? Is it that MIPS is returning 0 but the check is < 0? Sounds like MIPS should get fixed.

Quoted 4 lines

To solve this problem, we use U64_MAX as the only "invalid" return value -- this is still not fully correct, but has no problem in most cases.

I'm sort of okay with that, but...

Quoted 6 lines

Moreover, all vdso time-related functions should rely on the return value of __arch_use_vsyscall(), because update_vdso_data() and update_vsyscall_tz() also rely on it. So, in the core functions of __cvdso_gettimeofday(), __cvdso_clock_gettime() and __cvdso_clock_ getres(), if __arch_use_vsyscall() returns false, we use the fallback functions directly.

__arch_use_vsyscall() is not currently intended for use in the vDSO at all.

Quoted 67 lines
Fixes 00b26474c2f1613d7ab894c5 ("lib/vdso: Provide generic VDSO implementation")
Cc Arnd Bergmann <[email protected]>
Cc Paul Burton <[email protected]>
---
 arch/arm64/include/asm/vdso/vsyscall.h |  2 +-
 arch/mips/include/asm/vdso/vsyscall.h  |  2 +-
 include/asm-generic/vdso/vsyscall.h    |  2 +-
 lib/vdso/gettimeofday.c                | 12 +++++++++++-
 4 files changed, 14 insertions(+), 4 deletions(-)
arch/arm64/include/asm/vdso/vsyscall.h +1 -1
index 0c731bf..406e6de 100644
@@ -31,7 +31,7 @@ int __arm64_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __arm64_get_clock_mode
 
 static __always_inline
-int __arm64_use_vsyscall(struct vdso_data *vdata)
+int __arm64_use_vsyscall(const struct vdso_data *vdata)
 {
        return !vdata[CS_HRES_COARSE].clock_mode;
 }
arch/mips/include/asm/vdso/vsyscall.h +1 -1
index 1953147..8b10dd7 100644
@@ -29,7 +29,7 @@ int __mips_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __mips_get_clock_mode
 
 static __always_inline
-int __mips_use_vsyscall(struct vdso_data *vdata)
+int __mips_use_vsyscall(const struct vdso_data *vdata)
 {
        return (vdata[CS_HRES_COARSE].clock_mode != VDSO_CLOCK_NONE);
 }
include/asm-generic/vdso/vsyscall.h +1 -1
index e94b1978..ac05a625 100644
@@ -26,7 +26,7 @@ static __always_inline int __arch_get_clock_mode(struct timekeeper *tk)
 #endif /* __arch_get_clock_mode */
 
 #ifndef __arch_use_vsyscall
-static __always_inline int __arch_use_vsyscall(struct vdso_data *vdata)
+static __always_inline int __arch_use_vsyscall(const struct vdso_data *vdata)
 {
        return 1;
 }
lib/vdso/gettimeofday.c +1 -0
index e630e7f..4ad062e 100644
@@ -9,6 +9,7 @@
 #include <linux/hrtimer_defs.h>
 #include <vdso/datapage.h>
 #include <vdso/helpers.h>
+#include <vdso/vsyscall.h>
 
 /*
  * The generic vDSO implementation requires that gettimeofday.h

lib/vdso/gettimeofday.c static int do_hres(const struct vdso_data *vd, clockid_t clk, · line 55

Quoted 6 lines
lib/vdso/gettimeofday.c +1 -1
@@ -50,7 +51,7 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk,
                cycles = __arch_get_hw_counter(vd->clock_mode);
                ns = vdso_ts->nsec;
                last = vd->cycle_last;
-               if (unlikely((s64)cycles < 0))
+               if (unlikely(cycles == U64_MAX))
                        return -1;

I would actually prefer:

if (unlikely(cycles < last))

or perhaps:

if (unlikely((s64)(cycles-last) < 0))

which would have the nice side effect of getting rid of the annoying x86 special case in vdso_calc_delta(). The former version is compatible with U64_MAX, whereas the latter version would need the error case to return last-1 or similar. The benefit of the latter version is that it can survive wrap-around.

lib/vdso/gettimeofday.c __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) · line 97

Quoted 9 lines
lib/vdso/gettimeofday.c +3 -0
                ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
@@ -91,6 +92,9 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;
 
+       if (!__arch_use_vsyscall(vd))
+               return -1;
+

NAK. I don't think this is helpful or correct. It doesn't appear to do anything valid, and it's racy.

lib/vdso/gettimeofday.c __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) · line 154

Quoted 10 lines
lib/vdso/gettimeofday.c +3 -0
        /*
         * Convert the clockid to a bitmask and use it to check which
         * clocks are handled in the VDSO directly.
@@ -145,6 +149,9 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
        const struct vdso_data *vd = __arch_get_vdso_data();
 
+       if (!__arch_use_vsyscall(vd))
+               return gettimeofday_fallback(tv, tz);
+

Ditto.

lib/vdso/gettimeofday.c int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) · line 201

Quoted 10 lines
lib/vdso/gettimeofday.c +3 -0
        if (likely(tv != NULL)) {
                struct __kernel_timespec ts;
 
@@ -189,6 +196,9 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;
 
+       if (!__arch_use_vsyscall(vd))
+               return -1;
+

Ditto.

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#2 lore raw

Hi Andy and Hucan,

Quoted 6 lines · Andy Lutomirski
Quoted 5 lines · Huacai Chen

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

Quoted 8 lines
Quoted 2 lines

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

s/negtive/negative

What's the actual bug? Is it that MIPS is returning 0 but the check is < 0? Sounds like MIPS should get fixed.

I submitted a patch for this yesterday to the MIPS maintainers [1]. The MIPS32 r1 implementation had a bug when VDSO_CLOCK_NONE was set.

The issue has been reported by Maxime Bizon who tested the fix as well.

[1] https://patchwork.kernel.org/patch/11193391/

Signature · 7 lines
Regards,
Vincenzo

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
#3 lore raw
In reply to Andy Lutomirski

Hi, Andy,

Quoted 7 lines · Andy Lutomirski
Quoted 5 lines · Huacai Chen

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

Quoted 7 lines
Quoted 2 lines

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

s/negtive/negative

What's the actual bug? Is it that MIPS is returning 0 but the check is < 0? Sounds like MIPS should get fixed.

My original bug is what Vincenzo said, MIPS has a boot failure if no valid clock_mode, and surely MIPS need to fix. However, when I try to fix it, I found that clock_getres() has another problem, because __cvdso_clock_getres_common() get vd[CS_HRES_COARSE].hrtimer_res, but hrtimer_res is set in update_vdso_data() which relies on __arch_use_vsyscall().

Quoted 97 lines
Quoted 5 lines

To solve this problem, we use U64_MAX as the only "invalid" return value -- this is still not fully correct, but has no problem in most cases.

I'm sort of okay with that, but...

Quoted 6 lines

Moreover, all vdso time-related functions should rely on the return value of __arch_use_vsyscall(), because update_vdso_data() and update_vsyscall_tz() also rely on it. So, in the core functions of __cvdso_gettimeofday(), __cvdso_clock_gettime() and __cvdso_clock_ getres(), if __arch_use_vsyscall() returns false, we use the fallback functions directly.

__arch_use_vsyscall() is not currently intended for use in the vDSO at all.

Quoted 67 lines
Fixes 00b26474c2f1613d7ab894c5 ("lib/vdso: Provide generic VDSO implementation")
Cc Arnd Bergmann <[email protected]>
Cc Paul Burton <[email protected]>
---
 arch/arm64/include/asm/vdso/vsyscall.h |  2 +-
 arch/mips/include/asm/vdso/vsyscall.h  |  2 +-
 include/asm-generic/vdso/vsyscall.h    |  2 +-
 lib/vdso/gettimeofday.c                | 12 +++++++++++-
 4 files changed, 14 insertions(+), 4 deletions(-)
arch/arm64/include/asm/vdso/vsyscall.h +1 -1
index 0c731bf..406e6de 100644
@@ -31,7 +31,7 @@ int __arm64_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __arm64_get_clock_mode
 
 static __always_inline
-int __arm64_use_vsyscall(struct vdso_data *vdata)
+int __arm64_use_vsyscall(const struct vdso_data *vdata)
 {
        return !vdata[CS_HRES_COARSE].clock_mode;
 }
arch/mips/include/asm/vdso/vsyscall.h +1 -1
index 1953147..8b10dd7 100644
@@ -29,7 +29,7 @@ int __mips_get_clock_mode(struct timekeeper *tk)
 #define __arch_get_clock_mode __mips_get_clock_mode
 
 static __always_inline
-int __mips_use_vsyscall(struct vdso_data *vdata)
+int __mips_use_vsyscall(const struct vdso_data *vdata)
 {
        return (vdata[CS_HRES_COARSE].clock_mode != VDSO_CLOCK_NONE);
 }
include/asm-generic/vdso/vsyscall.h +1 -1
index e94b1978..ac05a625 100644
@@ -26,7 +26,7 @@ static __always_inline int __arch_get_clock_mode(struct timekeeper *tk)
 #endif /* __arch_get_clock_mode */
 
 #ifndef __arch_use_vsyscall
-static __always_inline int __arch_use_vsyscall(struct vdso_data *vdata)
+static __always_inline int __arch_use_vsyscall(const struct vdso_data *vdata)
 {
        return 1;
 }
lib/vdso/gettimeofday.c +1 -0
index e630e7f..4ad062e 100644
@@ -9,6 +9,7 @@
 #include <linux/hrtimer_defs.h>
 #include <vdso/datapage.h>
 #include <vdso/helpers.h>
+#include <vdso/vsyscall.h>
 
 /*
  * The generic vDSO implementation requires that gettimeofday.h

lib/vdso/gettimeofday.c static int do_hres(const struct vdso_data *vd, clockid_t clk, · line 55

Quoted 6 lines
lib/vdso/gettimeofday.c +1 -1
@@ -50,7 +51,7 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk,
                cycles = __arch_get_hw_counter(vd->clock_mode);
                ns = vdso_ts->nsec;
                last = vd->cycle_last;
-               if (unlikely((s64)cycles < 0))
+               if (unlikely(cycles == U64_MAX))
                        return -1;

I would actually prefer:

if (unlikely(cycles < last))

or perhaps:

Quoted 7 lines

if (unlikely((s64)(cycles-last) < 0))

which would have the nice side effect of getting rid of the annoying x86 special case in vdso_calc_delta(). The former version is compatible with U64_MAX, whereas the latter version would need the error case to return last-1 or similar. The benefit of the latter version is that it can survive wrap-around.

When you say if (unlikely(cycles < last)), do you means if (unlikely(cycles <= last))? If __arch_get_hw_counter() return U64_MAX every time, I don't think cycles can be less than last.

Huacai

Quoted 39 lines

lib/vdso/gettimeofday.c __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) · line 97

Quoted 10 lines
lib/vdso/gettimeofday.c +3 -0
                ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
@@ -91,6 +92,9 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;
 
+       if (!__arch_use_vsyscall(vd))
+               return -1;
+

NAK. I don't think this is helpful or correct. It doesn't appear to do anything valid, and it's racy.

lib/vdso/gettimeofday.c __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) · line 154

Quoted 10 lines
lib/vdso/gettimeofday.c +3 -0
        /*
         * Convert the clockid to a bitmask and use it to check which
         * clocks are handled in the VDSO directly.
@@ -145,6 +149,9 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
        const struct vdso_data *vd = __arch_get_vdso_data();
 
+       if (!__arch_use_vsyscall(vd))
+               return gettimeofday_fallback(tv, tz);
+

Ditto.

lib/vdso/gettimeofday.c int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) · line 201

Quoted 10 lines
lib/vdso/gettimeofday.c +3 -0
        if (likely(tv != NULL)) {
                struct __kernel_timespec ts;
 
@@ -189,6 +196,9 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
        if (unlikely((u32) clock >= MAX_CLOCKS))
                return -1;
 
+       if (!__arch_use_vsyscall(vd))
+               return -1;
+

Ditto.

#4 lore raw
Quoted 13 lines · Huacai Chen
Quoted 12 lines · Andy Lutomirski
Quoted 7 lines · Huacai Chen

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

s/negtive/negative

Quoted 8 lines
Quoted 2 lines

What's the actual bug? Is it that MIPS is returning 0 but the check is < 0? Sounds like MIPS should get fixed.

My original bug is what Vincenzo said, MIPS has a boot failure if no valid clock_mode, and surely MIPS need to fix. However, when I try to fix it, I found that clock_getres() has another problem, because __cvdso_clock_getres_common() get vd[CS_HRES_COARSE].hrtimer_res, but hrtimer_res is set in update_vdso_data() which relies on __arch_use_vsyscall().

__arch_use_vsyscall() is a pointless exercise TBH. The VDSO data should be updated unconditionally so all the trivial interfaces like time() and getres() just work independently of the functions which depend on the underlying clocksource.

This functions have a fallback operation already:

Let __arch_get_hw_counter() return U64_MAX and the syscall fallback is invoked.

__arch_use_vsyscall() should just be removed.

Thanks,

tglx

maintainer
#5 lore raw
Quoted 29 lines · Thomas Gleixner
Quoted 13 lines · Huacai Chen
Quoted 12 lines · Andy Lutomirski
Quoted 7 lines · Huacai Chen

In do_hres(), we currently use whether the return value of __arch_get_ hw_counter() is negtive to indicate fallback, but this is not a good idea. Because:

1, ARM64 returns ULL_MAX but MIPS returns 0 when clock_mode is invalid; 2, For a 64bit counter, a "negtive" value of counter is actually valid.

s/negtive/negative

Quoted 8 lines
Quoted 2 lines

What's the actual bug? Is it that MIPS is returning 0 but the check is < 0? Sounds like MIPS should get fixed.

My original bug is what Vincenzo said, MIPS has a boot failure if no valid clock_mode, and surely MIPS need to fix. However, when I try to fix it, I found that clock_getres() has another problem, because __cvdso_clock_getres_common() get vd[CS_HRES_COARSE].hrtimer_res, but hrtimer_res is set in update_vdso_data() which relies on __arch_use_vsyscall().

__arch_use_vsyscall() is a pointless exercise TBH. The VDSO data should be updated unconditionally so all the trivial interfaces like time() and getres() just work independently of the functions which depend on the underlying clocksource.

Quoted 5 lines

This functions have a fallback operation already:

Let __arch_get_hw_counter() return U64_MAX and the syscall fallback is invoked.

My thought was that __arch_get_hw_counter() could return last-1 to indicate failure, which would allow the two checks to be folded into one check. Or we could continue to use U64_MAX and rely on the fact that (s64)U64_MAX < 0, not worry about the cycle counter overflowing, and letting cycles < last catch it.

(And we should change it to return s64 at some point regardless -- all the math is signed, so the underlying types should be too IMO.)

#6 lore raw
Quoted 12 lines · Andy Lutomirski
Quoted 10 lines · Thomas Gleixner

__arch_use_vsyscall() is a pointless exercise TBH. The VDSO data should be updated unconditionally so all the trivial interfaces like time() and getres() just work independently of the functions which depend on the underlying clocksource.

This functions have a fallback operation already:

Let __arch_get_hw_counter() return U64_MAX and the syscall fallback is invoked.

Quoted 5 lines

My thought was that __arch_get_hw_counter() could return last-1 to indicate failure, which would allow the two checks to be folded into one check. Or we could continue to use U64_MAX and rely on the fact that (s64)U64_MAX < 0, not worry about the cycle counter overflowing, and letting cycles < last catch it.

This is not an overflow catch. It's solely to deal with the fact that on X86 you can observe (cycles < last) on multi socket systems under rare circumstances. Any other architecture does not have that issue AFAIK.

The wraparound of clocksources with a smaller width than 64bit is handled by:

    delta = (cycles - last) & mask;

which operates on unsigned values for obvious reasons.

Quoted 2 lines

(And we should change it to return s64 at some point regardless -- all the math is signed, so the underlying types should be too IMO.)

See above. delta is guaranteed to be >= 0 and the mult/shift is not signed either. All the base values which are in the VDSO are unsigned as well.

The weird typecast there:

    if ((s64)cycles < 0)

could as well be

    if (cycles == U64_MAX)

but the typecasted version creates better code.

I tried to fold the two operations (see patch below) and on all machines I tested on (various generations of Intel and AMD) the result is slower than what we have now by a couple of cycles, which is a lot for these functions (i.e. between 3-5%). I'm sure I tried that before and ended up with the existing code as the fastest variant.

Why? That's subject to speculation :)

Thanks,

tglx

8<----------

 arch/x86/include/asm/vdso/gettimeofday.h |   39 ++++++-------------------------
 lib/vdso/gettimeofday.c                  |   23 +++---------------
 2 files changed, 13 insertions(+), 49 deletions(-)
arch/x86/include/asm/vdso/gettimeofday.h +8 -31
@@ -235,10 +235,14 @@ static u64 vread_hvclock(void)
 }
 #endif
 
-static inline u64 __arch_get_hw_counter(s32 clock_mode)
+static inline u64 __arch_get_hw_counter(s32 clock_mode, u64 last, u64 mask)
 {
+	/*
+	 * Mask operation is not required as all VDSO clocksources are
+	 * 64bit wide.
+	 */
 	if (clock_mode == VCLOCK_TSC)
-		return (u64)rdtsc_ordered();
+		return (u64)rdtsc_ordered() - last;
 	/*
 	 * For any memory-mapped vclock type, we need to make sure that gcc
 	 * doesn't cleverly hoist a load before the mode check.  Otherwise we
@@ -248,13 +252,13 @@ static inline u64 __arch_get_hw_counter(
 #ifdef CONFIG_PARAVIRT_CLOCK
 	if (clock_mode == VCLOCK_PVCLOCK) {
 		barrier();
-		return vread_pvclock();
+		return vread_pvclock() - last;
 	}
 #endif
 #ifdef CONFIG_HYPERV_TIMER
 	if (clock_mode == VCLOCK_HVCLOCK) {
 		barrier();
-		return vread_hvclock();
+		return vread_hvclock() - last;
 	}
 #endif
 	return U64_MAX;
@@ -265,33 +269,6 @@ static __always_inline const struct vdso
 	return __vdso_data;
 }
 
-/*
- * x86 specific delta calculation.
- *
- * The regular implementation assumes that clocksource reads are globally
- * monotonic. The TSC can be slightly off across sockets which can cause
- * the regular delta calculation (@cycles - @last) to return a huge time
- * jump.
- *
- * Therefore it needs to be verified that @cycles are greater than
- * @last. If not then use @last, which is the base time of the current
- * conversion period.
- *
- * This variant also removes the masking of the subtraction because the
- * clocksource mask of all VDSO capable clocksources on x86 is U64_MAX
- * which would result in a pointless operation. The compiler cannot
- * optimize it away as the mask comes from the vdso data and is not compile
- * time constant.
- */
-static __always_inline
-u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
-{
-	if (cycles > last)
-		return (cycles - last) * mult;
-	return 0;
-}
-#define vdso_calc_delta vdso_calc_delta
-
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __ASM_VDSO_GETTIMEOFDAY_H */
lib/vdso/gettimeofday.c +5 -18
@@ -26,34 +26,21 @@
 #include <asm/vdso/gettimeofday.h>
 #endif /* ENABLE_COMPAT_VDSO */
 
-#ifndef vdso_calc_delta
-/*
- * Default implementation which works for all sane clocksources. That
- * obviously excludes x86/TSC.
- */
-static __always_inline
-u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
-{
-	return ((cycles - last) & mask) * mult;
-}
-#endif
-
 static int do_hres(const struct vdso_data *vd, clockid_t clk,
 		   struct __kernel_timespec *ts)
 {
 	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
-	u64 cycles, last, sec, ns;
+	u64 delta, sec, ns;
 	u32 seq;
 
 	do {
 		seq = vdso_read_begin(vd);
-		cycles = __arch_get_hw_counter(vd->clock_mode);
-		ns = vdso_ts->nsec;
-		last = vd->cycle_last;
-		if (unlikely((s64)cycles < 0))
+		delta = __arch_get_hw_counter(vd->clock_mode, vd->cycle_last,
+					      vd->mask);
+		if (unlikely((s64)delta < 0))
 			return -1;
 
-		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
+		ns = vdso_ts->nsec + delta * vd->mult;
 		ns >>= vd->shift;
 		sec = vdso_ts->sec;
 	} while (unlikely(vdso_read_retry(vd, seq)));