x86/vdso: on Intel, VDSO should handle CLOCK_MONOTONIC_RAW
Subsystems (3)
PATCH 2/2 x86/vdso: on Intel, VDSO should handle CLOCK_MONOTONIC_RAW
Currently the VDSO does not handle
clock_gettime( CLOCK_MONOTONIC_RAW, &ts )
on Intel / AMD - it calls
vdso_fallback_gettime()
for this clock, which issues a syscall, having an unacceptably high latency (minimum measurable time or time between measurements) of 300-700ns on 2 2.8-3.9ghz Haswell x86_64 Family'_'Model : 06_3C machines under various versions of Linux.
Sometimes, particularly when correlating elapsed time to performance counter values, code needs to know elapsed time from the perspective of the CPU no matter how "hot" / fast or "cold" / slow it might be running wrt NTP / PTP ; when code needs this, the latencies with a syscall are often unacceptably high.
I reported this as Bug #198161 :
'https://bugzilla.kernel.org/show_bug.cgi?id=198961'
and in previous posts with subjects matching 'CLOCK_MONOTONIC_RAW' .
This patch handles CLOCK_MONOTONIC_RAW clock_gettime() in the VDSO , by exporting the raw clock calibration, last cycles, last xtime_nsec, and last raw_sec value in the vsyscall_gtod_data during vsyscall_update() .
Now the new do_monotonic_raw() function in the vDSO has a latency of @ 24ns on average, and the test program:
tools/testing/selftest/timers/inconsistency-check.c
succeeds with arguments: '-c 4 -t 120' or any arbitrary -t value.
The patch is against Linus' latest 4.16-rc5 tree, current HEAD of :
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
.
This patch affects only files:
arch/x86/include/asm/vgtod.h
arch/x86/entry/vdso/vclock_gettime.c
arch/x86/entry/vdso/vdso.lds.S
arch/x86/entry/vdso/vdsox32.lds.S
arch/x86/entry/vdso/vdso32/vdso32.lds.S
arch/x86/entry/vsyscall/vsyscall_gtod.c
This is a second patch in the series,
which adds a record of the calibrated tsc frequency to the VDSO, and a new header:
uapi/asm/vdso_tsc_calibration.h
which defines a structure :
struct linux_tsc_calibration { u32 tsc_khz, mult, shift ; };and a getter function in the VDSO that can optionally be used by user-space code to implement sub-nanosecond precision clocks . This second patch is entirely optional but I think greatly expands the scope of user-space TSC readers .
Best Regards,
Jason Vas Dias .
---
arch/x86/entry/vdso/vclock_gettime.c renamed from arch/x86/entry/vdso/vclock_gettime.c.4.16-rc5-p1 +39 -0
@@ -21,6 +21,7 @@#include <linux/math64.h> #include <linux/time.h> #include <linux/kernel.h> +#include <uapi/asm/vdso_tsc_calibration.h> #define gtod (&VVAR(vsyscall_gtod_data))
@@ -385,3 +386,41 @@ notrace time_t __vdso_time(time_t *t) }time_t time (time_t *t) __attribute__((weak,alias ("__vdso_time" ))); + +extern unsigned +__vdso_linux_tsc_calibration (struct linux_tsc_calibration *); + +notrace unsigned +__vdso_linux_tsc_calibration (struct linux_tsc_calibration *tsc_cal) +{ +if ( (gtod->vclock_mode ==VCLOCK_TSC ) && (tsc_cal != ((void *)0UL )) ) + { + tsc_cal -> tsc_khz = gtod->tsc_khz; + tsc_cal -> mult = gtod->raw_mult; + tsc_cal -> shift = gtod->raw_shift; +return 1 ; + } +return 0 ; +} + +unsigned linux_tsc_calibration (void ) + __attribute((weak,alias ("__vdso_linux_tsc_calibration" ))); + +extern unsigned +__vdso_linux_tsc_calibration (struct linux_tsc_calibration *); + +notrace unsigned +__vdso_linux_tsc_calibration (struct linux_tsc_calibration *tsc_cal) +{ +if ( (gtod->vclock_mode ==VCLOCK_TSC ) && (tsc_cal != ((void *)0UL )) ) + { + tsc_cal -> tsc_khz = gtod->tsc_khz; + tsc_cal -> mult = gtod->raw_mult; + tsc_cal -> shift = gtod->raw_shift; +return 1 ; + } +return 0 ; +} + +unsigned linux_tsc_calibration (void ) + __attribute((weak,alias ("__vdso_linux_tsc_calibration" )));
arch/x86/entry/vdso/vdso.lds.S renamed from arch/x86/entry/vdso/vdso.lds.S.4.16-rc5-p1 +2 -0
@@ -25,6 +25,8 @@ VERSION { __vdso_getcpu; time; __vdso_time; + linux_tsc_calibration; + __vdso_linux_tsc_calibration; local: *; }; }
arch/x86/entry/vdso/vdso32/vdso32.lds.S renamed from arch/x86/entry/vdso/vdso32/vdso32.lds.S.4.16-rc5-p1 +1 -0
@@ -26,6 +26,7 @@ VERSION __vdso_clock_gettime; __vdso_gettimeofday; __vdso_time; + __vdso_linux_tsc_calibration; }; LINUX_2.5 {
arch/x86/entry/vdso/vdsox32.lds.S renamed from arch/x86/entry/vdso/vdsox32.lds.S.4.16-rc5-p1 +1 -0
@@ -21,6 +21,7 @@ VERSION { __vdso_gettimeofday; __vdso_getcpu; __vdso_time; + __vdso_linux_tsc_calibration; local: *; }; }
arch/x86/entry/vsyscall/vsyscall_gtod.c renamed from arch/x86/entry/vsyscall/vsyscall_gtod.c.4.16-rc5-p1 +3 -0
@@ -18,6 +18,8 @@#include <asm/vvar.h> #include <asm/cpufeature.h> +extern unsigned tsc_khz; +int vclocks_used__read_mostly ; DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data);
@@ -51,6 +53,7 @@ void update_vsyscall(struct timekeeper * vdata->raw_mult = tk->tkr_raw.mult; vdata->raw_shift = tk->tkr_raw.shift; vdata->has_rdtscp =static_cpu_has (X86_FEATURE_RDTSCP ); + vdata->tsc_khz = tsc_khz; vdata->wall_time_sec = tk->xtime_sec; vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
arch/x86/include/asm/vgtod.h renamed from arch/x86/include/asm/vgtod.h.4.16-rc5-p1 +1 -0
@@ -27,6 +27,7 @@ struct vsyscall_gtod_data {u32 raw_mult;u32 raw_shift;u32 has_rdtscp; +u32 tsc_khz;/* open coded 'struct timespec' */ u64 wall_time_snsec;
arch/x86/include/uapi/asm/vdso_tsc_calibration.h renamed from arch/x86/include/uapi/asm/vdso_tsc_calibration.h.4.16-rc5-p1 +73 -0
73 lines not on this page · raw message
---