Add 32 bit VDSO time function support
Subsystems (7)
v11 cover Add 32 bit VDSO time function support
From: Stefani Seibold <[email protected]>
This patch add the functions vdso_gettimeofday(), vdso_clock_gettime() and vdso_time() to the 32 bit VDSO.
The reason to do this was to get a fast reliable time stamp. Many developers uses TSC to get a fast time stamp, without knowing the pitfalls. VDSO time functions a fast and a reliable way, because the kernel knows the best time source and the P- and C-state of the CPU.
The helper library to use the VDSO functions can be download at
http://http://seibold.net/vdso.c
The libary is very small, only 228 lines of code. Compile it with gcc -Wall -O3 -fpic vdso.c -lrt -shared -o libvdso.so and use it with LD_PRELOAD=<path>/libvdso.so
This kind of helper must be integrated into glibc, for x86 64 bit and PowerPC it is already there.
Some linux 32 bit kernel benchmark results (all measurements are in nano seconds):
Intel(R) Celeron(TM) CPU 400MHz
Average time kernel call:
gettimeofday(): 1039
clock_gettime(): 1578
time(): 526
Average time VDSO call:
gettimeofday(): 378
clock_gettime(): 303
time(): 60
Celeron(R) Dual-Core CPU T3100 1.90GHz
Average time kernel call:
gettimeofday(): 209
clock_gettime(): 406
time(): 135
Average time VDSO call:
gettimeofday(): 51
clock_gettime(): 43
time(): 10
So you can see a performance increase between 4 and 13, depending on the CPU and the function.
The address layout of the VDSO has changed, because there is no fixed address space available on a x86 32 bit kernel, despite the name. Because someone decided to add an offset to the __FIXADDR_TOP for virtualization.
Also the IA32 Emulation uses the whole 4 GB address space, so there is no fixed address available.
This was the reason not depend on this kind of address and change the layout of the VDSO. The VDSO for a 32 bit application has now three pages:
^ Higher Address
|
+----------------------------------------+ + VDSO page (includes code) ro+x + +----------------------------------------+ + VVAR page (export kernel variables) ro + +----------------------------------------+ + HPET page (mapped registers) ro +----------------------------------------+ |
^ Lower Address
The VDSO page for a 32 bit resided still on 0xffffe000, the the VVAR and HPET page are mapped before.
In the non compat mode the VMA of the VDSO is now 3 pages for a 32 bit kernel. So this decrease the available logical address room by 2 pages.
The patch is against kernel 3.14 (e7651b819e90da924991d727d3c007200a18670d)
Changelog:
25.11.2012 - first release and proof of concept for linux 3.4 11.12.2012 - Port to linux 3.7 and code cleanup
12.12.2012 - fixes suggested by Andy Lutomirski
- fixes suggested by John Stultz
- use call VDSO32_vsyscall instead of int 80
- code cleanup
17.12.2012 - support for IA32_EMULATION, this includes
- code cleanup
- include cleanup to fix compile warnings and errors
- move out seqcount from seqlock, enable use in VDSO
- map FIXMAP and HPET into the 32 bit address space
18.12.2012 - split into separate patches
30.01.2014 - revamp the code
- code clean up
- VDSO layout changed
- no fixed addresses
- port to 3.14
01.02.2014 - code cleanup
02.02.2014 - code cleanup
- split into more patches
- use HPET_COUNTER instead of hard coded value
- fix changelog to the right year ;-)
02.02.2014 - reverse the mapping, this make the new VDSO 32 bit support
full compatible.
03.02.2014 - code cleanup
- fix comment
- fix ABI break in vdso32.lds.S
04.02.2014 - revamp IA32 emulation support
- introduce VVAR macro
- rearranged vsyscall_gtod_data struture for IA32 emulation support
- code cleanup
PATCH v11 1/9 Make vsyscall_gtod_data handling x86 generic
From: Stefani Seibold <[email protected]>
This patch move the vsyscall_gtod_data handling out of vsyscall_64.c into an additonal file vsyscall_gtod.c to make the functionality available for x86 32 bit kernel.
It also adds a new vsyscall_32.c which setup the VVAR page.
--- arch/x86/Kconfig | 4 +-- arch/x86/include/asm/clocksource.h | 4 --- arch/x86/include/asm/fixmap.h | 2 ++ arch/x86/include/asm/vvar.h | 6 ++++ arch/x86/kernel/Makefile | 3 +- arch/x86/kernel/hpet.c | 4 --- arch/x86/kernel/setup.c | 2 -- arch/x86/kernel/tsc.c | 2 -- arch/x86/kernel/vmlinux.lds.S | 3 -- arch/x86/kernel/vsyscall_32.c | 24 +++++++++++++++ arch/x86/kernel/vsyscall_64.c | 44 ---------------------------- arch/x86/kernel/vsyscall_gtod.c | 60 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 96 insertions(+), 62 deletions(-) create mode 100644 arch/x86/kernel/vsyscall_32.c create mode 100644 arch/x86/kernel/vsyscall_gtod.c
arch/x86/Kconfig +2 -2
@@ -107,9 +107,9 @@ config X86 select HAVE_ARCH_SOFT_DIRTY select CLOCKSOURCE_WATCHDOG select GENERIC_CLOCKEVENTS - select ARCH_CLOCKSOURCE_DATA if X86_64 + select ARCH_CLOCKSOURCE_DATA select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC) - select GENERIC_TIME_VSYSCALL if X86_64 + select GENERIC_TIME_VSYSCALL select KTIME_SCALAR if X86_32 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER
arch/x86/include/asm/clocksource.h +0 -4
@@ -3,8 +3,6 @@#ifndef _ASM_X86_CLOCKSOURCE_H#define _ASM_X86_CLOCKSOURCE_H -#ifdef CONFIG_X86_64 -#define VCLOCK_NONE 0/* No vDSO clock available. */ #define VCLOCK_TSC 1/* vDSO should use vread_tsc. */ #define VCLOCK_HPET 2/* vDSO should use vread_hpet. */
@@ -14,6 +12,4 @@ struct arch_clocksource_data { int vclock_mode; }; -#endif /* CONFIG_X86_64 */ -#endif /* _ASM_X86_CLOCKSOURCE_H */
arch/x86/include/asm/fixmap.h +2 -0
@@ -75,6 +75,8 @@ enum fixed_addresses {#ifdef CONFIG_X86_32 FIX_HOLE ,FIX_VDSO , +VVAR_PAGE , +VSYSCALL_HPET , #elseVSYSCALL_LAST_PAGE ,VSYSCALL_FIRST_PAGE =VSYSCALL_LAST_PAGE
arch/x86/include/asm/vvar.h +6 -0
@@ -17,7 +17,13 @@*/ /* Base address of vvars. This is not ABI. */ +#ifdef CONFIG_X86_64 #define VVAR_ADDRESS (-10*1024*1024 - 4096) +#else +extern char __vvar_page; + +#define VVAR_ADDRESS (&__vvar_page) +#endif #if defined(__VVAR_KERNEL_LDS)
arch/x86/kernel/Makefile +2 -1
@@ -26,7 +26,8 @@ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-y += probe_roms.o obj-$(CONFIG_X86_32) += i386_ksyms_32.o obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o -obj-y += syscall_$(BITS).o +obj-y += syscall_$(BITS).o vsyscall_gtod.o +obj-$(CONFIG_X86_32) += vsyscall_32.o obj-$(CONFIG_X86_64) += vsyscall_64.o obj-$(CONFIG_X86_64) += vsyscall_emu_64.o obj-$(CONFIG_SYSFS) += ksysfs.o
arch/x86/kernel/hpet.c +0 -4
@@ -74,9 +74,7 @@ static inline void hpet_writel(unsigned int d, unsigned int a)static inline void hpet_set_mapping (void ) { hpet_virt_address =ioremap_nocache (hpet_address,HPET_MMAP_SIZE ); -#ifdef CONFIG_X86_64 __set_fixmap (VSYSCALL_HPET , hpet_address,PAGE_KERNEL_VVAR_NOCACHE ); -#endif }static inline void hpet_clear_mapping (void )
@@ -752,9 +750,7 @@ static struct clocksource clocksource_hpet = { .mask = HPET_MASK, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .resume = hpet_resume_counter, -#ifdef CONFIG_X86_64 .archdata = { .vclock_mode = VCLOCK_HPET }, -#endif };static int hpet_clocksource_register (void )
arch/x86/kernel/setup.c +0 -2
@@ -1182,9 +1182,7 @@ void __init setup_arch(char **cmdline_p)tboot_probe (); -#ifdef CONFIG_X86_64 map_vsyscall (); -#endif generic_apic_probe ();
arch/x86/kernel/tsc.c +0 -2
@@ -988,9 +988,7 @@ static struct clocksource clocksource_tsc = { .mask = CLOCKSOURCE_MASK(64), .flags =CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, -#ifdef CONFIG_X86_64 .archdata = { .vclock_mode = VCLOCK_TSC }, -#endif };void mark_tsc_unstable (char *reason)
arch/x86/kernel/vmlinux.lds.S +0 -3
@@ -147,7 +147,6 @@ SECTIONS _edata = .; } :data -#ifdef CONFIG_X86_64 . = ALIGN(PAGE_SIZE); __vvar_page = .;
@@ -169,8 +168,6 @@ SECTIONS . = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE); -#endif /* CONFIG_X86_64 */ - /* Init code and data - will be freed after init */ . = ALIGN(PAGE_SIZE); .init.begin : AT(ADDR(.init.begin) - LOAD_OFFSET) {
arch/x86/kernel/vsyscall_32.c new file +24 -0
@@ -0,0 +1,24 @@ +/* +* Copyright (C) 2001 Andrea Arcangeli <[email protected]> SuSE +* Copyright 2003 Andi Kleen, SuSE Labs. +* +* Modified for x86 32 bit arch by Stefani Seibold <[email protected]> +* +* Thanks to [email protected] for some useful hint. +* Special thanks to Ingo Molnar for his early experience with +* a different vsyscall implementation for Linux/IA32 and for the name. +* +*/ + +#include <asm/vsyscall.h> +#include <asm/pgtable.h> +#include <asm/fixmap.h> + +void __init map_vsyscall (void ) +{ +extern char __vvar_page; +unsigned long physaddr_vvar_page =__pa_symbol (&__vvar_page); + +__set_fixmap (VVAR_PAGE , physaddr_vvar_page,PAGE_KERNEL_VVAR ); +} +
arch/x86/kernel/vsyscall_64.c +0 -44
62 lines not on this page · raw message
arch/x86/kernel/vsyscall_gtod.c new file +60 -0
60 lines not on this page · raw message
Signature · 1 line
1.8.5.3
Re: [PATCH v11 1/9] Make vsyscall_gtod_data handling x86 generic
Quoted 7 lines · On Tue, Feb 4, 2014 at 2:15 AM, <[email protected]> wrote:
From: Stefani Seibold <[email protected]>
This patch move the
vsyscall_gtod_datahandling out ofvsyscall_64.cinto an additonal filevsyscall_gtod.cto make the functionality available for x86 32 bit kernel.It also adds a new
vsyscall_32.cwhich setup the VVAR page.
Other than the unconditional __set_fixmap calls, I think this is okay.
(Can you *please* directly answer review comments rather than just sending out new versions at high speed. The current approach means that I need to reread the patch again every time to see how you've responded, and I'm currently disinclined to believe that a new version will actually address a review comment *at all*.)
--Andy
PATCH v11 2/9 Add new func _install_special_mapping() to mmap.c
In reply to [email protected]From: Stefani Seibold <[email protected]>
The _install_special_mapping() is the new base function for install_special_mapping(). This function will return a pointer of the created VMA or a error code in an ERR_PTR()
This new function will be needed by the for the vdso 32 bit support to map the additonal vvar and hpet pages into the 32 bit address space. This will be done with io_remap_pfn_range() and remap_pfn_range, which requieres a vm_area_struct.
--- include/linux/mm.h | 3 +++ mm/mmap.c | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-)
include/linux/mm.h +3 -0
@@ -1740,6 +1740,9 @@ extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);extern struct file *get_mm_exe_file (struct mm_struct *mm);extern int may_expand_vm (struct mm_struct *mm,unsigned long npages); +extern struct vm_area_struct *_install_special_mapping (struct mm_struct *mm, +unsigned long addr,unsigned long len, +unsigned long flags,struct page **pages);extern int install_special_mapping (struct mm_struct *mm,unsigned long addr,unsigned long len,unsigned long flags,struct page **pages);
mm/mmap.c +16 -4
@@ -2918,7 +2918,7 @@ static const struct vm_operations_struct special_mapping_vmops = {* The array pointer and the pages it points to are assumed to stay alive * for as long as this mapping might exist. */ -int install_special_mapping (struct mm_struct *mm, +struct vm_area_struct *_install_special_mapping (struct mm_struct *mm,unsigned long addr,unsigned long len,unsigned long vm_flags,struct page **pages) {
@@ -2927,7 +2927,7 @@ int install_special_mapping(struct mm_struct *mm, vma =kmem_cache_zalloc (vm_area_cachep,GFP_KERNEL );if (unlikely (vma ==NULL )) -return -ENOMEM ; +return ERR_PTR (-ENOMEM );INIT_LIST_HEAD (&vma->anon_vma_chain); vma->vm_mm = mm;
@@ -2948,11 +2948,23 @@ int install_special_mapping(struct mm_struct *mm,perf_event_mmap (vma); -return 0 ; +return vma; out:kmem_cache_free (vm_area_cachep, vma); -return ret; +return ERR_PTR (ret); +} + +int install_special_mapping (struct mm_struct *mm, +unsigned long addr,unsigned long len, +unsigned long vm_flags,struct page **pages) +{ +struct vm_area_struct *vma =_install_special_mapping (mm, + addr, len, vm_flags, pages); + +if (IS_ERR (vma)) +return PTR_ERR (vma); +return 0 ; }static DEFINE_MUTEX (mm_all_locks_mutex);
Signature · 1 line
1.8.5.3
PATCH v11 3/9 revamp vclock_gettime.c
In reply to [email protected]From: Stefani Seibold <[email protected]>
This intermediate patch revamps the vclock_gettime.c by moving some functions around. It is only for spliting purpose, to make whole the 32 bit vdso timer patch easier to review.
--- arch/x86/vdso/vclock_gettime.c | 85 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 43 deletions(-)
arch/x86/vdso/vclock_gettime.c +42 -43
@@ -26,41 +26,26 @@#define gtod (&VVAR(vsyscall_gtod_data)) -notrace static cycle_t vread_tsc(void) +static notrace cycle_t vread_hpet(void) { -cycle_t ret; -u64 last; - -/* -* Empirically, a fence (of type that depends on the CPU) -* before rdtsc is enough to ensure that rdtsc is ordered -* with respect to loads. The various CPU manuals are unclear -* as to whether rdtsc can be reordered with later loads, -* but no one has ever seen it happen. -*/ -rdtsc_barrier (); - ret = (cycle_t )vget_cycles (); - - last =VVAR (vsyscall_gtod_data).clock.cycle_last; - -if (likely (ret >= last)) -return ret; +return readl ((const void __iomem *)fix_to_virt (VSYSCALL_HPET ) +HPET_COUNTER ); +} -/* -* GCC likes to generate cmov here, but this branch is extremely -* predictable (it's just a funciton of time and the likely is -* very likely) and there's a data dependence, so force GCC -* to generate a branch instead. I don't barrier() because -* we don't actually need a barrier, and if this function -* ever gets inlined it will generate worse code. -*/ - asmvolatile ("" ); -return last; +notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) +{ +long ret; + asm("syscall" :"=a" (ret) : +"0" (__NR_clock_gettime),"D" (clock),"S" (ts) :"memory" ); +return ret; } -static notrace cycle_t vread_hpet(void) +notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) { -return readl ((const void __iomem *)fix_to_virt (VSYSCALL_HPET ) +HPET_COUNTER ); +long ret; + + asm("syscall" :"=a" (ret) : +"0" (__NR_gettimeofday),"D" (tv),"S" (tz) :"memory" ); +return ret; }#ifdef CONFIG_PARAVIRT_CLOCK
@@ -133,23 +118,37 @@ static notrace cycle_t vread_pvclock(int *mode) }#endif -notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) +notrace static cycle_t vread_tsc(void) { -long ret; - asm("syscall" :"=a" (ret) : -"0" (__NR_clock_gettime),"D" (clock),"S" (ts) :"memory" ); -return ret; -} +cycle_t ret; +u64 last; -notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) -{ -long ret; +/* +* Empirically, a fence (of type that depends on the CPU) +* before rdtsc is enough to ensure that rdtsc is ordered +* with respect to loads. The various CPU manuals are unclear +* as to whether rdtsc can be reordered with later loads, +* but no one has ever seen it happen. +*/ +rdtsc_barrier (); + ret = (cycle_t )vget_cycles (); - asm("syscall" :"=a" (ret) : -"0" (__NR_gettimeofday),"D" (tv),"S" (tz) :"memory" ); -return ret; -} + last =VVAR (vsyscall_gtod_data).clock.cycle_last; +if (likely (ret >= last)) +return ret; + +/* +* GCC likes to generate cmov here, but this branch is extremely +* predictable (it's just a funciton of time and the likely is +* very likely) and there's a data dependence, so force GCC +* to generate a branch instead. I don't barrier() because +* we don't actually need a barrier, and if this function +* ever gets inlined it will generate worse code. +*/ + asmvolatile ("" ); +return last; +} notrace static inline u64 vgetsns(int *mode) {
Signature · 1 line
1.8.5.3
PATCH v11 4/9 vclock_gettime.c __vdso_clock_gettime cleanup
In reply to [email protected]From: Stefani Seibold <[email protected]>
This patch is a small code cleanup for the __vdso_clock_gettime() function.
It removes the unneeded return values from do_monotonic_coarse() and do_realtime_coarse() and add a fallback label for doing the kernel gettimeofday() system call.
--- arch/x86/vdso/vclock_gettime.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-)
arch/x86/vdso/vclock_gettime.c +14 -13
@@ -209,7 +209,7 @@ notrace static int do_monotonic(struct timespec *ts)return mode; } -notrace static int do_realtime_coarse(struct timespec *ts) +notrace static void do_realtime_coarse(struct timespec *ts) {unsigned long seq; do {
@@ -217,10 +217,9 @@ notrace static int do_realtime_coarse(struct timespec *ts) ts->tv_sec = gtod->wall_time_coarse.tv_sec; ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; }while (unlikely (read_seqcount_retry (>od->seq, seq))); -return 0 ; } -notrace static int do_monotonic_coarse(struct timespec *ts) +notrace static void do_monotonic_coarse(struct timespec *ts) {unsigned long seq; do {
@@ -228,30 +227,32 @@ notrace static int do_monotonic_coarse(struct timespec *ts) ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; }while (unlikely (read_seqcount_retry (>od->seq, seq))); - -return 0 ; } notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) { -int ret =VCLOCK_NONE ; -switch (clock) {case CLOCK_REALTIME : - ret =do_realtime (ts); +if (do_realtime (ts) ==VCLOCK_NONE ) + goto fallback;break ;case CLOCK_MONOTONIC : - ret =do_monotonic (ts); +if (do_monotonic (ts) ==VCLOCK_NONE ) + goto fallback;break ;case CLOCK_REALTIME_COARSE : -return do_realtime_coarse (ts); +do_realtime_coarse (ts); +break ;case CLOCK_MONOTONIC_COARSE : -return do_monotonic_coarse (ts); +do_monotonic_coarse (ts); +break ; +default : + goto fallback; } -if (ret ==VCLOCK_NONE ) -return vdso_fallback_gettime (clock, ts);return 0 ; +fallback: +return vdso_fallback_gettime (clock, ts); }int clock_gettime (clockid_t ,struct timespec *) __attribute__((weak,alias ("__vdso_clock_gettime" )));
Signature · 1 line
1.8.5.3
PATCH v11 5/9 replace VVAR(vsyscall_gtod_data) by gtod macro
In reply to [email protected]From: Stefani Seibold <[email protected]>
There a currently more than 30 users of the gtod macro, so replace the last VVAR(vsyscall_gtod_data) by gtod macro.
--- arch/x86/vdso/vclock_gettime.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/vdso/vclock_gettime.c +3 -3
@@ -109,7 +109,7 @@ static notrace cycle_t vread_pvclock(int *mode) *mode =VCLOCK_NONE ;/* refer to tsc.c read_tsc() comment for rationale */ - last =VVAR (vsyscall_gtod_data).clock.cycle_last; + last = gtod->clock.cycle_last;if (likely (ret >= last))return ret;
@@ -133,7 +133,7 @@ notrace static cycle_t vread_tsc(void)rdtsc_barrier (); ret = (cycle_t )vget_cycles (); - last =VVAR (vsyscall_gtod_data).clock.cycle_last; + last = gtod->clock.cycle_last;if (likely (ret >= last))return ret;
@@ -288,7 +288,7 @@ int gettimeofday(struct timeval *, struct timezone *) notrace time_t __vdso_time(time_t *t) {/* This is atomic on x86_64 so we don't need any locks. */ -time_t result =ACCESS_ONCE (VVAR (vsyscall_gtod_data).wall_time_sec); +time_t result =ACCESS_ONCE (gtod->wall_time_sec);if (t) *t = result;
Signature · 1 line
1.8.5.3
PATCH v11 6/9 cleanup __vdso_gettimeofday
In reply to [email protected]From: Stefani Seibold <[email protected]>
This patch do a little cleanup for the __vdso_gettimeofday() function.
It kick out an unneeded ret local variable and makes the code faster if only the timezone is needed.
--- arch/x86/vdso/vclock_gettime.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
arch/x86/vdso/vclock_gettime.c +2 -5
@@ -259,13 +259,12 @@ int clock_gettime(clockid_t, struct timespec *) notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) { -long ret =VCLOCK_NONE ; -if (likely (tv !=NULL )) {BUILD_BUG_ON (offsetof(struct timeval , tv_usec) != offsetof(struct timespec , tv_nsec) ||sizeof (*tv) !=sizeof (struct timespec )); - ret =do_realtime ((struct timespec *)tv); +if (unlikely (do_realtime (&tmp) ==VCLOCK_NONE )) +return vdso_fallback_gtod (tv, tz); tv->tv_usec /=1000 ; }if (unlikely (tz !=NULL )) {
@@ -274,8 +273,6 @@ notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) tz->tz_dsttime = gtod->sys_tz.tz_dsttime; } -if (ret ==VCLOCK_NONE ) -return vdso_fallback_gtod (tv, tz);return 0 ; }int gettimeofday (struct timeval *,struct timezone *)
Signature · 1 line
1.8.5.3
PATCH v11 7/9 introduce VVAR marco for vdso32
In reply to [email protected]From: Stefani Seibold <[email protected]>
This patch revamp the vvar.h for introduce the VVAR macro for vdso32.
--- arch/x86/include/asm/vvar.h | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-)
arch/x86/include/asm/vvar.h +21 -11
@@ -16,15 +16,6 @@* you mess up, the linker will catch it.) */ -/* Base address of vvars. This is not ABI. */ -#ifdef CONFIG_X86_64 -#define VVAR_ADDRESS (-10*1024*1024 - 4096) -#else -extern char __vvar_page; - -#define VVAR_ADDRESS (&__vvar_page) -#endif -#if defined(__VVAR_KERNEL_LDS)/* The kernel linker script defines its own magic to put vvars in the
@@ -35,16 +26,35 @@ extern char __vvar_page;#else +#ifdef BUILD_VDSO32 + +#define DECLARE_VVAR (offset, type, name) \ + extern type vvar_ ## name __attribute__((visibility("hidden"))); + +#define VVAR (name) (vvar_ ## name) + +#else + +/* Base address of vvars. This is not ABI. */ +#ifdef CONFIG_X86_64 +#define VVAR_ADDRESS (-10*1024*1024 - 4096) +#else +extern char __vvar_page; + +#define VVAR_ADDRESS (&__vvar_page) +#endif +#define DECLARE_VVAR (offset, type, name) \ static type const * const vvaraddr_ ## name = \ (void *)(VVAR_ADDRESS + (offset)); +#define VVAR (name) (*vvaraddr_ ## name) +#endif +#define DEFINE_VVAR (type, name) \ type name \ __attribute__((section(".vvar_" #name), aligned(16))) __visible -#define VVAR (name) (*vvaraddr_ ## name) -#endif /* DECLARE_VVAR(offset, type, name) */
Signature · 1 line
1.8.5.3
Re: [PATCH v11 7/9] introduce VVAR marco for vdso32
Quoted 3 lines · On Tue, Feb 4, 2014 at 2:15 AM, <[email protected]> wrote:
From: Stefani Seibold <[email protected]>
This patch revamp the
vvar.hfor introduce the VVAR macro for vdso32.
I like this approach better. Thanks.
Quoted 67 lines
Signed-off-by Stefani Seibold <[email protected]>--- arch/x86/include/asm/vvar.h | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-)arch/x86/include/asm/vvar.h +21
-12@@ -16,15 +16,6 @@* you mess up, the linker will catch it.) */ -/* Base address of vvars. This is not ABI. */ -#ifdef CONFIG_X86_64 -#define VVAR_ADDRESS (-10*1024*1024 - 4096) -#else -extern char __vvar_page; - -#define VVAR_ADDRESS (&__vvar_page) -#endif -#if defined(__VVAR_KERNEL_LDS)/* The kernel linker script defines its own magic to put vvars in the @@ -35,16 +26,35 @@ extern char __vvar_page;#else +#ifdef BUILD_VDSO32 + +#define DECLARE_VVAR (offset, type, name) \ + extern type vvar_ ## name __attribute__((visibility("hidden"))); + +#define VVAR (name) (vvar_ ## name) + +#else + +/* Base address of vvars. This is not ABI. */ +#ifdef CONFIG_X86_64 +#define VVAR_ADDRESS (-10*1024*1024 - 4096) +#else +extern char __vvar_page; + +#define VVAR_ADDRESS (&__vvar_page) +#endif +#define DECLARE_VVAR (offset, type, name) \ static type const * const vvaraddr_ ## name = \ (void *)(VVAR_ADDRESS + (offset)); +#define VVAR (name) (*vvaraddr_ ## name) +#endif +#define DEFINE_VVAR (type, name) \ type name \ __attribute__((section(".vvar_" #name), aligned(16))) __visible -#define VVAR (name) (*vvaraddr_ ## name) -#endif /* DECLARE_VVAR(offset, type, name) */ --1.8.5.3
Signature · 2 lines
Andy Lutomirski AMA Capital Management, LLC
PATCH v11 8/9 Add 32 bit VDSO time support for 32 bit kernel
In reply to [email protected]From: Stefani Seibold <[email protected]>
This patch add the time support for 32 bit a VDSO to a 32 bit kernel.
For 32 bit programs running on a 32 bit kernel, the same mechanism is used as for 64 bit programs running on a 64 bit kernel.
--- arch/x86/include/asm/vdso.h | 3 ++ arch/x86/include/asm/vdso32.h | 11 +++++++ arch/x86/vdso/Makefile | 7 +++++ arch/x86/vdso/vclock_gettime.c | 54 ++++++++++++++++++++++++++++++++-- arch/x86/vdso/vdso-layout.lds.S | 22 ++++++++++++++ arch/x86/vdso/vdso32-setup.c | 55 +++++++++++++++++++++++++++++++---- arch/x86/vdso/vdso32/vclock_gettime.c | 16 ++++++++++ arch/x86/vdso/vdso32/vdso32.lds.S | 9 ++++++ 8 files changed, 169 insertions(+), 8 deletions(-) create mode 100644 arch/x86/include/asm/vdso32.h create mode 100644 arch/x86/vdso/vdso32/vclock_gettime.c
arch/x86/include/asm/vdso.h +3 -0
@@ -2,6 +2,9 @@#define _ASM_X86_VDSO_H#if definedCONFIG_X86_32 || definedCONFIG_COMPAT + +#include <asm/vdso32.h> +extern const char VDSO32_PRELINK [];/*
arch/x86/include/asm/vdso32.h new file +11 -0
@@ -0,0 +1,11 @@ +#ifndef _ASM_X86_VDSO32_H +#define _ASM_X86_VDSO32_H + +#define VDSO_BASE_PAGE 0 +#define VDSO_VVAR_PAGE 1 +#define VDSO_HPET_PAGE 2 +#define VDSO_PAGES 3 +#define VDSO_PREV_PAGES 2 +#define VDSO_OFFSET (x) ((x) * PAGE_SIZE) + +#endif
arch/x86/vdso/Makefile +7 -0
@@ -145,8 +145,15 @@ KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) $(vdso32-images:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_32) $(vdso32-images:%=$(obj)/%.dbg): asflags-$(CONFIG_X86_64) += -m32 +KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS)) +KBUILD_CFLAGS_32 := $(filter-out -mcmodel=kernel,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=3 -freg-struct-return -fpic +$(vdso32-images:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) + $(vdso32-images:%=$(obj)/%.dbg): $(obj)/vdso32-%.so.dbg: FORCE \ $(obj)/vdso32/vdso32.lds \ + $(obj)/vdso32/vclock_gettime.o \ $(obj)/vdso32/note.o \ $(obj)/vdso32/%.o $(call if_changed,vdso)
arch/x86/vdso/vclock_gettime.c +52 -2
89 lines not on this page · raw message
arch/x86/vdso/vdso-layout.lds.S +22 -0
@@ -6,6 +6,24 @@ SECTIONS { +#ifdef BUILD_VDSO32 +#include <asm/vdso32.h> + + .hpet_sect : { + hpet_page = . - VDSO_OFFSET(VDSO_HPET_PAGE); + } :text :hpet_sect + + .vvar_sect : { + vvar = . - VDSO_OFFSET(VDSO_VVAR_PAGE); + + /* Place all vvars at the offsets in asm/vvar.h. */ +#define EMIT_VVAR(name, offset) vvar_ ## name = vvar + offset; +#define __VVAR_KERNEL_LDS +#include <asm/vvar.h> +#undef __VVAR_KERNEL_LDS +#undef EMIT_VVAR + } :text :vvar_sect +#endif . = VDSO_PRELINK + SIZEOF_HEADERS; .hash : { *(.hash) } :text
@@ -61,4 +79,8 @@ PHDRS dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ note PT_NOTE FLAGS(4); /* PF_R */ eh_frame_hdr PT_GNU_EH_FRAME; +#ifdef BUILD_VDSO32 + vvar_sect PT_NULL FLAGS(4); /* PF_R */ + hpet_sect PT_NULL FLAGS(4); /* PF_R */ +#endif }
arch/x86/vdso/vdso32-setup.c +49 -6
92 lines not on this page · raw message
arch/x86/vdso/vdso32/vclock_gettime.c new file +16 -0
@@ -0,0 +1,16 @@ +#define BUILD_VDSO32 + +#ifdef CONFIG_X86_64 + +#define _ASM_X86_PAGE_H + +#define __pa (x) 0 +#define __va (x) 0 + +#undef CONFIG_ILLEGAL_POINTER_VALUE +#define CONFIG_ILLEGAL_POINTER_VALUE 0 + +#endif + +#include "../vclock_gettime.c" +
arch/x86/vdso/vdso32/vdso32.lds.S +9 -0
@@ -8,6 +8,9 @@ * values visible using the asm-x86/vdso.h macros from the kernel proper. */ +#include <asm/page.h> + +#define BUILD_VDSO32 #define VDSO_PRELINK 0 #include "../vdso-layout.lds.S"
@@ -24,6 +27,9 @@ VERSION __kernel_vsyscall; __kernel_sigreturn; __kernel_rt_sigreturn; + __vdso_clock_gettime; + __vdso_gettimeofday; + __vdso_time; local: *; }; }
@@ -35,3 +41,6 @@ VDSO32_PRELINK = VDSO_PRELINK; VDSO32_vsyscall = __kernel_vsyscall; VDSO32_sigreturn = __kernel_sigreturn; VDSO32_rt_sigreturn = __kernel_rt_sigreturn; +VDSO32_clock_gettime = clock_gettime; +VDSO32_gettimeofday = gettimeofday; +VDSO32_time = time;
Signature · 1 line
1.8.5.3
PATCH v11 9/9 Add 32 bit VDSO time support for 64 bit kernel
In reply to [email protected]From: Stefani Seibold <[email protected]>
This patch add the VDSO time support for the IA32 Emulation Layer.
Due the nature of the kernel headers and the LP64 compiler where the size of a long and a pointer differs against a 32 bit compiler, there is a lot of type hacking necessary.
The vsyscall_gtod_data struture must be a little bit rearranged, to serve for 32- and 64-bit code access.
--- arch/x86/include/asm/vgtod.h | 23 +++++---- arch/x86/kernel/vsyscall_gtod.c | 10 ++-- arch/x86/vdso/vclock_gettime.c | 93 +++++++++++++++++++++++------------ arch/x86/vdso/vdso32/vclock_gettime.c | 11 +++++ include/uapi/linux/time.h | 2 +- 5 files changed, 92 insertions(+), 47 deletions(-)
arch/x86/include/asm/vgtod.h +13 -10
@@ -4,16 +4,16 @@#include <asm/vsyscall.h> #include <linux/clocksource.h> -struct vsyscall_gtod_data { -seqcount_t seq; - -struct {/* extract of a clocksource struct */ -int vclock_mode; -cycle_t cycle_last; -cycle_t mask; -u32 mult; -u32 shift; - } clock; +/* +* vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time +* so the structure must be packed +*/ +struct __attribute__((packed))vsyscall_gtod_data { +int vclock_mode; +cycle_t cycle_last; +cycle_t mask; +u32 mult; +u32 shift;/* open coded 'struct timespec' */ time_t wall_time_sec;
@@ -24,6 +24,9 @@ struct vsyscall_gtod_data {struct timezone sys_tz;struct timespec wall_time_coarse;struct timespec monotonic_time_coarse; + +/* must be at the end, because the size depends on the kernel config */ +seqcount_t seq; };extern struct vsyscall_gtod_data vsyscall_gtod_data;
arch/x86/kernel/vsyscall_gtod.c +5 -5
@@ -28,11 +28,11 @@ void update_vsyscall(struct timekeeper *tk)write_seqcount_begin (&vdata->seq);/* copy vsyscall data */ - vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode; - vdata->clock.cycle_last = tk->clock->cycle_last; - vdata->clock.mask = tk->clock->mask; - vdata->clock.mult = tk->mult; - vdata->clock.shift = tk->shift; + vdata->vclock_mode = tk->clock->archdata.vclock_mode; + vdata->cycle_last = tk->clock->cycle_last; + vdata->mask = tk->clock->mask; + vdata->mult = tk->mult; + vdata->shift = tk->shift; vdata->wall_time_sec = tk->xtime_sec; vdata->wall_time_snsec = tk->xtime_nsec;
arch/x86/vdso/vclock_gettime.c +62 -31
232 lines not on this page · raw message
arch/x86/vdso/vdso32/vclock_gettime.c +11 -0
@@ -2,6 +2,12 @@#ifdef CONFIG_X86_64 +typedef signed long long __kernel_long_t ; +typedef unsigned long long __kernel_ulong_t ; +#define __kernel_long_t __kernel_long_t + +#include <generated/asm/unistd_32_ia32.h> +#define _ASM_X86_PAGE_H#define __pa (x) 0
@@ -10,6 +16,11 @@#undef CONFIG_ILLEGAL_POINTER_VALUE#define CONFIG_ILLEGAL_POINTER_VALUE 0 +#else + +#define __NR_ia32_clock_gettime __NR_clock_gettime +#define __NR_ia32_gettimeofday __NR_gettimeofday +#endif #include "../vclock_gettime.c"
include/uapi/linux/time.h +1 -1
@@ -8,7 +8,7 @@#define _STRUCT_TIMESPECstruct timespec {__kernel_time_t tv_sec;/* seconds */ -long tv_nsec;/* nanoseconds */ +__kernel_long_t tv_nsec;/* nanoseconds */ };#endif
Signature · 1 line
1.8.5.3
Re: [PATCH v11 9/9] Add 32 bit VDSO time support for 64 bit kernel
Quoted 52 lines · On Tue, Feb 4, 2014 at 2:15 AM, <[email protected]> wrote:
From: Stefani Seibold <[email protected]>
This patch add the VDSO time support for the IA32 Emulation Layer.
Due the nature of the kernel headers and the LP64 compiler where the size of a long and a pointer differs against a 32 bit compiler, there is a lot of type hacking necessary.
The
vsyscall_gtod_datastruture must be a little bit rearranged, to serve for 32- and 64-bit code access.Signed-off-by Stefani Seibold <[email protected]>--- arch/x86/include/asm/vgtod.h | 23 +++++---- arch/x86/kernel/vsyscall_gtod.c | 10 ++-- arch/x86/vdso/vclock_gettime.c | 93 +++++++++++++++++++++++------------ arch/x86/vdso/vdso32/vclock_gettime.c | 11 +++++ include/uapi/linux/time.h | 2 +- 5 files changed, 92 insertions(+), 47 deletions(-)arch/x86/include/asm/vgtod.h +10
-10@@ -4,16 +4,16 @@#include <asm/vsyscall.h> #include <linux/clocksource.h> -struct vsyscall_gtod_data { -seqcount_t seq; - -struct {/* extract of a clocksource struct */ -int vclock_mode; -cycle_t cycle_last; -cycle_t mask; -u32 mult; -u32 shift; - } clock; +/* +* vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time +* so the structure must be packed +*/ +struct __attribute__((packed))vsyscall_gtod_data { +int vclock_mode; +cycle_t cycle_last; +cycle_t mask; +u32 mult; +u32 shift;/* open coded 'struct timespec' */ time_t wall_time_sec;
arch/x86/include/asm/vgtod.h struct vsyscall_gtod_data { · line 29
Quoted 6 lines
arch/x86/include/asm/vgtod.h +3
-0@@ -24,6 +24,9 @@ struct vsyscall_gtod_data {struct timezone sys_tz;struct timespec wall_time_coarse;struct timespec monotonic_time_coarse; + +/* must be at the end, because the size depends on the kernel config */ +seqcount_t seq;
Please don't move this. You are probably increasing the number of cachelines needed to do vclock_gettime if lockdep is off. (vclock_gettime with CLOCK_REALTIME or CLOCK_MONOTONIC is probably the most important case.) I'd much rather this be just 'unsigned seq' or 'u32 seq' and stay at the beginning.
arch/x86/include/asm/vgtod.h notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) · line 81
Quoted 13 lines
arch/x86/include/asm/vgtod.h +9
-0@@ -57,6 +70,18 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) u8 hpet_page __attribute__((visibility("hidden"))); +struct api_timeval { +s32 tv_sec;/* seconds */ +s32 tv_usec;/* microseconds */ +}; + +struct api_timespec { +s32 tv_sec;/* seconds */ +s32 tv_nsec;/* microseconds */ +};
These ought to be unifiable between 32-bit and 64-bit. Isn't 'long tv_sec; long tv_nsec;' correct in all cases?
--Andy