Skip to contentSkip to conversation

Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu

PATCH New Andi Kleen 1 participant 1 message Nov 17, 2006

Subsystems (2)
lore
Share & tools
lore

Andi Kleen

author
#0 lore raw

PATCH 1/1 Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu

[Patch for review. Any comments and testing appreciated. However with the so far missing glibc interface it can be only tested with custom test programs.]

This implements new vDSO for x86-64. The concept is similar to the existing vDSOs on i386 and PPC. x86-64 has had static vsyscalls before, but these are not flexible enough anymore.

A vDSO is a ELF shared library supplied by the kernel that is mapped into user address space. The vDSO mapping is randomized for each process for security reasons.

Doing this was needed for clock_gettime, because clock_gettime always needs a syscall fallback and having one at a fixed address would have made buffer overflow exploits too easy to write.

The vdso can be disabled with vdso=0

It currently includes a new gettimeofday implemention and optimized clock_gettime(). The gettimeofday implementation is slightly faster than the one in the old vsyscall. clock_gettime is significantly faster than the syscall for CLOCK_MONOTONIC and CLOCK_REALTIME.

The new calls are generally faster than the old vsyscall.

On a P4 system (v* = vDSO, other glibc):

All numbers in cycles

vgetcpu(&cpu, &node, NULL) took 52
old_vgetcpu(&cpu, &node, NULL) took 52
[ Essentially the same code. No difference ] gettimeofday(&tv, NULL) took 513
vgettimeofday(&tv, NULL) took 335
[ I think that the new vgettimeofday is faster is because
it avoids a division and is somewhat cleaner than the old code. I plan to port these improvements over to the old gettimeofday
so this difference should disappear ]
clock_gettime(CLOCK_REALTIME, &ts) took 1454 vclock_gettime(CLOCK_REALTIME, &ts) took 305 clock_gettime(CLOCK_MONOTONIC, &ts) took 1616 vclock_gettime(CLOCK_MONOTONIC, &ts) took 309
[ Difference between system call and using the ring 3 fast path ] vclock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) took 1673 clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) took 338
[ For clock_gettime(CLOCK_PROCESS_CPUTIME_ID) it is right now slower,
because glibc seems to do something magic that doesn't require system calls. The vDSO would do a system call to get the information. From what i can see the information glibc returns is dubious so this might be actually a glibc weakness (I don't know how it can figure out the per process CPU time without asking the kernel). The returned values also are different. ]

Advantages over the old x86-64 vsyscalls:
- Extensible
- Randomized
- Cleaner
- Easier to virtualize (the old static address range previously causes overhead e.g. for Xen because it has to create special page tables for it)

Weak points:
- This does some tricks with binutils. Might break on different versions (tested with gcc 4.0 / binutils 2.6.16.91.0.2 from SUSE 10.0)
- glibc support still to be written
- The vDSO varies in the full mmap range (2/3*TASK_SIZE-TASK_SIZE). This might be inconvenient for some virtual address space hungry applications because it is in the middle of the address space. Should be restricted to a smaller range.

The VM interface is partly based on Ingo Molnar's i386 version.

TBD fix address space range issue

---
 Documentation/kernel-parameters.txt |    2 
 Documentation/x86_64/mm.txt         |    2 
 arch/x86_64/Makefile                |    3 
 arch/x86_64/ia32/ia32_binfmt.c      |    1 
 arch/x86_64/kernel/time.c           |   10 --
 arch/x86_64/kernel/vmlinux.lds.S    |   10 ++
 arch/x86_64/mm/init.c               |   17 ++++
 arch/x86_64/vdso/Makefile           |   48 +++++++++++
 arch/x86_64/vdso/vclock_gettime.c   |  145 ++++++++++++++++++++++++++++++++++++
 arch/x86_64/vdso/vdso-note.S        |   25 ++++++
 arch/x86_64/vdso/vdso-start.S       |    2 
 arch/x86_64/vdso/vdso.S             |    2 
 arch/x86_64/vdso/vdso.lds.S         |   75 ++++++++++++++++++
 arch/x86_64/vdso/vextern.h          |   21 +++++
 arch/x86_64/vdso/vgetcpu.c          |   49 ++++++++++++
 arch/x86_64/vdso/vma.c              |  117 +++++++++++++++++++++++++++++
 arch/x86_64/vdso/vvar.c             |   10 ++
 include/asm-x86_64/auxvec.h         |    2 
 include/asm-x86_64/elf.h            |   13 +++
 include/asm-x86_64/mmu.h            |    1 
 include/asm-x86_64/pgtable.h        |    6 +
 include/asm-x86_64/timex.h          |    7 +
 include/asm-x86_64/vsyscall.h       |    8 +
 23 files changed, 565 insertions(+), 11 deletions(-)
Documentation/x86_64/mm.txt +1 -1
Index: linux/Documentation/x86_64/mm.txt
===================================================================
@@ -12,7 +12,7 @@ ffffc20000000000 - ffffe1ffffffffff (=45
 ... unused hole ...
 ffffffff80000000 - ffffffff82800000 (=40MB)   kernel text mapping, from phys 0
 ... unused hole ...
-ffffffff88000000 - fffffffffff00000 (=1919MB) module mapping space
+ffffffff88000000 - ffffffffff600000           module mapping space
 
 The direct mapping covers all memory in the system upto the highest
 memory address (this means in some cases it can also include PCI memory
arch/x86_64/ia32/ia32_binfmt.c +1 -0
Index: linux/arch/x86_64/ia32/ia32_binfmt.c
===================================================================
@@ -33,6 +33,7 @@
 
 int sysctl_vsyscall32 = 1;
 
+#undef ARCH_DLINFO
 #define ARCH_DLINFO do {  \
 	if (sysctl_vsyscall32) { \
 	NEW_AUX_ENT(AT_SYSINFO, (u32)(u64)VSYSCALL32_VSYSCALL); \
arch/x86_64/kernel/vmlinux.lds.S +10 -0
Index: linux/arch/x86_64/kernel/vmlinux.lds.S
===================================================================
@@ -111,6 +111,9 @@ SECTIONS
   .xtime : AT(VLOAD(.xtime)) { *(.xtime) }
   xtime = VVIRT(.xtime);
 
+  .wall_to_monotonic : AT(VLOAD(.wall_to_monotonic)) { *(.wall_to_monotonic) }
+  wall_to_monotonic = VVIRT(.wall_to_monotonic);
+
   . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
   .jiffies : AT(VLOAD(.jiffies)) { *(.jiffies) }
   jiffies = VVIRT(.jiffies);
@@ -159,6 +162,13 @@ SECTIONS
   . = ALIGN(4096);
   __smp_alt_end = .;
 
+  /* Blob that is mapped into user space */
+  . = ALIGN(4096);
+  vdso_start = . ;
+  .vdso  : AT(ADDR(.vdso) - LOAD_OFFSET) { *(.vdso) }
+  . = ALIGN(4096);
+  vdso_end = .;
+
   . = ALIGN(4096);		/* Init code and data */
   __init_begin = .;
   .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
arch/x86_64/kernel/time.c +3 -7
Index: linux/arch/x86_64/kernel/time.c
===================================================================
@@ -58,13 +58,6 @@ DEFINE_SPINLOCK(i8253_lock);
 int nohpet __initdata = 0;
 static int notsc __initdata = 0;
 
-#define USEC_PER_TICK (USEC_PER_SEC / HZ)
-#define NSEC_PER_TICK (NSEC_PER_SEC / HZ)
-#define FSEC_PER_TICK (FSEC_PER_SEC / HZ)
-
-#define NS_SCALE	10 /* 2^10, carefully chosen */
-#define US_SCALE	32 /* 2^32, arbitralrily chosen */
-
 unsigned int cpu_khz;					/* TSC clocks / usec, not used here */
 EXPORT_SYMBOL(cpu_khz);
 static unsigned long hpet_period;			/* fsecs / HPET clock */
@@ -79,6 +72,7 @@ struct vxtime_data __vxtime __section_vx
 volatile unsigned long __jiffies __section_jiffies = INITIAL_JIFFIES;
 struct timespec __xtime __section_xtime;
 struct timezone __sys_tz __section_sys_tz;
+struct timespec __wall_to_monotonic __section_wall_to_monotonic;
 
 /*
  * do_gettimeoffset() returns microseconds since last timer interrupt was
@@ -464,6 +458,7 @@ static unsigned int cyc2ns_scale __read_
 
 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
 {
+	vxtime.cyc2ns_scale =
 	cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / cpu_khz;
 }
 
@@ -998,6 +993,7 @@ void time_init_gtod(void)
 	printk(KERN_INFO "time.c: Detected %d.%03d MHz processor.\n",
 		cpu_khz / 1000, cpu_khz % 1000);
 	vxtime.quot = (USEC_PER_SEC << US_SCALE) / vxtime_hz;
+	vxtime.quot_nsec = (NSEC_PER_SEC << NS_SCALE) / vxtime_hz;
 	vxtime.tsc_quot = (USEC_PER_MSEC << US_SCALE) / cpu_khz;
 	vxtime.last_tsc = get_cycles_sync();
 
arch/x86_64/mm/init.c +17 -0
Index: linux/arch/x86_64/mm/init.c
===================================================================
@@ -152,6 +152,14 @@ static __init void set_pte_phys(unsigned
 	__flush_tlb_one(vaddr);
 }
 
+void __init
+set_kernel_map(void *vaddr,unsigned long len,unsigned long phys,pgprot_t prot)
+{
+	void *end = vaddr + ALIGN(len, PAGE_SIZE);
+	for (; vaddr < end; vaddr += PAGE_SIZE, phys += PAGE_SIZE)
+		set_pte_phys((unsigned long)vaddr, phys, prot);
+}
+
 /* NOTE: this is meant to be run only at boot */
 void __init 
 __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
@@ -773,3 +781,12 @@ int in_gate_area_no_task(unsigned long a
 {
 	return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
 }
+
+const char *arch_vma_name(struct vm_area_struct *vma)
+{
+	if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
+		return "[vdso]";
+	if (vma == &gate_vma)
+		return "[vsyscall]";
+	return NULL;
+}
arch/x86_64/vdso/vdso-note.S new file +25 -0
Index: linux/arch/x86_64/vdso/vdso-note.S
===================================================================
@@ -0,0 +1,25 @@
+/*
+ * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text.
+ * Here we can supply some information useful to userland.
+ */
+
+#include <linux/uts.h>
+#include <linux/version.h>
+
+#define ASM_ELF_NOTE_BEGIN(name, flags, vendor, type)			      \
+	.section name, flags;						      \
+	.balign 4;							      \
+	.long 1f - 0f;		/* name length */			      \
+	.long 3f - 2f;		/* data length */			      \
+	.long type;		/* note type */				      \
+0:	.asciz vendor;		/* vendor name */			      \
+1:	.balign 4;							      \
+2:
+
+#define ASM_ELF_NOTE_END						      \
+3:	.balign 4;		/* pad out section */			      \
+	.previous
+
+	ASM_ELF_NOTE_BEGIN(".note.kernel-version", "a", UTS_SYSNAME, 0)
+	.long LINUX_VERSION_CODE
+	ASM_ELF_NOTE_END
arch/x86_64/vdso/vdso.lds.S new file +75 -0

75 lines not on this page · raw message

arch/x86_64/vdso/Makefile new file +48 -0

48 lines not on this page · raw message

arch/x86_64/vdso/vclock_gettime.c new file +145 -0

145 lines not on this page · raw message

arch/x86_64/vdso/vma.c new file +117 -0

117 lines not on this page · raw message

arch/x86_64/vdso/vdso.S new file +2 -0
Index: linux/arch/x86_64/vdso/vdso.S
===================================================================
@@ -0,0 +1,2 @@
+	.section ".vdso","a"
+	.incbin "arch/x86_64/vdso/vdso.so"
arch/x86_64/vdso/vdso-start.S new file +2 -0
Index: linux/arch/x86_64/vdso/vdso-start.S
===================================================================
@@ -0,0 +1,2 @@
+	.globl vdso_kernel_start
+vdso_kernel_start:
arch/x86_64/Makefile +2 -1
Index: linux/arch/x86_64/Makefile
===================================================================
@@ -82,7 +82,8 @@ head-y := arch/x86_64/kernel/head.o arch
 libs-y 					+= arch/x86_64/lib/
 core-y					+= arch/x86_64/kernel/ \
 					   arch/x86_64/mm/ \
-					   arch/x86_64/crypto/
+					   arch/x86_64/crypto/ \
+					   arch/x86_64/vdso/
 core-$(CONFIG_IA32_EMULATION)		+= arch/x86_64/ia32/
 drivers-$(CONFIG_PCI)			+= arch/x86_64/pci/
 drivers-$(CONFIG_OPROFILE)		+= arch/x86_64/oprofile/
include/asm-x86_64/pgtable.h +6 -0
Index: linux/include/asm-x86_64/pgtable.h
===================================================================
@@ -419,6 +419,9 @@ void vmalloc_sync_all(void);
 
 extern int kern_addr_valid(unsigned long addr); 
 
+extern void set_kernel_map(void *vaddr, unsigned long len,
+			   unsigned long phys, pgprot_t prot);
+
 #define io_remap_pfn_range(vma, vaddr, pfn, size, prot)		\
 		remap_pfn_range(vma, vaddr, pfn, size, prot)
 
@@ -447,4 +450,7 @@ extern int kern_addr_valid(unsigned long
 #define __HAVE_ARCH_PTE_SAME
 #include <asm-generic/pgtable.h>
 
+extern void
+fix_set_pte_phys(unsigned long vaddr, unsigned long phys, pgprot_t prot);
+
 #endif /* _X86_64_PGTABLE_H */
include/asm-x86_64/mmu.h +1 -0
Index: linux/include/asm-x86_64/mmu.h
===================================================================
@@ -15,6 +15,7 @@ typedef struct { 
 	rwlock_t ldtlock; 
 	int size;
 	struct semaphore sem; 
+	void *vdso;
 } mm_context_t;
 
 #endif
include/asm-x86_64/vsyscall.h +7 -1
Index: linux/include/asm-x86_64/vsyscall.h
===================================================================
@@ -23,6 +23,7 @@ enum vsyscall_num {
 #define __section_sysctl_vsyscall __attribute__ ((unused, __section__ (".sysctl_vsyscall"), aligned(16)))
 #define __section_xtime __attribute__ ((unused, __section__ (".xtime"), aligned(16)))
 #define __section_xtime_lock __attribute__ ((unused, __section__ (".xtime_lock"), aligned(16)))
+#define __section_wall_to_monotonic __attribute__ ((unused, __section__ (".wall_to_monotonic"), aligned(16)))
 
 #define VXTIME_TSC	1
 #define VXTIME_HPET	2
@@ -33,11 +34,13 @@ enum vsyscall_num {
 
 struct vxtime_data {
 	long hpet_address;	/* HPET base address */
-	int last;
 	unsigned long last_tsc;
 	long quot;
 	long tsc_quot;
+	int last;
 	int mode;
+	long quot_nsec;
+	long cyc2ns_scale;
 };
 
 #define hpet_readl(a)           readl((const void __iomem *)fix_to_virt(FIX_HPET_BASE) + a)
@@ -49,7 +52,9 @@ extern int __vgetcpu_mode;
 extern struct timespec __xtime;
 extern volatile unsigned long __jiffies;
 extern struct timezone __sys_tz;
+extern int __sysctl_vsyscall;
 extern seqlock_t __xtime_lock;
+extern struct timespec __wall_to_monotonic;
 
 /* kernel space (writeable) */
 extern struct vxtime_data vxtime;
@@ -57,6 +62,7 @@ extern int vgetcpu_mode;
 extern struct timezone sys_tz;
 extern int sysctl_vsyscall;
 extern seqlock_t xtime_lock;
+extern struct timespec wall_to_monotonic;
 
 extern int sysctl_vsyscall;
 
include/asm-x86_64/auxvec.h +2 -0
Index: linux/include/asm-x86_64/auxvec.h
===================================================================
@@ -1,4 +1,6 @@
 #ifndef __ASM_X86_64_AUXVEC_H
 #define __ASM_X86_64_AUXVEC_H
 
+#define AT_SYSINFO_EHDR		33
+
 #endif
include/asm-x86_64/timex.h +7 -0
Index: linux/include/asm-x86_64/timex.h
===================================================================
@@ -46,4 +46,11 @@ extern int read_current_timer(unsigned l
 
 extern struct vxtime_data vxtime;
 
+#define USEC_PER_TICK (USEC_PER_SEC / HZ)
+#define NSEC_PER_TICK (NSEC_PER_SEC / HZ)
+#define FSEC_PER_TICK (FSEC_PER_SEC / HZ)
+
+#define NS_SCALE	10 /* 2^10, carefully chosen */
+#define US_SCALE	32 /* 2^32, arbitralrily chosen */
+
 #endif
include/asm-x86_64/elf.h +13 -0
Index: linux/include/asm-x86_64/elf.h
===================================================================
@@ -163,6 +163,19 @@ extern int dump_task_fpu (struct task_st
 /* 1GB for 64bit, 8MB for 32bit */
 #define STACK_RND_MASK (test_thread_flag(TIF_IA32) ? 0x7ff : 0x3fffff)
 
+
+#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
+struct linux_binprm;
+extern int arch_setup_additional_pages(struct linux_binprm *bprm,
+                                       int executable_stack);
+
+extern int vdso_enabled;
+
+#define ARCH_DLINFO						\
+do if (vdso_enabled) {						\
+	NEW_AUX_ENT(AT_SYSINFO_EHDR,(unsigned long)current->mm->context.vdso);\
+} while (0)
+
 #endif
 
 #endif
arch/x86_64/vdso/vextern.h new file +21 -0
Index: linux/arch/x86_64/vdso/vextern.h
===================================================================
@@ -0,0 +1,21 @@
+#ifndef VEXTERN
+#include <asm/vsyscall.h>
+#define VEXTERN(x) \
+	extern typeof(x) *vdso_ ## x __attribute__((visibility("hidden")));
+#endif
+
+#define VMAGIC 0xfeedbabeabcdefabUL
+
+/* Any kernel variables used in the vDSO must be exported in the main
+   kernel's vmlinux.lds.S/vsyscall.h/proper __section and
+   put into vextern.h and be referenced as a pointer with vdso prefix.
+   The main kernel later fills in the values.   */
+
+VEXTERN(xtime_lock)
+VEXTERN(xtime)
+VEXTERN(vxtime)
+VEXTERN(jiffies)
+VEXTERN(sysctl_vsyscall)
+VEXTERN(wall_to_monotonic)
+VEXTERN(vgetcpu_mode)
+VEXTERN(sys_tz)
arch/x86_64/vdso/vgetcpu.c new file +49 -0

49 lines not on this page · raw message

arch/x86_64/vdso/vvar.c new file +10 -0
Index: linux/arch/x86_64/vdso/vvar.c
===================================================================
@@ -0,0 +1,10 @@
+/* Define vDSO variables. These are part of the vDSO */
+#include <linux/kernel.h>
+#include <linux/time.h>
+#include <asm/vsyscall.h>
+#include <asm/timex.h>
+
+#define __hidden __attribute__((visibility("hidden")))
+
+#define VEXTERN(x) typeof (__ ## x) *vdso_ ## x __hidden = (void *)VMAGIC;
+#include "vextern.h"
Documentation/kernel-parameters.txt +1 -1
Index: linux/Documentation/kernel-parameters.txt
===================================================================
@@ -1682,7 +1682,7 @@ and is between 256 and 4096 characters. 
 	usbhid.mousepoll=
 			[USBHID] The interval which mice are to be polled at.
 
-	vdso=		[IA-32]
+	vdso=		[IA32,X86-64]
 			vdso=1: enable VDSO (default)
 			vdso=0: disable VDSO mapping