Add a thread cpu time implementation to vDSO (v2)
cover Add a thread cpu time implementation to vDSO (v2)
Changes since last rev:
* Drop the direct vsyscall (hope clock_gettime_ns() addresses the gap)
* Add a compile time check for vvar page overflow
* Handle return value from the syscall
* Pack the VVAR page more tightly
* Handle NR_CPUS > 64
Arun Sharma (1):
Extend VVAR support to multiple pages
Kumar Sundararajan (1):
Add a thread cpu time implementation to vDSO
Documentation/cgroups/timer_slack.txt | 81 ---------------- drivers/Makefile | 3 - drivers/fb/Makefile | 1 - drivers/fb/tsc/Makefile | 2 - drivers/fb/tsc/tsc.c | 29 ------ fs/select.c | 7 +- include/linux/cgroup_subsys.h | 7 -- include/linux/prctl.h | 6 - include/linux/sched.h | 11 -- include/linux/topology.h | 2 - init/Kconfig | 8 -- kernel/Makefile | 1 - kernel/cgroup_timer_slack.c | 169 --------------------------------- kernel/fork.c | 4 - kernel/futex.c | 4 +- kernel/hrtimer.c | 2 +- kernel/sched_fair.c | 54 ----------- kernel/sys.c | 3 - 18 files changed, 8 insertions(+), 386 deletions(-) delete mode 100644 Documentation/cgroups/timer_slack.txt delete mode 100644 drivers/fb/Makefile delete mode 100644 drivers/fb/tsc/Makefile delete mode 100644 drivers/fb/tsc/tsc.c delete mode 100644 kernel/cgroup_timer_slack.c
Signature · 1 line
1.7.4
PATCH 1/2 Extend VVAR support to multiple pages
--- arch/x86/include/asm/fixmap.h | 1 + arch/x86/include/asm/vvar.h | 3 ++- arch/x86/kernel/vmlinux.lds.S | 2 +- arch/x86/kernel/vsyscall_64.c | 1 + 4 files changed, 5 insertions(+), 2 deletions(-)
arch/x86/include/asm/fixmap.h +1 -0
@@ -78,6 +78,7 @@ enum fixed_addresses {VSYSCALL_LAST_PAGE ,VSYSCALL_FIRST_PAGE =VSYSCALL_LAST_PAGE + ((VSYSCALL_END -VSYSCALL_START ) >>PAGE_SHIFT ) -1 , +VVAR_PAGE2 ,VVAR_PAGE , VSYSCALL_HPET, #endif
arch/x86/include/asm/vvar.h +2 -1
@@ -17,7 +17,8 @@*/ /* Base address of vvars. This is not ABI. */ -#define VVAR_ADDRESS (-10*1024*1024 - 4096) +#define VVAR_NUM_PAGES 2 +#define VVAR_ADDRESS (-10*1024*1024 - VVAR_NUM_PAGES * 4096)#if defined(__VVAR_KERNEL_LDS)
arch/x86/kernel/vmlinux.lds.S +1 -1
@@ -171,7 +171,7 @@ SECTIONS } :data - . = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE); + . = ALIGN(__vvar_page + VVAR_NUM_PAGES*PAGE_SIZE, PAGE_SIZE); #endif /* CONFIG_X86_64 */
arch/x86/kernel/vsyscall_64.c +1 -0
@@ -282,6 +282,7 @@ void __init map_vsyscall(void) (unsigned long)VSYSCALL_START);__set_fixmap (VVAR_PAGE , physaddr_vvar_page,PAGE_KERNEL_VVAR ); +__set_fixmap (VVAR_PAGE2 , physaddr_vvar_page +PAGE_SIZE ,PAGE_KERNEL_VVAR );BUILD_BUG_ON ((unsigned long )__fix_to_virt (VVAR_PAGE ) != (unsigned long )VVAR_ADDRESS ); }
Signature · 1 line
1.7.4
PATCH 2/2 Add a thread cpu time implementation to vDSO
In reply to Arun SharmaFrom: Kumar Sundararajan <[email protected]>
This primarily speeds up clock_gettime(CLOCK_THREAD_CPUTIME_ID, ..) We use the following method to compute the thread cpu time:
t0 = process start
t1 = most recent context switch time
t2 = time at which the vsyscall is invoked
thread_cpu_time = sum(time slices between t0 to t1) + (t2 - t1)
= current->se.sum_exec_runtime + now - sched_clock()
At context switch time We stash away
adj_sched_time = sum_exec_runtime - sched_clock()
in a per-cpu struct in the VVAR page (which has now been extended to two pages) and then compute
thread_cpu_time = adj_sched_time + now
All computations are done in nanosecs on systems where TSC is stable. If TSC is unstable, we fallback to a regular syscall.
Benchmark data:
Baseline:
for (i = 0; i < 100000000; i++) {
clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
sum += ts.tv_sec * NSECS_PER_SEC + ts.tv_nsec;
}vclock_gettime:
vclock_gettime = dlsym(vdso, "__vdso_clock_gettime");
for (i = 0; i < 100000000; i++) {
(*vclock_gettime)(CLOCK_THREAD_CPUTIME_ID, &ts);
sum += ts.tv_sec * NSECS_PER_SEC + ts.tv_nsec;
}Baseline: 19.34 secsvclock_gettime: 4.74 secs
This should speed up profilers that need to query thread cpu time a lot to do fine-grained timestamps.
No statistically significant regression was detected on x86_64 context switch code. Most archs that don't support vsyscalls will have this code disabled via jump labels.
--- arch/x86/include/asm/timer.h | 18 +++++++++++++----- arch/x86/include/asm/vvar.h | 1 + arch/x86/kernel/tsc.c | 16 ++++++++++++++++ arch/x86/kernel/vsyscall_64.c | 1 + arch/x86/vdso/vclock_gettime.c | 36 ++++++++++++++++++++++++++++++++++-- arch/x86/vdso/vma.c | 5 +++++ include/linux/jiffies.h | 16 ++++++++++++++++ kernel/sched.c | 8 ++++++++ 8 files changed, 94 insertions(+), 7 deletions(-)
arch/x86/include/asm/timer.h +13 -5
@@ -55,19 +55,27 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);#define CYC2NS_SCALE_FACTOR 10/* 2^10, carefully chosen */ -static inline unsigned long long __cycles_2_ns (unsigned long long cyc) +static inline unsigned long long ___cycles_2_ns (unsigned long long cyc, +unsigned long long scale, +unsigned long long offset) {unsigned long long quot;unsigned long long rem; -int cpu =smp_processor_id (); -unsigned long long ns =per_cpu (cyc2ns_offset, cpu); +unsigned long long ns = offset; quot = (cyc >>CYC2NS_SCALE_FACTOR ); rem = cyc & ((1ULL <<CYC2NS_SCALE_FACTOR ) -1 ); - ns += quot *per_cpu (cyc2ns, cpu) + - ((rem *per_cpu (cyc2ns, cpu)) >>CYC2NS_SCALE_FACTOR ); + ns += quot * scale + ((rem * scale) >>CYC2NS_SCALE_FACTOR );return ns; } +static inline unsigned long long __cycles_2_ns (unsigned long long cyc) +{ +int cpu =smp_processor_id (); +unsigned long long offset =per_cpu (cyc2ns_offset, cpu); +unsigned long long scale =per_cpu (cyc2ns, cpu); +return ___cycles_2_ns (cyc, scale, offset); +} +static inline unsigned long long cycles_2_ns (unsigned long long cyc) {unsigned long long ns;
arch/x86/include/asm/vvar.h +1 -0
@@ -47,5 +47,6 @@ DECLARE_VVAR(0, volatile unsigned long, jiffies) DECLARE_VVAR(16, int, vgetcpu_mode) DECLARE_VVAR(128, struct vsyscall_gtod_data, vsyscall_gtod_data) +DECLARE_VVAR(256, struct vcpu_data, vcpu_data)#undef DECLARE_VVAR
arch/x86/kernel/tsc.c +16 -0
@@ -623,6 +623,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)if (cpu_khz) { *scale = (NSEC_PER_MSEC <<CYC2NS_SCALE_FACTOR )/cpu_khz; *offset = ns_now - (tsc_now * *scale >>CYC2NS_SCALE_FACTOR ); + vcpu_data.vpercpu[cpu].cyc2ns = *scale; + vcpu_data.vpercpu[cpu].cyc2ns_offset = *offset; }sched_clock_idle_wakeup_event (0 );
@@ -786,6 +788,8 @@ void mark_tsc_unstable(char *reason) tsc_unstable =1 ; sched_clock_stable =0 ;disable_sched_clock_irqtime (); + vcpu_data.thread_cputime_disabled =1 ; +jump_label_dec (&vcpu_data_enabled);printk (KERN_INFO "Marking TSC unstable due to %s\n" , reason);/* Change only the rating, when not registered */ if (clocksource_tsc.mult)
@@ -943,6 +947,15 @@ static int __init init_tsc_clocksource(void)*/ device_initcall (init_tsc_clocksource); +/* Should be optimized away at compile time */ +static noinline int check_vvar_overflow(void) +{ +long vcpu_data_offset = (long ) vvaraddr_vcpu_data -VVAR_ADDRESS ; +size_t size = (vcpu_data_offset +sizeof (vcpu_data) + +sizeof (struct vpercpu_data ) *CONFIG_NR_CPUS ); +return (size >VVAR_NUM_PAGES *PAGE_SIZE ); +} +void __init tsc_init (void ) {u64 lpj;
@@ -979,6 +992,9 @@ void __init tsc_init(void)/* now allow native_sched_clock() to use rdtsc */ tsc_disabled =0 ; + vcpu_data.tsc_khz = tsc_khz; + vcpu_data.thread_cputime_disabled =check_vvar_overflow () + || !cpu_has (&boot_cpu_data,X86_FEATURE_RDTSCP );if (!no_sched_irq_time)enable_sched_clock_irqtime ();
arch/x86/kernel/vsyscall_64.c +1 -0
@@ -56,6 +56,7 @@ DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) = { .lock = __SEQLOCK_UNLOCKED(__vsyscall_gtod_data.lock), }; +DEFINE_VVAR(struct vcpu_data, vcpu_data);static enum {EMULATE ,NATIVE ,NONE } vsyscall_mode =NATIVE ;
arch/x86/vdso/vclock_gettime.c +32 -0
@@ -18,6 +18,7 @@#include <asm/vsyscall.h> #include <asm/fixmap.h> #include <asm/vgtod.h> +#include <asm/timer.h> #include <asm/timex.h> #include <asm/hpet.h> #include <asm/unistd.h>
@@ -154,8 +155,32 @@ notrace static noinline int do_monotonic_coarse(struct timespec *ts)return 0 ; } +notrace static noinline unsigned long do_thread_cpu_time(void) +{ +unsigned int p; +u_int64_t tscval; +unsigned long long adj_sched_time, scale, offset; +const struct vcpu_data *vp = &VVAR (vcpu_data); +int cpu; + +do { +native_read_tscp (&p); + cpu = p &0xfff ; + adj_sched_time = vp->vpercpu[cpu].adj_sched_time; + scale = vp->vpercpu[cpu].cyc2ns; + offset = vp->vpercpu[cpu].cyc2ns_offset; +rdtscpll (tscval, p); + cpu = p &0xfff ; + }while (unlikely (adj_sched_time != vp->vpercpu[cpu].adj_sched_time)); + +return ___cycles_2_ns (tscval, scale, offset) + adj_sched_time; +} + notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) { +unsigned long ns; +const struct vcpu_data *vp = &VVAR (vcpu_data); +switch (clock) {case CLOCK_REALTIME :if (likely (gtod->clock.vclock_mode !=VCLOCK_NONE ))
@@ -169,6 +194,13 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)return do_realtime_coarse (ts);case CLOCK_MONOTONIC_COARSE :return do_monotonic_coarse (ts); +case CLOCK_THREAD_CPUTIME_ID : +if (vp->thread_cputime_disabled) +break ; + ns =do_thread_cpu_time (); + ts->tv_sec =0 ; +timespec_add_ns (ts, ns); +return 0 ; }return vdso_fallback_gettime (clock, ts);
arch/x86/vdso/vma.c +5 -0
@@ -10,6 +10,7 @@#include <linux/init.h> #include <linux/random.h> #include <linux/elf.h> +#include <linux/jump_label.h> #include <asm/vsyscall.h> #include <asm/vgtod.h> #include <asm/proto.h>
@@ -24,6 +25,8 @@ extern unsigned short vdso_sync_cpuid;extern struct page *vdso_pages[];static unsigned vdso_size; +struct jump_label_key vcpu_data_enabled; +static void __init patch_vdso (void *vdso,size_t len) {Elf64_Ehdr *hdr = vdso;
@@ -66,6 +69,8 @@ static int __init init_vdso(void)for (i =0 ; i < npages; i++) vdso_pages[i] =virt_to_page (vdso_start + i*PAGE_SIZE ); +if (!vcpu_data.thread_cputime_disabled) +jump_label_inc (&vcpu_data_enabled);return 0 ; }subsys_initcall (init_vdso);
include/linux/jiffies.h +16 -0
@@ -312,4 +312,20 @@ extern unsigned long nsecs_to_jiffies(u64 n);#define TIMESTAMP_SIZE 30 +struct vpercpu_data { +unsigned long long adj_sched_time; +unsigned long long cyc2ns_offset; +unsigned long cyc2ns; +}____cacheline_aligned ; + +struct vcpu_data { +unsigned int tsc_khz; +unsigned int thread_cputime_disabled; +struct vpercpu_data vpercpu[0 ]; +}; +extern struct vcpu_data vcpu_data; + +struct jump_label_key ; +extern struct jump_label_key vcpu_data_enabled; +#endif
kernel/sched.c +8 -0
@@ -3216,6 +3216,14 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)kprobe_flush_task (prev);put_task_struct (prev); } + +#if CONFIG_NR_CPUS <=64 +if (static_branch (&vcpu_data_enabled)) { +int cpu =smp_processor_id (); + vcpu_data.vpercpu[cpu].adj_sched_time = + current->se.sum_exec_runtime -sched_clock (); + } +#endif }#ifdef CONFIG_SMP
Signature · 1 line
1.7.4
Re: [PATCH 2/2] Add a thread cpu time implementation to vDSO
arch/x86/kernel/tsc.c static int __init init_tsc_clocksource(void) · line 957
Quoted 8 lines · Arun Sharma
arch/x86/kernel/tsc.c +8
-0+/* Should be optimized away at compile time */ +static noinline int check_vvar_overflow(void) +{ +long vcpu_data_offset = (long ) vvaraddr_vcpu_data -VVAR_ADDRESS ; +size_t size = (vcpu_data_offset +sizeof (vcpu_data) + +sizeof (struct vpercpu_data ) *CONFIG_NR_CPUS ); +return (size>VVAR_NUM_PAGES *PAGE_SIZE ); +}
Forgot to drop the "noinline" part (which I had added for debug purposes, so I could inspect the generated code). Fixed in my repo.
-Arun