cputime: Full dynticks task/cputime accounting v6
cover cputime: Full dynticks task/cputime accounting v6
Hi,
This version fixes everything I had on the TODO list. I'm just not very happy with the dependency on CONFIG_64BIT. It is necessary because cputime_t maps to unsigned long long in order to implement a per nanosec granularity fitting in a reasonable range. And 32 bits archs can't read 64 bits cputime_t safely (they read low and high part separately, this can race against an update that does an arithmetic carry after incrementing the low part). Usual utime/stime/gtime/... are fine because they are protected with the new seqcount. But kernel_cpustat is not protected on the read side.
I don't know what the right fix is. May be we could use per cpu spinlocks or seqlocks from both read/update side of kernel_cpustat. Hmm, or perhaps take curr_task(cpu)->vtime_seqlock before reading it. This requires rq lock though.
Anyway I believe that specific issue can wait for now. The patchset is already complicated enough and it depends on context_tracking that is only implemented on x86-64 yet. I guess once that gets merged and we get a 32 arch to support the context_tracking, we can start to think about that.
This set is pullable from:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
vtime/generic-v6
Changes since v5:
- Handle guest cputime
- Handle idle tasks. Those were special because they don't start from schedule() so they need their own initializations.
- Fix build error with CONFIG_IRQ_TIME_ACCOUNTING (thanks Wu Fenguang)
- Depend on 64 bits archs to make kern_cpustat reading to be atomic
Thanks.
---
Frederic Weisbecker (8):
context_tracking: Export context state for generic vtime
cputime: Librarize per nsecs resolution cputime definitions
cputime: Move default nsecs_to_cputime() to jiffies based cputime
file
cputime: Generic on-demand virtual cputime accounting
cputime: Allow dynamic switch between tick/virtual based cputime
accounting
cputime: Use accessors to read task cputime stats
kvm: Prepare to add generic guest entry/exit callbacks
cputime: Safely read cputime of full dynticks CPUs
arch/alpha/kernel/osf_sys.c | 6 +- arch/ia64/include/asm/cputime.h | 92 +-------- arch/ia64/include/asm/thread_info.h | 4 +- arch/ia64/include/asm/xen/minstate.h | 2 +- arch/ia64/kernel/asm-offsets.c | 2 +- arch/ia64/kernel/entry.S | 16 +- arch/ia64/kernel/fsys.S | 4 +- arch/ia64/kernel/head.S | 4 +- arch/ia64/kernel/ivt.S | 8 +- arch/ia64/kernel/minstate.h | 2 +- arch/ia64/kernel/time.c | 4 +- arch/powerpc/configs/chroma_defconfig | 2 +- arch/powerpc/configs/corenet64_smp_defconfig | 2 +- arch/powerpc/configs/pasemi_defconfig | 2 +- arch/powerpc/include/asm/cputime.h | 6 +- arch/powerpc/include/asm/lppaca.h | 2 +- arch/powerpc/include/asm/ppc_asm.h | 4 +- arch/powerpc/kernel/entry_64.S | 4 +- arch/powerpc/kernel/time.c | 4 +- arch/powerpc/platforms/pseries/dtl.c | 6 +- arch/powerpc/platforms/pseries/setup.c | 6 +- arch/s390/kernel/vtime.c | 6 +- arch/x86/kernel/apm_32.c | 11 +- drivers/isdn/mISDN/stack.c | 7 +- fs/binfmt_elf.c | 8 +- fs/binfmt_elf_fdpic.c | 7 +- fs/proc/array.c | 4 +- include/asm-generic/cputime.h | 66 +----- include/asm-generic/cputime_jiffies.h | 72 ++++++ include/asm-generic/cputime_nsecs.h | 104 +++++++++ include/linux/context_tracking.h | 28 +++ include/linux/hardirq.h | 4 +- include/linux/init_task.h | 11 + include/linux/kernel_stat.h | 2 +- include/linux/kvm_host.h | 55 ++++- include/linux/sched.h | 40 ++++ include/linux/tsacct_kern.h | 3 + include/linux/vtime.h | 59 ++++-- init/Kconfig | 23 ++- kernel/acct.c | 6 +- kernel/context_tracking.c | 41 ++-- kernel/cpu.c | 4 +- kernel/delayacct.c | 7 +- kernel/exit.c | 10 +- kernel/fork.c | 6 + kernel/posix-cpu-timers.c | 28 ++- kernel/sched/core.c | 1 + kernel/sched/cputime.c | 298 +++++++++++++++++++++++--- kernel/signal.c | 12 +- kernel/softirq.c | 6 +- kernel/time/tick-sched.c | 5 +- kernel/tsacct.c | 44 +++- 52 files changed, 840 insertions(+), 320 deletions(-) create mode 100644 include/asm-generic/cputime_jiffies.h create mode 100644 include/asm-generic/cputime_nsecs.h
Signature · 1 line
1.7.5.4
PATCH 1/8 context_tracking: Export context state for generic vtime
Export the context state: whether we run in user / kernel from the context tracking subsystem point of view.
This is going to be used by the generic virtual cputime accounting subsystem that is needed to implement the full dynticks.
--- include/linux/context_tracking.h | 28 ++++++++++++++++++++++++++++ kernel/context_tracking.c | 16 +--------------- 2 files changed, 29 insertions(+), 15 deletions(-)
include/linux/context_tracking.h +28 -0
@@ -3,12 +3,40 @@#ifdef CONFIG_CONTEXT_TRACKING #include <linux/sched.h> +#include <linux/percpu.h> + +struct context_tracking { +/* +* When active is false, probes are unset in order +* to minimize overhead: TIF flags are cleared +* and calls to user_enter/exit are ignored. This +* may be further optimized using static keys. +*/ +bool active; +enum { +IN_KERNEL =0 , +IN_USER , + } state; +}; + +DECLARE_PER_CPU(struct context_tracking, context_tracking); + +static inline bool context_tracking_in_user (void ) +{ +return __this_cpu_read (context_tracking.state) ==IN_USER ; +} + +static inline bool context_tracking_active (void ) +{ +return __this_cpu_read (context_tracking.active); +}extern void user_enter (void );extern void user_exit (void );extern void context_tracking_task_switch (struct task_struct *prev,struct task_struct *next);#else +static inline bool context_tracking_in_user (void ) {return false; }static inline void user_enter (void ) { }static inline void user_exit (void ) { } static inline void context_tracking_task_switch(struct task_struct *prev,
kernel/context_tracking.c +1 -15
@@ -17,24 +17,10 @@#include <linux/context_tracking.h> #include <linux/rcupdate.h> #include <linux/sched.h> -#include <linux/percpu.h> #include <linux/hardirq.h> -struct context_tracking { -/* -* When active is false, probes are unset in order -* to minimize overhead: TIF flags are cleared -* and calls to user_enter/exit are ignored. This -* may be further optimized using static keys. -*/ -bool active; -enum { -IN_KERNEL =0 , -IN_USER , - } state; -}; -static DEFINE_PER_CPU(struct context_tracking, context_tracking) = { +DEFINE_PER_CPU(struct context_tracking, context_tracking) = {#ifdef CONFIG_CONTEXT_TRACKING_FORCE .active = true,#endif
Signature · 1 line
1.7.5.4
PATCH 2/8 cputime: Librarize per nsecs resolution cputime definitions
In reply to Frederic WeisbeckerThe full dynticks cputime accounting that we'll soon introduce will rely on sched_clock(). And its clock can have a per nanosecond granularity.
To prepare for this, we need to have a cputime_t implementation that has this precision.
ia64 virtual cputime accounting already uses that granularity so all we need is to librarize its implementation in the asm generic headers.
Also librarize the default per jiffy granularity cputime_t as well so that we can easily pick either implementation depending on the cputime accounting config we choose.
--- arch/ia64/include/asm/cputime.h | 88 ++---------------------------- include/asm-generic/cputime.h | 62 +--------------------- include/asm-generic/cputime_jiffies.h | 66 ++++++++++++++++++++++ include/asm-generic/cputime_nsecs.h | 96 +++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 145 deletions(-) create mode 100644 include/asm-generic/cputime_jiffies.h create mode 100644 include/asm-generic/cputime_nsecs.h
arch/ia64/include/asm/cputime.h +4 -84
@@ -19,91 +19,11 @@#define __IA64_CPUTIME_H#ifndef CONFIG_VIRT_CPU_ACCOUNTING -#include <asm-generic/cputime.h> +# include <asm-generic/cputime.h> #else - -#include <linux/time.h> -#include <linux/jiffies.h> -#include <asm/processor.h> - -typedef u64 __nocast cputime_t ; -typedef u64 __nocast cputime64_t ; - -#define cputime_one_jiffy jiffies_to_cputime(1) - -/* -* Convert cputime <-> jiffies (HZ) -*/ -#define cputime_to_jiffies (__ct) \ - ((__force u64)(__ct) / (NSEC_PER_SEC / HZ)) -#define jiffies_to_cputime (__jif) \ - (__force cputime_t)((__jif) * (NSEC_PER_SEC / HZ)) -#define cputime64_to_jiffies64 (__ct) \ - ((__force u64)(__ct) / (NSEC_PER_SEC / HZ)) -#define jiffies64_to_cputime64 (__jif) \ - (__force cputime64_t)((__jif) * (NSEC_PER_SEC / HZ)) - -/* -* Convert cputime <-> microseconds -*/ -#define cputime_to_usecs (__ct) \ - ((__force u64)(__ct) / NSEC_PER_USEC) -#define usecs_to_cputime (__usecs) \ - (__force cputime_t)((__usecs) * NSEC_PER_USEC) -#define usecs_to_cputime64 (__usecs) \ - (__force cputime64_t)((__usecs) * NSEC_PER_USEC) - -/* -* Convert cputime <-> seconds -*/ -#define cputime_to_secs (__ct) \ - ((__force u64)(__ct) / NSEC_PER_SEC) -#define secs_to_cputime (__secs) \ - (__force cputime_t)((__secs) * NSEC_PER_SEC) - -/* -* Convert cputime <-> timespec (nsec) -*/ -static inline cputime_t timespec_to_cputime (const struct timespec *val) -{ -u64 ret = val->tv_sec *NSEC_PER_SEC + val->tv_nsec; -return (__force cputime_t ) ret; -} -static inline void cputime_to_timespec (const cputime_t ct,struct timespec *val) -{ - val->tv_sec = (__force u64 ) ct /NSEC_PER_SEC ; - val->tv_nsec = (__force u64 ) ct %NSEC_PER_SEC ; -} - -/* -* Convert cputime <-> timeval (msec) -*/ -static inline cputime_t timeval_to_cputime (struct timeval *val) -{ -u64 ret = val->tv_sec *NSEC_PER_SEC + val->tv_usec *NSEC_PER_USEC ; -return (__force cputime_t ) ret; -} -static inline void cputime_to_timeval (const cputime_t ct,struct timeval *val) -{ - val->tv_sec = (__force u64 ) ct /NSEC_PER_SEC ; - val->tv_usec = ((__force u64 ) ct %NSEC_PER_SEC ) /NSEC_PER_USEC ; -} - -/* -* Convert cputime <-> clock (USER_HZ) -*/ -#define cputime_to_clock_t (__ct) \ - ((__force u64)(__ct) / (NSEC_PER_SEC / USER_HZ)) -#define clock_t_to_cputime (__x) \ - (__force cputime_t)((__x) * (NSEC_PER_SEC / USER_HZ)) - -/* -* Convert cputime64 to clock. -*/ -#define cputime64_to_clock_t (__ct) \ - cputime_to_clock_t((__force cputime_t)__ct) - +# include <asm/processor.h> +# include <asm-generic/cputime-nsecs.h> extern void arch_vtime_task_switch (struct task_struct *tsk); -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* __IA64_CPUTIME_H */
include/asm-generic/cputime.h +1 -61
67 lines not on this page · raw message
include/asm-generic/cputime_jiffies.h new file +66 -0
66 lines not on this page · raw message
include/asm-generic/cputime_nsecs.h new file +96 -0
96 lines not on this page · raw message
Signature · 1 line
1.7.5.4
PATCH 3/8 cputime: Move default nsecs_to_cputime() to jiffies based cputime file
In reply to Frederic WeisbeckerIf the architecture doesn't provide an implementation of nsecs_to_cputime(), the cputime accounting core uses a default one that converts the nanoseconds to jiffies. However this only makes sense if we use the jiffies based cputime.
For now it doesn't matter much because this API is only called on code that uses jiffies based cputime accounting.
But the code may evolve and this API may be used more broadly in the future. Keeping this default implementation around is very error prone as it may introduce a bug and hide it on architectures that don't override this API.
Fix this by moving this definition to the jiffies based cputime headers as it is the only place where it belongs to.
--- include/asm-generic/cputime_jiffies.h | 10 ++++++++-- kernel/sched/cputime.c | 4 ---- 2 files changed, 8 insertions(+), 6 deletions(-)
include/asm-generic/cputime_jiffies.h +8 -2
@@ -13,8 +13,14 @@ typedef u64 __nocast cputime64_t;#define cputime64_to_jiffies64 (__ct) (__force u64)(__ct)#define jiffies64_to_cputime64 (__jif) (__force cputime64_t)(__jif) -#define nsecs_to_cputime64 (__ct) \ - jiffies64_to_cputime64(nsecs_to_jiffies64(__ct)) + +/* +* Convert nanoseconds to cputime +*/ +#define nsecs_to_cputime64 (__nsec) \ + jiffies64_to_cputime64(nsecs_to_jiffies64(__nsec)) +#define nsecs_to_cputime (__nsec) \ + jiffies_to_cputime(nsecs_to_jiffies(__nsec))/*
kernel/sched/cputime.c +0 -4
@@ -505,10 +505,6 @@ EXPORT_SYMBOL_GPL(vtime_account);#else -#ifndef nsecs_to_cputime -# define nsecs_to_cputime (__nsecs) nsecs_to_jiffies(__nsecs) -#endif -static cputime_t scale_utime (cputime_t utime,cputime_t rtime,cputime_t total) {u64 temp = (__force u64 ) rtime;
Signature · 1 line
1.7.5.4
PATCH 4/8 cputime: Generic on-demand virtual cputime accounting
In reply to Frederic WeisbeckerIf we want to stop the tick further idle, we need to be able to account the cputime without using the tick.
Virtual based cputime accounting solves that problem by hooking into kernel/user boundaries.
However implementing CONFIG_VIRT_CPU_ACCOUNTING require low level hooks and involves more overhead. But we already have a generic context tracking subsystem that is required for RCU needs by archs which plan to shut down the tick outside idle.
This patch implements a generic virtual based cputime accounting that relies on these generic kernel/user hooks.
There are some upsides of doing this:
- This requires no arch code to implement CONFIG_VIRT_CPU_ACCOUNTING if context tracking is already built (already necessary for RCU in full tickless mode).
- We can rely on the generic context tracking subsystem to dynamically (de)activate the hooks, so that we can switch anytime between virtual and tick based accounting. This way we don't have the overhead of the virtual accounting when the tick is running periodically.
And one downside:
- There is probably more overhead than a native virtual based cputime accounting. But this relies on hooks that are already set anyway.
--- arch/ia64/include/asm/cputime.h | 6 +- arch/ia64/include/asm/thread_info.h | 4 +- arch/ia64/include/asm/xen/minstate.h | 2 +- arch/ia64/kernel/asm-offsets.c | 2 +- arch/ia64/kernel/entry.S | 16 +++--- arch/ia64/kernel/fsys.S | 4 +- arch/ia64/kernel/head.S | 4 +- arch/ia64/kernel/ivt.S | 8 ++-- arch/ia64/kernel/minstate.h | 2 +- arch/ia64/kernel/time.c | 4 +- arch/powerpc/configs/chroma_defconfig | 2 +- arch/powerpc/configs/corenet64_smp_defconfig | 2 +- arch/powerpc/configs/pasemi_defconfig | 2 +- arch/powerpc/include/asm/cputime.h | 6 +- arch/powerpc/include/asm/lppaca.h | 2 +- arch/powerpc/include/asm/ppc_asm.h | 4 +- arch/powerpc/kernel/entry_64.S | 4 +- arch/powerpc/kernel/time.c | 4 +- arch/powerpc/platforms/pseries/dtl.c | 6 +- arch/powerpc/platforms/pseries/setup.c | 6 +- include/asm-generic/cputime.h | 8 +++- include/asm-generic/cputime_nsecs.h | 8 +++ include/linux/kernel_stat.h | 2 +- include/linux/vtime.h | 16 +++++++ init/Kconfig | 23 +++++++++- kernel/context_tracking.c | 6 ++- kernel/sched/cputime.c | 61 ++++++++++++++++++++++++-- 27 files changed, 160 insertions(+), 54 deletions(-)
arch/ia64/include/asm/cputime.h +3 -3
@@ -11,19 +11,19 @@* as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. * -* If we have CONFIG_VIRT_CPU_ACCOUNTING, we measure cpu time in nsec. +* If we have CONFIG_VIRT_CPU_ACCOUNTING_NATIVE, we measure cpu time in nsec. * Otherwise we measure cpu time in jiffies using the generic definitions. */ #ifndef __IA64_CPUTIME_H#define __IA64_CPUTIME_H -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE # include <asm-generic/cputime.h> #else # include <asm/processor.h> # include <asm-generic/cputime-nsecs.h> extern void arch_vtime_task_switch (struct task_struct *tsk); -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __IA64_CPUTIME_H */
arch/ia64/include/asm/thread_info.h +2 -2
@@ -31,7 +31,7 @@ struct thread_info {mm_segment_t addr_limit;/* user-level address space limit */ int preempt_count;/* 0=premptable, <0=BUG; will also serve as bh-counter */ struct restart_block restart_block; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE __u64 ac_stamp;__u64 ac_leave;__u64 ac_stime;
@@ -69,7 +69,7 @@ struct thread_info {#define task_stack_page (tsk) ((void *)(tsk))#define __HAVE_THREAD_FUNCTIONS -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #define setup_thread_stack (p, org) \ *task_thread_info(p) = *task_thread_info(org); \ task_thread_info(p)->ac_stime = 0; \
arch/ia64/include/asm/xen/minstate.h +1 -1
@@ -1,5 +1,5 @@ -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* read ar.itc in advance, and use it before leaving bank 0 */ #define XEN_ACCOUNT_GET_STAMP \ MOV_FROM_ITC(pUStk, p6, r20, r2);
arch/ia64/kernel/asm-offsets.c +1 -1
@@ -41,7 +41,7 @@ void foo(void)DEFINE (TI_FLAGS , offsetof(struct thread_info , flags));DEFINE (TI_CPU , offsetof(struct thread_info , cpu));DEFINE (TI_PRE_COUNT , offsetof(struct thread_info , preempt_count)); -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE DEFINE (TI_AC_STAMP , offsetof(struct thread_info , ac_stamp));DEFINE (TI_AC_LEAVE , offsetof(struct thread_info , ac_leave));DEFINE (TI_AC_STIME , offsetof(struct thread_info , ac_stime));
arch/ia64/kernel/entry.S +8 -8
@@ -724,7 +724,7 @@ GLOBAL_ENTRY(__paravirt_leave_syscall) #endif .global __paravirt_work_processed_syscall; __paravirt_work_processed_syscall: -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE adds r2=PT(LOADRS)+16,r12 MOV_FROM_ITC(pUStk, p9, r22, r19) // fetch time at leave adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
@@ -762,7 +762,7 @@ __paravirt_work_processed_syscall: ld8 r29=[r2],16 // M0|1 load cr.ipsr ld8 r28=[r3],16 // M0|1 load cr.iip -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE (pUStk) add r14=TI_AC_LEAVE+IA64_TASK_SIZE,r13 ;; ld8 r30=[r2],16 // M0|1 load cr.ifs
@@ -793,7 +793,7 @@ __paravirt_work_processed_syscall: ld8.fill r1=[r3],16 // M0|1 load r1 (pUStk) mov r17=1 // A ;; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE (pUStk) st1 [r15]=r17 // M2|3 #else (pUStk) st1 [r14]=r17 // M2|3
@@ -813,7 +813,7 @@ __paravirt_work_processed_syscall: shr.u r18=r19,16 // I0|1 get byte size of existing "dirty" partition COVER // B add current frame into dirty partition & set cr.ifs ;; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE mov r19=ar.bsp // M2 get new backing store pointer st8 [r14]=r22 // M save time at leave mov f10=f0 // F clear f10
@@ -948,7 +948,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) adds r16=PT(CR_IPSR)+16,r12 adds r17=PT(CR_IIP)+16,r12 -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE .pred.rel.mutex pUStk,pKStk MOV_FROM_PSR(pKStk, r22, r29) // M2 read PSR now that interrupts are disabled MOV_FROM_ITC(pUStk, p9, r22, r29) // M fetch time at leave
@@ -981,7 +981,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) ;; ld8.fill r12=[r16],16 ld8.fill r13=[r17],16 -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE (pUStk) adds r3=TI_AC_LEAVE+IA64_TASK_SIZE,r18 #else (pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18
@@ -989,7 +989,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) ;; ld8 r20=[r16],16 // ar.fpsr ld8.fill r15=[r17],16 -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE (pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18 // deferred #endif ;;
@@ -997,7 +997,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) ld8.fill r2=[r17] (pUStk) mov r17=1 ;; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE // mmi_ : ld8 st1 shr;; mmi_ : st8 st1 shr;; // mib : mov add br -> mib : ld8 add br // bbb_ : br nop cover;; mbb_ : mov br cover;;
arch/ia64/kernel/fsys.S +2 -2
@@ -529,7 +529,7 @@ GLOBAL_ENTRY(paravirt_fsys_bubble_down) nop.i 0 ;; mov ar.rsc=0 // M2 set enforced lazy mode, pl 0, LE, loadrs=0 -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE MOV_FROM_ITC(p0, p6, r30, r23) // M get cycle for accounting #else nop.m 0
@@ -555,7 +555,7 @@ GLOBAL_ENTRY(paravirt_fsys_bubble_down) cmp.ne pKStk,pUStk=r0,r0 // A set pKStk <- 0, pUStk <- 1 br.call.sptk.many b7=ia64_syscall_setup // B ;; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE // mov.m r30=ar.itc is called in advance add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
arch/ia64/kernel/head.S +2 -2
@@ -1073,7 +1073,7 @@ END(ia64_native_sched_clock) sched_clock = ia64_native_sched_clock #endif -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE GLOBAL_ENTRY(cycle_to_cputime) alloc r16=ar.pfs,1,0,0,0 addl r8=THIS_CPU(ia64_cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0
@@ -1091,7 +1091,7 @@ GLOBAL_ENTRY(cycle_to_cputime) shrp r8=r9,r8,IA64_NSEC_PER_CYC_SHIFT br.ret.sptk.many rp END(cycle_to_cputime) -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #ifdef CONFIG_IA64_BRL_EMU
arch/ia64/kernel/ivt.S +4 -4
@@ -784,7 +784,7 @@ ENTRY(break_fault) (p8) adds r28=16,r28 // A switch cr.iip to next bundle (p9) adds r8=1,r8 // A increment ei to next slot -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ;; mov b6=r30 // I0 setup syscall handler branch reg early #else
@@ -801,7 +801,7 @@ ENTRY(break_fault) // /////////////////////////////////////////////////////////////////////// st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting #else mov b6=r30 // I0 setup syscall handler branch reg early
@@ -817,7 +817,7 @@ ENTRY(break_fault) cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited? br.call.sptk.many b7=ia64_syscall_setup // B 1: -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE // mov.m r30=ar.itc is called in advance, and r13 is current add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
@@ -1043,7 +1043,7 @@ END(ia64_syscall_setup) DBG_FAULT(16) FAULT(16) -#if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE) +#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE) /* * There is no particular reason for this code to be here, other than * that there happens to be space here that would go unused otherwise.
arch/ia64/kernel/minstate.h +1 -1
@@ -4,7 +4,7 @@#include "entry.h" #include "paravirt_inst.h" -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* read ar.itc in advance, and use it before leaving bank 0 */ #define ACCOUNT_GET_STAMP \ (pUStk) mov.m r20=ar.itc;
arch/ia64/kernel/time.c +2 -2
@@ -77,7 +77,7 @@ static struct clocksource clocksource_itc = { };static struct clocksource *itc_clocksource; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #include <linux/kernel_stat.h>
@@ -142,7 +142,7 @@ void vtime_account_idle(struct task_struct *tsk) account_idle_time(vtime_delta(tsk)); } -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static irqreturn_t timer_interrupt (int irq,void *dev_id)
arch/powerpc/configs/chroma_defconfig +1 -1
@@ -1,6 +1,6 @@ CONFIG_PPC64=y CONFIG_PPC_BOOK3E_64=y -# CONFIG_VIRT_CPU_ACCOUNTING is not set +# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set CONFIG_SMP=y CONFIG_NR_CPUS=256 CONFIG_EXPERIMENTAL=y
arch/powerpc/configs/corenet64_smp_defconfig +1 -1
@@ -1,6 +1,6 @@ CONFIG_PPC64=y CONFIG_PPC_BOOK3E_64=y -# CONFIG_VIRT_CPU_ACCOUNTING is not set +# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_EXPERIMENTAL=y
arch/powerpc/configs/pasemi_defconfig +1 -1
@@ -1,6 +1,6 @@ CONFIG_PPC64=y CONFIG_ALTIVEC=y -# CONFIG_VIRT_CPU_ACCOUNTING is not set +# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_EXPERIMENTAL=y
arch/powerpc/include/asm/cputime.h +3 -3
@@ -8,7 +8,7 @@* as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. * -* If we have CONFIG_VIRT_CPU_ACCOUNTING, we measure cpu time in +* If we have CONFIG_VIRT_CPU_ACCOUNTING_NATIVE, we measure cpu time in * the same units as the timebase. Otherwise we measure cpu time * in jiffies using the generic definitions. */
@@ -16,7 +16,7 @@#ifndef __POWERPC_CPUTIME_H#define __POWERPC_CPUTIME_H -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #include <asm-generic/cputime.h> #ifdef __KERNEL__static inline void setup_cputime_one_jiffy (void ) { }
@@ -231,5 +231,5 @@ static inline cputime_t clock_t_to_cputime(const unsigned long clk)static inline void arch_vtime_task_switch (struct task_struct *tsk) { }#endif /* __KERNEL__ */ -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __POWERPC_CPUTIME_H */
arch/powerpc/include/asm/lppaca.h +1 -1
@@ -145,7 +145,7 @@ struct dtl_entry {extern struct kmem_cache *dtl_cache;/* -* When CONFIG_VIRT_CPU_ACCOUNTING = y, the cpu accounting code controls +* When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls * reading from the dispatch trace log. If other code wants to consume * DTL entries, it can set this pointer to a function that will get * called once for each DTL entry that gets processed.
arch/powerpc/include/asm/ppc_asm.h +2 -2
@@ -24,7 +24,7 @@* user_time and system_time fields in the paca. */ -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #define ACCOUNT_CPU_USER_ENTRY (ra, rb)#define ACCOUNT_CPU_USER_EXIT (ra, rb)#define ACCOUNT_STOLEN_TIME
@@ -70,7 +70,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)#endif /* CONFIG_PPC_SPLPAR */ -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ /* * Macros for storing registers into and loading registers from
arch/powerpc/kernel/entry_64.S +2 -2
@@ -94,7 +94,7 @@ system_call_common: addi r9,r1,STACK_FRAME_OVERHEAD ld r11,exception_marker@toc(r2) std r11,-16(r9) /* "regshere" marker */ -#if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR) +#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR) BEGIN_FW_FTR_SECTION beq 33f /* if from user, see if there are any DTL entries to process */
@@ -110,7 +110,7 @@ BEGIN_FW_FTR_SECTION addi r9,r1,STACK_FRAME_OVERHEAD 33: END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) -#endif /* CONFIG_VIRT_CPU_ACCOUNTING && CONFIG_PPC_SPLPAR */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */ /* * A syscall should always be called with interrupts enabled
arch/powerpc/kernel/time.c +2 -2
@@ -143,7 +143,7 @@ EXPORT_SYMBOL_GPL(ppc_proc_freq);unsigned long ppc_tb_freq;EXPORT_SYMBOL_GPL (ppc_tb_freq); -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Factors for converting from cputime_t (timebase ticks) to * jiffies, microseconds, seconds, and clock_t (1/USER_HZ seconds).
@@ -377,7 +377,7 @@ void vtime_account_user(struct task_struct *tsk) account_user_time(tsk, utime, utimescaled); } -#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */ +#else /* ! CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #define calc_cputime_factors ()#endif
arch/powerpc/platforms/pseries/dtl.c +3 -3
@@ -57,7 +57,7 @@ static u8 dtl_event_mask = 0x7;*/ static int dtl_buf_entries =N_DISPATCH_LOG ; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE struct dtl_ring { u64 write_index; struct dtl_entry *write_ptr;
@@ -142,7 +142,7 @@ static u64 dtl_current_index(struct dtl *dtl) return per_cpu(dtl_rings, dtl->cpu).write_index; } -#else /* CONFIG_VIRT_CPU_ACCOUNTING */ +#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static int dtl_start (struct dtl *dtl) {
@@ -188,7 +188,7 @@ static u64 dtl_current_index(struct dtl *dtl) {return lppaca_of (dtl->cpu).dtl_idx; } -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static int dtl_enable (struct dtl *dtl) {
arch/powerpc/platforms/pseries/setup.c +3 -3
@@ -281,7 +281,7 @@ static struct notifier_block pci_dn_reconfig_nb = {struct kmem_cache *dtl_cache; -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Allocate space for the dispatch trace log for all possible cpus * and register the buffers with the hypervisor. This is used for
@@ -332,12 +332,12 @@ static int alloc_dispatch_logs(void) return 0; } -#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ +#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static inline int alloc_dispatch_logs (void ) {return 0 ; } -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static int alloc_dispatch_log_kmem_cache (void ) {
include/asm-generic/cputime.h +7 -1
@@ -4,6 +4,12 @@#include <linux/time.h> #include <linux/jiffies.h> -#include <asm-generic/cputime_jiffies.h> +#ifndef CONFIG_VIRT_CPU_ACCOUNTING +# include <asm-generic/cputime_jiffies.h> +#endif + +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +# include <asm-generic/cputime_nsecs.h> +#endif #endif
include/asm-generic/cputime_nsecs.h +8 -0
@@ -26,6 +26,7 @@ typedef u64 __nocast cputime64_t;*/ #define cputime_to_jiffies (__ct) \ ((__force u64)(__ct) / (NSEC_PER_SEC / HZ)) +#define cputime_to_scaled (__ct) (__ct)#define jiffies_to_cputime (__jif) \ (__force cputime_t)((__jif) * (NSEC_PER_SEC / HZ))#define cputime64_to_jiffies64 (__ct) \
@@ -33,6 +34,13 @@ typedef u64 __nocast cputime64_t;#define jiffies64_to_cputime64 (__jif) \ (__force cputime64_t)((__jif) * (NSEC_PER_SEC / HZ)) + +/* +* Convert cputime <-> nanoseconds +*/ +#define nsecs_to_cputime (__nsecs) ((__force u64)(__nsecs)) + +/* * Convert cputime <-> microseconds */
include/linux/kernel_stat.h +1 -1
@@ -127,7 +127,7 @@ extern void account_system_time(struct task_struct *, int, cputime_t, cputime_t)extern void account_steal_time (cputime_t );extern void account_idle_time (cputime_t ); -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick (struct task_struct *tsk,int user) {vtime_account_user (tsk);
include/linux/vtime.h +16 -0
@@ -14,9 +14,25 @@ extern void vtime_account(struct task_struct *tsk);static inline void vtime_task_switch (struct task_struct *prev) { }static inline void vtime_account_system (struct task_struct *tsk) { }static inline void vtime_account_system_irqsafe (struct task_struct *tsk) { } +static inline void vtime_account_user (struct task_struct *tsk) { }static inline void vtime_account (struct task_struct *tsk) { }#endif +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +static inline void arch_vtime_task_switch (struct task_struct *tsk) { } +static inline void vtime_user_enter (struct task_struct *tsk) +{ +vtime_account_system (tsk); +} +static inline void vtime_user_exit (struct task_struct *tsk) +{ +vtime_account_user (tsk); +} +#else +static inline void vtime_user_enter (struct task_struct *tsk) { } +static inline void vtime_user_exit (struct task_struct *tsk) { } +#endif +#ifdef CONFIG_IRQ_TIME_ACCOUNTING extern void irqtime_account_irq (struct task_struct *tsk);#else
init/Kconfig +22 -1
@@ -326,6 +326,9 @@ source "kernel/time/Kconfig" menu "CPU/Task time and stats accounting" +config VIRT_CPU_ACCOUNTING + bool + choice prompt "Cputime accounting" default TICK_CPU_ACCOUNTING if !PPC64
@@ -342,9 +345,10 @@ config TICK_CPU_ACCOUNTING If unsure, say Y. -config VIRT_CPU_ACCOUNTING +config VIRT_CPU_ACCOUNTING_NATIVE bool "Deterministic task and CPU time accounting" depends on HAVE_VIRT_CPU_ACCOUNTING + select VIRT_CPU_ACCOUNTING help Select this option to enable more accurate task and CPU time accounting. This is done by reading a CPU counter on each
@@ -354,6 +358,23 @@ config VIRT_CPU_ACCOUNTING this also enables accounting of stolen time on logically-partitioned systems. +config VIRT_CPU_ACCOUNTING_GEN + bool "Full dynticks CPU time accounting" + depends on HAVE_CONTEXT_TRACKING && 64BIT + select VIRT_CPU_ACCOUNTING + select CONTEXT_TRACKING + help + Select this option to enable task and CPU time accounting on full + dynticks systems. This accounting is implemented by watching every + kernel-user boundaries using the context tracking subsystem. + The accounting is thus performed at the expense of some significant + overhead. + + For now this is only useful if you are working on the full + dynticks subsystem development. + + If unsure, say N. + config IRQ_TIME_ACCOUNTING bool "Fine granularity task level IRQ time accounting" depends on HAVE_IRQ_TIME_ACCOUNTING
kernel/context_tracking.c +4 -2
@@ -56,7 +56,7 @@ void user_enter(void)local_irq_save (flags);if (__this_cpu_read (context_tracking.active) &&__this_cpu_read (context_tracking.state) !=IN_USER ) { -__this_cpu_write (context_tracking.state,IN_USER ); +vtime_user_enter (current);/* * At this stage, only low level arch entry code remains and * then we'll run in userspace. We can assume there won't be
@@ -65,6 +65,7 @@ void user_enter(void)* on the tick. */ rcu_user_enter (); +__this_cpu_write (context_tracking.state,IN_USER ); }local_irq_restore (flags); }
@@ -90,12 +91,13 @@ void user_exit(void)local_irq_save (flags);if (__this_cpu_read (context_tracking.state) ==IN_USER ) { -__this_cpu_write (context_tracking.state,IN_KERNEL );/* * We are going to run code that may use RCU. Inform * RCU core about that (ie: we may need the tick again). */ rcu_user_exit (); +vtime_user_exit (current); +__this_cpu_write (context_tracking.state,IN_KERNEL ); }local_irq_restore (flags); }
kernel/sched/cputime.c +57 -4
83 lines not on this page · raw message
Signature · 1 line
1.7.5.4
PATCH 5/8 cputime: Allow dynamic switch between tick/virtual based cputime accounting
In reply to Frederic WeisbeckerAllow to dynamically switch between tick and virtual based cputime accounting. This way we can provide a kind of "on-demand" virtual based cputime accounting. In this mode, the kernel relies on the context tracking subsystem to dynamically probe on kernel boundaries.
This is in preparation for being able to stop the timer tick in more places than just the idle state. Doing so will depend on CONFIG_VIRT_CPU_ACCOUNTING_GEN which makes it possible to account the cputime without the tick by hooking on kernel/user boundaries.
Depending whether the tick is stopped or not, we can switch between tick and vtime based accounting anytime in order to minimize the overhead associated to user hooks.
--- include/linux/vtime.h | 8 ++++++++ kernel/sched/cputime.c | 41 +++++++++++++++++++++++++++++++---------- kernel/time/tick-sched.c | 5 ++++- 3 files changed, 43 insertions(+), 11 deletions(-)
include/linux/vtime.h +8 -0
@@ -10,12 +10,20 @@ extern void vtime_account_system_irqsafe(struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk);extern void vtime_account_user (struct task_struct *tsk);extern void vtime_account (struct task_struct *tsk); + +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +extern bool vtime_accounting_enabled (void );#else +static inline bool vtime_accounting_enabled (void ) {return true; } +#endif + +#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ static inline void vtime_task_switch (struct task_struct *prev) { }static inline void vtime_account_system (struct task_struct *tsk) { }static inline void vtime_account_system_irqsafe (struct task_struct *tsk) { }static inline void vtime_account_user (struct task_struct *tsk) { }static inline void vtime_account (struct task_struct *tsk) { } +static inline bool vtime_accounting_enabled (void ) {return false; }#endif #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
kernel/sched/cputime.c +31 -10
108 lines not on this page · raw message
kernel/time/tick-sched.c +4 -1
@@ -631,8 +631,11 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)static void tick_nohz_account_idle_ticks (struct tick_sched *ts) { -#ifndef CONFIG_VIRT_CPU_ACCOUNTING +#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE unsigned long ticks; + +if (vtime_accounting_enabled ()) +return ;/* * We stopped the tick in idle. Update process times would miss the * time we slept as update_process_times does only a 1 tick
Signature · 1 line
1.7.5.4
PATCH 6/8 cputime: Use accessors to read task cputime stats
In reply to Frederic WeisbeckerThis is in preparation for the full dynticks feature. While remotely reading the cputime of a task running in a full dynticks CPU, we'll need to do some extra-computation. This way we can account the time it spent tickless in userspace since its last cputime snapshot.
--- arch/alpha/kernel/osf_sys.c | 6 +++- arch/x86/kernel/apm_32.c | 11 +++++---- drivers/isdn/mISDN/stack.c | 7 +++++- fs/binfmt_elf.c | 8 +++++- fs/binfmt_elf_fdpic.c | 7 ++++- fs/proc/array.c | 4 +- include/linux/sched.h | 23 ++++++++++++++++++++++ include/linux/tsacct_kern.h | 3 ++ kernel/acct.c | 6 +++- kernel/cpu.c | 4 ++- kernel/delayacct.c | 7 ++++- kernel/exit.c | 10 +++++--- kernel/posix-cpu-timers.c | 28 +++++++++++++++++++++----- kernel/sched/cputime.c | 13 ++++++----- kernel/signal.c | 12 +++++++--- kernel/tsacct.c | 44 +++++++++++++++++++++++++++++++++--------- 16 files changed, 144 insertions(+), 49 deletions(-)
arch/alpha/kernel/osf_sys.c +4 -2
@@ -1139,6 +1139,7 @@ struct rusage32 { SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) {struct rusage32 r; +cputime_t utime, stime;if (who !=RUSAGE_SELF && who !=RUSAGE_CHILDREN )return -EINVAL ;
@@ -1146,8 +1147,9 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru)memset (&r,0 ,sizeof (r));switch (who) {case RUSAGE_SELF : -jiffies_to_timeval32 (current->utime, &r.ru_utime); -jiffies_to_timeval32 (current->stime, &r.ru_stime); +task_cputime (current, &utime, &stime); +jiffies_to_timeval32 (utime, &r.ru_utime); +jiffies_to_timeval32 (stime, &r.ru_stime); r.ru_minflt = current->min_flt; r.ru_majflt = current->maj_flt;break ;
arch/x86/kernel/apm_32.c +6 -5
@@ -899,6 +899,7 @@ static void apm_cpu_idle(void)static int use_apm_idle;/* = 0 */ static unsigned int last_jiffies;/* = 0 */ static unsigned int last_stime;/* = 0 */ +cputime_t stime;int apm_idle_done =0 ;unsigned int jiffies_since_last_check = jiffies - last_jiffies;
@@ -906,23 +907,23 @@ static void apm_cpu_idle(void)WARN_ONCE (1 ,"deprecated apm_cpu_idle will be deleted in 2012" ); recalc: +task_cputime (current,NULL , &stime);if (jiffies_since_last_check >IDLE_CALC_LIMIT ) { use_apm_idle =0 ; - last_jiffies = jiffies; - last_stime = current->stime; }else if (jiffies_since_last_check > idle_period) {unsigned int idle_percentage; - idle_percentage = current->stime - last_stime; + idle_percentage = stime - last_stime; idle_percentage *=100 ; idle_percentage /= jiffies_since_last_check; use_apm_idle = (idle_percentage > idle_threshold);if (apm_info.forbid_idle) use_apm_idle =0 ; - last_jiffies = jiffies; - last_stime = current->stime; } + last_jiffies = jiffies; + last_stime = stime; + bucket =IDLE_LEAKY_MAX ;while (!need_resched ()) {
drivers/isdn/mISDN/stack.c +6 -1
@@ -18,6 +18,7 @@#include <linux/slab.h> #include <linux/mISDNif.h> #include <linux/kthread.h> +#include <linux/sched.h> #include "core.h" static u_int *debug;
@@ -202,6 +203,9 @@ static int mISDNStackd(void *data) {struct mISDNstack *st = data; +#ifdef MISDN_MSG_STATS +cputime_t utime, stime; +#endif int err =0 ;sigfillset (¤t->blocked);
@@ -303,9 +307,10 @@ mISDNStackd(void *data)"msg %d sleep %d stopped\n" ,dev_name (&st->dev->dev), st->msg_cnt, st->sleep_cnt, st->stopped_cnt); +task_cputime (st->thread, &utime, &stime);printk (KERN_DEBUG "mISDNStackd daemon for %s utime(%ld) stime(%ld)\n" , -dev_name (&st->dev->dev), st->thread->utime, st->thread->stime); +dev_name (&st->dev->dev), utime, stime);printk (KERN_DEBUG "mISDNStackd daemon for %s nvcsw(%ld) nivcsw(%ld)\n" ,dev_name (&st->dev->dev), st->thread->nvcsw, st->thread->nivcsw);
fs/binfmt_elf.c +6 -2
@@ -33,6 +33,7 @@#include <linux/elf.h> #include <linux/utsname.h> #include <linux/coredump.h> +#include <linux/sched.h> #include <asm/uaccess.h> #include <asm/param.h> #include <asm/page.h>
@@ -1320,8 +1321,11 @@ static void fill_prstatus(struct elf_prstatus *prstatus,cputime_to_timeval (cputime.utime, &prstatus->pr_utime);cputime_to_timeval (cputime.stime, &prstatus->pr_stime); }else { -cputime_to_timeval (p->utime, &prstatus->pr_utime); -cputime_to_timeval (p->stime, &prstatus->pr_stime); +cputime_t utime, stime; + +task_cputime (p, &utime, &stime); +cputime_to_timeval (utime, &prstatus->pr_utime); +cputime_to_timeval (stime, &prstatus->pr_stime); }cputime_to_timeval (p->signal->cutime, &prstatus->pr_cutime);cputime_to_timeval (p->signal->cstime, &prstatus->pr_cstime);
fs/binfmt_elf_fdpic.c +5 -2
@@ -1375,8 +1375,11 @@ static void fill_prstatus(struct elf_prstatus *prstatus,cputime_to_timeval (cputime.utime, &prstatus->pr_utime);cputime_to_timeval (cputime.stime, &prstatus->pr_stime); }else { -cputime_to_timeval (p->utime, &prstatus->pr_utime); -cputime_to_timeval (p->stime, &prstatus->pr_stime); +cputime_t utime, stime; + +task_cputime (p, &utime, &stime); +cputime_to_timeval (utime, &prstatus->pr_utime); +cputime_to_timeval (stime, &prstatus->pr_stime); }cputime_to_timeval (p->signal->cutime, &prstatus->pr_cutime);cputime_to_timeval (p->signal->cstime, &prstatus->pr_cstime);
fs/proc/array.c +2 -2
@@ -449,7 +449,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,do { min_flt += t->min_flt; maj_flt += t->maj_flt; - gtime += t->gtime; + gtime +=task_gtime (t); t =next_thread (t); }while (t != task);
@@ -472,7 +472,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, min_flt = task->min_flt; maj_flt = task->maj_flt;task_cputime_adjusted (task, &utime, &stime); - gtime = task->gtime; + gtime =task_gtime (task); }/* scale priority and nice values from timeslices to -20..20 */
include/linux/sched.h +23 -0
@@ -1792,6 +1792,29 @@ static inline void put_task_struct(struct task_struct *t)__put_task_struct (t); } +static inline cputime_t task_gtime (struct task_struct *t) +{ +return t->gtime; +} + +static inline void task_cputime (struct task_struct *t, +cputime_t *utime,cputime_t *stime) +{ +if (utime) + *utime = t->utime; +if (stime) + *stime = t->stime; +} + +static inline void task_cputime_scaled (struct task_struct *t, +cputime_t *utimescaled, +cputime_t *stimescaled) +{ +if (utimescaled) + *utimescaled = t->utimescaled; +if (stimescaled) + *stimescaled = t->stimescaled; +}extern void task_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st);extern void thread_group_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st);
include/linux/tsacct_kern.h +3 -0
@@ -23,12 +23,15 @@ static inline void bacct_add_tsk(struct user_namespace *user_ns,#ifdef CONFIG_TASK_XACCT extern void xacct_add_tsk (struct taskstats *stats,struct task_struct *p);extern void acct_update_integrals (struct task_struct *tsk); +extern void acct_account_cputime (struct task_struct *tsk);extern void acct_clear_integrals (struct task_struct *tsk);#else static inline void xacct_add_tsk (struct taskstats *stats,struct task_struct *p) {}static inline void acct_update_integrals (struct task_struct *tsk) {} +static inline void acct_account_cputime (struct task_struct *tsk) +{}static inline void acct_clear_integrals (struct task_struct *tsk) {}#endif /* CONFIG_TASK_XACCT */
kernel/acct.c +4 -2
@@ -566,6 +566,7 @@ out:void acct_collect (long exitcode,int group_dead) {struct pacct_struct *pacct = ¤t->signal->pacct; +cputime_t utime, stime;unsigned long vsize =0 ;if (group_dead && current->mm) {
@@ -593,8 +594,9 @@ void acct_collect(long exitcode, int group_dead) pacct->ac_flag |=ACORE ;if (current->flags &PF_SIGNALED ) pacct->ac_flag |=AXSIG ; - pacct->ac_utime += current->utime; - pacct->ac_stime += current->stime; +task_cputime (current, &utime, &stime); + pacct->ac_utime += utime; + pacct->ac_stime += stime; pacct->ac_minflt += current->min_flt; pacct->ac_majflt += current->maj_flt;spin_unlock_irq (¤t->sighand->siglock);
kernel/cpu.c +3 -1
@@ -224,11 +224,13 @@ void clear_tasks_mm_cpumask(int cpu)static inline void check_for_tasks (int cpu) {struct task_struct *p; +cputime_t utime, stime;write_lock_irq (&tasklist_lock);for_each_process (p) { +task_cputime (p, &utime, &stime);if (task_cpu (p) == cpu && p->state ==TASK_RUNNING && - (p->utime || p->stime)) + (utime || stime)) printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d " "(state = %ld, flags = %x)\n", p->comm, task_pid_nr(p), cpu,
kernel/delayacct.c +5 -2
@@ -106,6 +106,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)unsigned long long t2, t3;unsigned long flags;struct timespec ts; +cputime_t utime, stime, stimescaled, utimescaled;/* Though tsk->delays accessed later, early exit avoids * unnecessary returning of other data
@@ -114,12 +115,14 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) goto done; tmp = (s64 )d->cpu_run_real_total; -cputime_to_timespec (tsk->utime + tsk->stime, &ts); +task_cputime (tsk, &utime, &stime); +cputime_to_timespec (utime + stime, &ts); tmp +=timespec_to_ns (&ts); d->cpu_run_real_total = (tmp < (s64 )d->cpu_run_real_total) ?0 : tmp; tmp = (s64 )d->cpu_scaled_run_real_total; -cputime_to_timespec (tsk->utimescaled + tsk->stimescaled, &ts); +task_cputime_scaled (tsk, &utimescaled, &stimescaled); +cputime_to_timespec (utimescaled + stimescaled, &ts); tmp +=timespec_to_ns (&ts); d->cpu_scaled_run_real_total = (tmp < (s64 )d->cpu_scaled_run_real_total) ?0 : tmp;
kernel/exit.c +6 -4
@@ -85,6 +85,7 @@ static void __exit_signal(struct task_struct *tsk)bool group_dead =thread_group_leader (tsk);struct sighand_struct *sighand;struct tty_struct *uninitialized_var (tty ); +cputime_t utime, stime; sighand =rcu_dereference_check (tsk->sighand,lockdep_tasklist_lock_is_held ());
@@ -123,9 +124,10 @@ static void __exit_signal(struct task_struct *tsk)* We won't ever get here for the group leader, since it * will have been the last reference on the signal_struct. */ - sig->utime += tsk->utime; - sig->stime += tsk->stime; - sig->gtime += tsk->gtime; +task_cputime (tsk, &utime, &stime); + sig->utime += utime; + sig->stime += stime; + sig->gtime +=task_gtime (tsk); sig->min_flt += tsk->min_flt; sig->maj_flt += tsk->maj_flt; sig->nvcsw += tsk->nvcsw;
@@ -1092,7 +1094,7 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p) sig = p->signal; psig->cutime += tgutime + sig->cutime; psig->cstime += tgstime + sig->cstime; - psig->cgtime += p->gtime + sig->gtime + sig->cgtime; + psig->cgtime +=task_gtime (p) + sig->gtime + sig->cgtime; psig->cmin_flt += p->min_flt + sig->min_flt + sig->cmin_flt; psig->cmaj_flt +=
kernel/posix-cpu-timers.c +22 -6
@@ -155,11 +155,19 @@ static void bump_cpu_timer(struct k_itimer *timer,static inline cputime_t prof_ticks (struct task_struct *p) { -return p->utime + p->stime; +cputime_t utime, stime; + +task_cputime (p, &utime, &stime); + +return utime + stime; }static inline cputime_t virt_ticks (struct task_struct *p) { -return p->utime; +cputime_t utime; + +task_cputime (p, &utime,NULL ); + +return utime; } static int
@@ -471,18 +479,23 @@ static void cleanup_timers(struct list_head *head,*/ void posix_cpu_timers_exit (struct task_struct *tsk) { +cputime_t utime, stime; +add_device_randomness ((const void *) &tsk->se.sum_exec_runtime,sizeof (unsigned long long )); +task_cputime (tsk, &utime, &stime);cleanup_timers (tsk->cpu_timers, - tsk->utime, tsk->stime, tsk->se.sum_exec_runtime); + utime, stime, tsk->se.sum_exec_runtime); }void posix_cpu_timers_exit_group (struct task_struct *tsk) {struct signal_struct *const sig = tsk->signal; +cputime_t utime, stime; +task_cputime (tsk, &utime, &stime);cleanup_timers (tsk->signal->cpu_timers, - tsk->utime + sig->utime, tsk->stime + sig->stime, + utime + sig->utime, stime + sig->stime, tsk->se.sum_exec_runtime + sig->sum_sched_runtime); }
@@ -1226,11 +1239,14 @@ static inline int task_cputime_expired(const struct task_cputime *sample,static inline int fastpath_timer_check (struct task_struct *tsk) {struct signal_struct *sig; +cputime_t utime, stime; + +task_cputime (tsk, &utime, &stime);if (!task_cputime_zero (&tsk->cputime_expires)) {struct task_cputime task_sample = { - .utime = tsk->utime, - .stime = tsk->stime, + .utime = utime, + .stime = stime, .sum_exec_runtime = tsk->se.sum_exec_runtime };
kernel/sched/cputime.c +7 -6
@@ -164,7 +164,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime,task_group_account_field (p, index, (__force u64 ) cputime);/* Account for user time used */ -acct_update_integrals (p); +acct_account_cputime (p); }/*
@@ -214,7 +214,7 @@ void __account_system_time(struct task_struct *p, cputime_t cputime,task_group_account_field (p, index, (__force u64 ) cputime);/* Account for system time used */ -acct_update_integrals (p); +acct_account_cputime (p); }/*
@@ -296,6 +296,7 @@ static __always_inline bool steal_account_process_tick(void)void thread_group_cputime (struct task_struct *tsk,struct task_cputime *times) {struct signal_struct *sig = tsk->signal; +cputime_t utime, stime;struct task_struct *t; times->utime = sig->utime;
@@ -309,8 +310,9 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) t = tsk; do { - times->utime += t->utime; - times->stime += t->stime; + task_cputime(tsk, &utime, &stime); + times->utime += utime; + times->stime += stime; times->sum_exec_runtime += task_sched_runtime(t); } while_each_thread(tsk, t); out:
@@ -588,11 +590,10 @@ static void cputime_adjust(struct task_cputime *curr,void task_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st) {struct task_cputime cputime = { - .utime = p->utime, - .stime = p->stime, .sum_exec_runtime = p->se.sum_exec_runtime, }; +task_cputime (p, &cputime.utime, &cputime.stime);cputime_adjust (&cputime, &p->prev_cputime, ut, st); }
kernel/signal.c +8 -4
@@ -1638,6 +1638,7 @@ bool do_notify_parent(struct task_struct *tsk, int sig)unsigned long flags;struct sighand_struct *psig;bool autoreap = false; +cputime_t utime, stime;BUG_ON (sig ==-1 );
@@ -1675,8 +1676,9 @@ bool do_notify_parent(struct task_struct *tsk, int sig) task_uid(tsk));rcu_read_unlock (); - info.si_utime =cputime_to_clock_t (tsk->utime + tsk->signal->utime); - info.si_stime =cputime_to_clock_t (tsk->stime + tsk->signal->stime); +task_cputime (tsk, &utime, &stime); + info.si_utime =cputime_to_clock_t (utime + tsk->signal->utime); + info.si_stime =cputime_to_clock_t (stime + tsk->signal->stime); info.si_status = tsk->exit_code &0x7f ;if (tsk->exit_code &0x80 )
@@ -1740,6 +1742,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk,unsigned long flags;struct task_struct *parent;struct sighand_struct *sighand; +cputime_t utime, stime;if (for_ptracer) { parent = tsk->parent;
@@ -1758,8 +1761,9 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, info.si_uid =from_kuid_munged (task_cred_xxx (parent, user_ns),task_uid (tsk));rcu_read_unlock (); - info.si_utime =cputime_to_clock_t (tsk->utime); - info.si_stime =cputime_to_clock_t (tsk->stime); +task_cputime (tsk, &utime, &stime); + info.si_utime =cputime_to_clock_t (utime); + info.si_stime =cputime_to_clock_t (stime); info.si_code = why;switch (why) {
kernel/tsacct.c +34 -10
74 lines not on this page · raw message
Signature · 1 line
1.7.5.4
PATCH 7/8 kvm: Prepare to add generic guest entry/exit callbacks
In reply to Frederic WeisbeckerDo some ground preparatory work before adding guest_enter() and guest_exit() context tracking callbacks. Those will be later used to read the guest cputime safely when we run in full dynticks mode.
--- include/linux/kvm_host.h | 39 ++++++++++++++++++++++++++++++--------- include/linux/vtime.h | 2 -- kernel/sched/cputime.c | 10 ---------- 3 files changed, 30 insertions(+), 21 deletions(-)
include/linux/kvm_host.h +30 -9
@@ -22,6 +22,7 @@#include <linux/rcupdate.h> #include <linux/ratelimit.h> #include <linux/err.h> +#include <linux/irqflags.h> #include <asm/signal.h> #include <linux/kvm.h>
@@ -740,15 +741,36 @@ static inline int kvm_deassign_device(struct kvm *kvm, }#endif /* CONFIG_IOMMU_API */ -static inline void kvm_guest_enter (void ) +static inline void guest_enter (void ) { -BUG_ON (preemptible ());/* * This is running in ioctl context so we can avoid * the call to vtime_account() with its unnecessary idle check. */ -vtime_account_system_irqsafe (current); +vtime_account_system (current); current->flags |=PF_VCPU ; +} + +static inline void guest_exit (void ) +{ +/* +* This is running in ioctl context so we can avoid +* the call to vtime_account() with its unnecessary idle check. +*/ +vtime_account_system (current); + current->flags &= ~PF_VCPU ; +} + +static inline void kvm_guest_enter (void ) +{ +unsigned long flags; + +BUG_ON (preemptible ()); + +local_irq_save (flags); +guest_enter (); +local_irq_restore (flags); +/* KVM does not hold any references to rcu protected data when it * switches CPU into a guest mode. In fact switching to a guest mode * is very similar to exiting to userspase from rcu point of view. In
@@ -761,12 +783,11 @@ static inline void kvm_guest_enter(void)static inline void kvm_guest_exit (void ) { -/* -* This is running in ioctl context so we can avoid -* the call to vtime_account() with its unnecessary idle check. -*/ -vtime_account_system_irqsafe (current); - current->flags &= ~PF_VCPU ; +unsigned long flags; + +local_irq_save (flags); +guest_exit (); +local_irq_restore (flags); }/*
include/linux/vtime.h +0 -2
@@ -6,7 +6,6 @@ struct task_struct;#ifdef CONFIG_VIRT_CPU_ACCOUNTING extern void vtime_task_switch (struct task_struct *prev);extern void vtime_account_system (struct task_struct *tsk); -extern void vtime_account_system_irqsafe (struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk);extern void vtime_account_user (struct task_struct *tsk);extern void vtime_account (struct task_struct *tsk);
@@ -20,7 +19,6 @@ static inline bool vtime_accounting_enabled(void) { return true; }#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ static inline void vtime_task_switch (struct task_struct *prev) { }static inline void vtime_account_system (struct task_struct *tsk) { } -static inline void vtime_account_system_irqsafe (struct task_struct *tsk) { }static inline void vtime_account_user (struct task_struct *tsk) { }static inline void vtime_account (struct task_struct *tsk) { }static inline bool vtime_accounting_enabled (void ) {return false; }
kernel/sched/cputime.c +0 -10
@@ -465,16 +465,6 @@ void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime *st = cputime.stime; } -void vtime_account_system_irqsafe (struct task_struct *tsk) -{ -unsigned long flags; - -local_irq_save (flags); -vtime_account_system (tsk); -local_irq_restore (flags); -} -EXPORT_SYMBOL_GPL (vtime_account_system_irqsafe); -#ifndef __ARCH_HAS_VTIME_TASK_SWITCHvoid vtime_task_switch (struct task_struct *prev) {
Signature · 1 line
1.7.5.4
PATCH 8/8 cputime: Safely read cputime of full dynticks CPUs
In reply to Frederic WeisbeckerWhile remotely reading the cputime of a task running in a full dynticks CPU, the values stored in utime/stime fields of struct task_struct may be stale. Its values may be those of the last kernel <-> user transition time snapshot and we need to add the tickless time spent since this snapshot.
To fix this, flush the cputime of the dynticks CPUs on kernel <-> user transition and record the time / context where we did this. Then on top of this snapshot and the current time, perform the fixup on the reader side from task_times() accessors.
--- arch/s390/kernel/vtime.c | 6 +- include/linux/hardirq.h | 4 +- include/linux/init_task.h | 11 +++ include/linux/kvm_host.h | 20 ++++- include/linux/sched.h | 27 +++++- include/linux/vtime.h | 47 +++++------ kernel/context_tracking.c | 19 +++++ kernel/fork.c | 6 ++ kernel/sched/core.c | 1 + kernel/sched/cputime.c | 195 ++++++++++++++++++++++++++++++++++++++++++--- kernel/softirq.c | 6 +- 11 files changed, 291 insertions(+), 51 deletions(-)
arch/s390/kernel/vtime.c +3 -3
@@ -127,7 +127,7 @@ void vtime_account_user(struct task_struct *tsk)* Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -void vtime_account (struct task_struct *tsk) +void vtime_account_irq_enter (struct task_struct *tsk) {struct thread_info *ti =task_thread_info (tsk);u64 timer, system;
@@ -145,10 +145,10 @@ void vtime_account(struct task_struct *tsk)virt_timer_forward (system); } -EXPORT_SYMBOL_GPL (vtime_account); +EXPORT_SYMBOL_GPL (vtime_account_irq_enter);void vtime_account_system (struct task_struct *tsk) -__attribute__((alias ("vtime_account" ))); +__attribute__((alias ("vtime_account_irq_enter" )));EXPORT_SYMBOL_GPL (vtime_account_system);void __kprobes vtime_stop_cpu (void )
include/linux/hardirq.h +2 -2
@@ -153,7 +153,7 @@ extern void rcu_nmi_exit(void);*/ #define __irq_enter () \ do { \ - vtime_account_irq_enter(current); \ + account_irq_enter_time(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ } while (0)
@@ -169,7 +169,7 @@ extern void irq_enter(void);#define __irq_exit () \ do { \ trace_hardirq_exit(); \ - vtime_account_irq_exit(current); \ + account_irq_exit_time(current); \ sub_preempt_count(HARDIRQ_OFFSET); \ } while (0)
include/linux/init_task.h +11 -0
@@ -10,6 +10,7 @@#include <linux/pid_namespace.h> #include <linux/user_namespace.h> #include <linux/securebits.h> +#include <linux/seqlock.h> #include <net/net_namespace.h> #ifdef CONFIG_SMP
@@ -141,6 +142,15 @@ extern struct task_group root_task_group;# define INIT_PERF_EVENTS (tsk)#endif +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +# define INIT_VTIME (tsk) \ + .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \ + .vtime_snap = 0, \ + .vtime_snap_whence = VTIME_SYS, +#else +# define INIT_VTIME (tsk) +#endif +#define INIT_TASK_COMM "swapper"/*
@@ -210,6 +220,7 @@ extern struct task_group root_task_group; INIT_TRACE_RECURSION \ INIT_TASK_RCU_PREEMPT(tsk) \ INIT_CPUSET_SEQ \ + INIT_VTIME(tsk) \ }
include/linux/kvm_host.h +18 -2
@@ -741,7 +741,7 @@ static inline int kvm_deassign_device(struct kvm *kvm, }#endif /* CONFIG_IOMMU_API */ -static inline void guest_enter (void ) +static inline void __guest_enter (void ) {/* * This is running in ioctl context so we can avoid
@@ -751,7 +751,7 @@ static inline void guest_enter(void) current->flags |=PF_VCPU ; } -static inline void guest_exit (void ) +static inline void __guest_exit (void ) {/* * This is running in ioctl context so we can avoid
@@ -761,6 +761,22 @@ static inline void guest_exit(void) current->flags &= ~PF_VCPU ; } +#ifdef CONFIG_CONTEXT_TRACKING +extern void guest_enter (void ); +extern void guest_exit (void ); + +#else /* !CONFIG_CONTEXT_TRACKING */ +static inline void guest_enter (void ) +{ +__guest_enter (); +} + +static inline void guest_exit (void ) +{ +__guest_exit (); +} +#endif /* !CONFIG_CONTEXT_TRACKING */ +static inline void kvm_guest_enter (void ) {unsigned long flags;
include/linux/sched.h +22 -5
@@ -1368,6 +1368,15 @@ struct task_struct {#ifndef CONFIG_VIRT_CPU_ACCOUNTING struct cputime prev_cputime;#endif +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +seqlock_t vtime_seqlock; +unsigned long long vtime_snap; +enum { +VTIME_SLEEPING =0 , +VTIME_USER , +VTIME_SYS , + } vtime_snap_whence; +#endif unsigned long nvcsw, nivcsw;/* context switch counts */ struct timespec start_time;/* monotonic time */ struct timespec real_start_time;/* boot based time */
@@ -1792,11 +1801,13 @@ static inline void put_task_struct(struct task_struct *t)__put_task_struct (t); } -static inline cputime_t task_gtime (struct task_struct *t) -{ -return t->gtime; -} - +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +extern void task_cputime (struct task_struct *t, +cputime_t *utime,cputime_t *stime); +extern void task_cputime_scaled (struct task_struct *t, +cputime_t *utimescaled,cputime_t *stimescaled); +extern cputime_t task_gtime (struct task_struct *t); +#else static inline void task_cputime (struct task_struct *t,cputime_t *utime,cputime_t *stime) {
@@ -1815,6 +1826,12 @@ static inline void task_cputime_scaled(struct task_struct *t,if (stimescaled) *stimescaled = t->stimescaled; } + +static inline cputime_t task_gtime (struct task_struct *t) +{ +return t->gtime; +} +#endif extern void task_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st);extern void thread_group_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st);
include/linux/vtime.h +23 -24
83 lines not on this page · raw message
kernel/context_tracking.c +19 -0
@@ -15,6 +15,7 @@*/ #include <linux/context_tracking.h> +#include <linux/kvm_host.h> #include <linux/rcupdate.h> #include <linux/sched.h> #include <linux/hardirq.h>
@@ -102,6 +103,24 @@ void user_exit(void)local_irq_restore (flags); } +#ifdef CONFIG_KVM +void guest_enter (void ) +{ +if (vtime_accounting_enabled ()) +vtime_guest_enter (current); +else +__guest_enter (); +} + +void guest_exit (void ) +{ +if (vtime_accounting_enabled ()) +vtime_guest_exit (current); +else +__guest_exit (); +} +#endif +/** * context_tracking_task_switch - context switch the syscall callbacks
kernel/fork.c +6 -0
@@ -1233,6 +1233,12 @@ static struct task_struct *copy_process(unsigned long clone_flags,#ifndef CONFIG_VIRT_CPU_ACCOUNTING p->prev_cputime.utime = p->prev_cputime.stime =0 ;#endif +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +seqlock_init (&p->vtime_seqlock); + p->vtime_snap =0 ; + p->vtime_snap_whence =VTIME_SLEEPING ; +#endif +#if defined(SPLIT_RSS_COUNTING )memset (&p->rss_stat,0 ,sizeof (p->rss_stat));#endif
kernel/sched/core.c +1 -0
@@ -4666,6 +4666,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)*/ idle->sched_class = &idle_sched_class;ftrace_graph_init_idle_task (idle, cpu); +vtime_init_idle (idle);#if defined(CONFIG_SMP )sprintf (idle->comm,"%s/%d" ,INIT_TASK_COMM , cpu);#endif
kernel/sched/cputime.c +183 -12
243 lines not on this page · raw message
kernel/softirq.c +3 -3
@@ -221,7 +221,7 @@ asmlinkage void __do_softirq(void) current->flags &= ~PF_MEMALLOC ; pending =local_softirq_pending (); -vtime_account_irq_enter (current); +account_irq_enter_time (current);__local_bh_disable ((unsigned long )__builtin_return_address (0 ),SOFTIRQ_OFFSET );
@@ -272,7 +272,7 @@ restart:lockdep_softirq_exit (); -vtime_account_irq_exit (current); +account_irq_exit_time (current);__local_bh_enable (SOFTIRQ_OFFSET );tsk_restore_flags (current, old_flags,PF_MEMALLOC ); }
@@ -341,7 +341,7 @@ static inline void invoke_softirq(void)*/ void irq_exit (void ) { -vtime_account_irq_exit (current); +account_irq_exit_time (current);trace_hardirq_exit ();sub_preempt_count (IRQ_EXIT_OFFSET );if (!in_interrupt () &&local_softirq_pending ())
Signature · 1 line
1.7.5.4