sched/nohz: Make kcpustat's CPUTIME_SYSTEM vtime aware v2 (Partially fix kcpustat on nohz_full)
Subsystems (8)
Frederic Weisbecker
maintainercover sched/nohz: Make kcpustat's CPUTIME_SYSTEM vtime aware v2 (Partially fix kcpustat on nohz_full)
Thanks to the recent patches that have brought a sensible rq->curr RCU lifecycle, here comes a simplified rework of
"[PATCH 00/25] sched/nohz: Make kcpustat vtime aware"
See for the record and a summary: https://lore.kernel.org/lkml/[email protected]/
Since I'm trying to cut and simplify the series, this one only fixes the reading of kcpustat's system field for now. But that also provides all the necessary preparation work for the rest.
User and guest time will have their own series because niceness brings issues of its own.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
nohz/kcpustat-v2
HEAD: e179e89320c53a96c5d585af38126cfb124da789
Thanks,
Frederic
---
Frederic Weisbecker (14):
sched/vtime: Record CPU under seqcount for kcpustat needs
sched/cputime: Add vtime idle task state
sched/cputime: Add vtime guest task state
context_tracking: Remove context_tracking_active()
context_tracking: s/context_tracking_is_enabled/context_tracking_enabled()
context_tracking: Rename context_tracking_is_cpu_enabled() to context_tracking_enabled_this_cpu()
context_tracking: Introduce context_tracking_enabled_cpu()
sched/vtime: Rename vtime_accounting_cpu_enabled() to vtime_accounting_enabled_this_cpu() sched/vtime: Introduce vtime_accounting_enabled_cpu()
context_tracking: Check static key on context_tracking_enabled_*cpu() sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
procfs: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
leds: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
arch/x86/entry/calling.h | 2 +- drivers/cpufreq/cpufreq.c | 2 +- drivers/leds/trigger/ledtrig-activity.c | 2 +- fs/proc/stat.c | 4 +- include/linux/context_tracking.h | 26 +++---- include/linux/context_tracking_state.h | 21 +++--- include/linux/kernel_stat.h | 11 +++ include/linux/sched.h | 9 ++- include/linux/tick.h | 2 +- include/linux/vtime.h | 23 +++--- kernel/context_tracking.c | 6 +- kernel/sched/cputime.c | 119 ++++++++++++++++++++++++++++---- kernel/time/tick-sched.c | 2 +- 13 files changed, 172 insertions(+), 57 deletions(-)
Frederic Weisbecker
maintainerPATCH 1/14 sched/vtime: Record CPU under seqcount for kcpustat needs
In order to compute the kcpustat delta on a nohz CPU, we'll need to fetch the task running on that target. Checking that its vtime state snapshot actually refers to the relevant target involves recording that CPU under the seqcount locked on task switch.
This is a step toward making kcpustat moving forward on full nohz CPUs.
--- include/linux/sched.h | 1 + kernel/sched/cputime.c | 3 +++ 2 files changed, 4 insertions(+)
include/linux/sched.h +1 -0
@@ -259,6 +259,7 @@ struct vtime {seqcount_t seqcount;unsigned long long starttime;enum vtime_state state; +unsigned int cpu;u64 utime;u64 stime;u64 gtime;
kernel/sched/cputime.c +3 -0
@@ -818,6 +818,7 @@ void vtime_task_switch_generic(struct task_struct *prev)else __vtime_account_kernel (prev, vtime); vtime->state =VTIME_INACTIVE ; + vtime->cpu =-1 ;write_seqcount_end (&vtime->seqcount); vtime = ¤t->vtime;
@@ -825,6 +826,7 @@ void vtime_task_switch_generic(struct task_struct *prev)write_seqcount_begin (&vtime->seqcount); vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); + vtime->cpu =smp_processor_id ();write_seqcount_end (&vtime->seqcount); }
@@ -837,6 +839,7 @@ void vtime_init_idle(struct task_struct *t, int cpu)write_seqcount_begin (&vtime->seqcount); vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); + vtime->cpu = cpu;write_seqcount_end (&vtime->seqcount);local_irq_restore (flags); }
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 2/14 sched/cputime: Add vtime idle task state
In reply to Frederic WeisbeckerRecord idle as a VTIME state instead of guessing it from VTIME_SYS and is_idle_task(). This is going to simplify the cputime read side especially as its state machine is going to further expand in order to fully support kcpustat on nohz_full.
--- include/linux/sched.h | 6 ++++-- kernel/sched/cputime.c | 13 ++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-)
include/linux/sched.h +4 -2
@@ -249,10 +249,12 @@ struct prev_cputime {enum vtime_state {/* Task is sleeping or running in a CPU with VTIME inactive: */ VTIME_INACTIVE =0 , -/* Task runs in userspace in a CPU with VTIME active: */ -VTIME_USER , +/* Task is idle */ +VTIME_IDLE ,/* Task runs in kernelspace in a CPU with VTIME active: */ VTIME_SYS , +/* Task runs in userspace in a CPU with VTIME active: */ +VTIME_USER , };struct vtime {
kernel/sched/cputime.c +8 -5
@@ -813,7 +813,7 @@ void vtime_task_switch_generic(struct task_struct *prev)struct vtime *vtime = &prev->vtime;write_seqcount_begin (&vtime->seqcount); -if (is_idle_task (prev)) +if (vtime->state ==VTIME_IDLE )vtime_account_idle (prev);else __vtime_account_kernel (prev, vtime);
@@ -824,7 +824,10 @@ void vtime_task_switch_generic(struct task_struct *prev) vtime = ¤t->vtime;write_seqcount_begin (&vtime->seqcount); - vtime->state =VTIME_SYS ; +if (is_idle_task (current)) + vtime->state =VTIME_IDLE ; +else + vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); vtime->cpu =smp_processor_id ();write_seqcount_end (&vtime->seqcount);
@@ -837,7 +840,7 @@ void vtime_init_idle(struct task_struct *t, int cpu)local_irq_save (flags);write_seqcount_begin (&vtime->seqcount); - vtime->state =VTIME_SYS ; + vtime->state =VTIME_IDLE ; vtime->starttime =sched_clock (); vtime->cpu = cpu;write_seqcount_end (&vtime->seqcount);
@@ -888,8 +891,8 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) *utime = t->utime; *stime = t->stime; -/* Task is sleeping, nothing to add */ -if (vtime->state ==VTIME_INACTIVE ||is_idle_task (t)) +/* Task is sleeping or idle, nothing to add */ +if (vtime->state <VTIME_SYS )continue ; delta =vtime_delta (vtime);
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 3/14 sched/cputime: Add vtime guest task state
In reply to Frederic WeisbeckerRecord guest as a VTIME state instead of guessing it from VTIME_SYS and PF_VCPU. This is going to simplify the cputime read side especially as its state machine is going to further expand in order to fully support kcpustat on nohz_full.
--- include/linux/sched.h | 2 ++ kernel/sched/cputime.c | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-)
include/linux/sched.h +2 -0
@@ -255,6 +255,8 @@ enum vtime_state {VTIME_SYS ,/* Task runs in userspace in a CPU with VTIME active: */ VTIME_USER , +/* Task runs as guests in a CPU with VTIME active: */ + VTIME_GUEST, }; struct vtime {
kernel/sched/cputime.c +11 -7
@@ -733,7 +733,7 @@ static void __vtime_account_kernel(struct task_struct *tsk, struct vtime *vtime) {/* We might have scheduled out from guest path */ -if (tsk->flags &PF_VCPU ) +if (vtime->state ==VTIME_GUEST )vtime_account_guest (tsk, vtime);else vtime_account_system (tsk, vtime);
@@ -788,6 +788,7 @@ void vtime_guest_enter(struct task_struct *tsk)write_seqcount_begin (&vtime->seqcount);vtime_account_system (tsk, vtime); tsk->flags |=PF_VCPU ; + vtime->state =VTIME_GUEST ;write_seqcount_end (&vtime->seqcount); }EXPORT_SYMBOL_GPL (vtime_guest_enter);
@@ -799,6 +800,7 @@ void vtime_guest_exit(struct task_struct *tsk)write_seqcount_begin (&vtime->seqcount);vtime_account_guest (tsk, vtime); tsk->flags &= ~PF_VCPU ; + vtime->state =VTIME_SYS ;write_seqcount_end (&vtime->seqcount); }EXPORT_SYMBOL_GPL (vtime_guest_exit);
@@ -826,6 +828,8 @@ void vtime_task_switch_generic(struct task_struct *prev)write_seqcount_begin (&vtime->seqcount);if (is_idle_task (current)) vtime->state =VTIME_IDLE ; +else if (current->flags &PF_VCPU ) + vtime->state =VTIME_GUEST ;else vtime->state =VTIME_SYS ; vtime->starttime =sched_clock ();
@@ -860,7 +864,7 @@ u64 task_gtime(struct task_struct *t) seq =read_seqcount_begin (&vtime->seqcount); gtime = t->gtime; -if (vtime->state ==VTIME_SYS && t->flags &PF_VCPU ) +if (vtime->state ==VTIME_GUEST ) gtime += vtime->gtime +vtime_delta (vtime); }while (read_seqcount_retry (&vtime->seqcount, seq));
@@ -898,13 +902,13 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) delta =vtime_delta (vtime);/* -* Task runs either in user or kernel space, add pending nohz time to -* the right place. +* Task runs either in user (including guest) or kernel space, +* add pending nohz time to the right place. */ -if (vtime->state ==VTIME_USER || t->flags &PF_VCPU ) - *utime += vtime->utime + delta; -else if (vtime->state ==VTIME_SYS ) +if (vtime->state ==VTIME_SYS ) *stime += vtime->stime + delta; +else + *utime += vtime->utime + delta; }while (read_seqcount_retry (&vtime->seqcount, seq)); }#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 4/14 context_tracking: Remove context_tracking_active()
In reply to Frederic WeisbeckerThis function is a leftover from old removal or rename. We can drop it.
--- include/linux/context_tracking_state.h | 1 - 1 file changed, 1 deletion(-)
include/linux/context_tracking_state.h +0 -1
@@ -42,7 +42,6 @@ static inline bool context_tracking_in_user(void) }#else static inline bool context_tracking_in_user (void ) {return false; } -static inline bool context_tracking_active (void ) {return false; }static inline bool context_tracking_is_enabled (void ) {return false; }static inline bool context_tracking_cpu_is_enabled (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 5/14 context_tracking: s/context_tracking_is_enabled/context_tracking_enabled()
In reply to Frederic WeisbeckerRemove the superfluous "is" in the middle of the name. We want to standardize the naming so that it can be expanded through suffixes:
context_tracking_enabled()
context_tracking_enabled_cpu()
context_tracking_enabled_this_cpu()
--- arch/x86/entry/calling.h | 2 +- include/linux/context_tracking.h | 20 ++++++++++---------- include/linux/context_tracking_state.h | 8 ++++---- include/linux/tick.h | 2 +- include/linux/vtime.h | 2 +- kernel/context_tracking.c | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-)
arch/x86/entry/calling.h +1 -1
@@ -354,7 +354,7 @@ For 32-bit we have the following conventions - kernel is built with .macro CALL_enter_from_user_mode#ifdef CONFIG_CONTEXT_TRACKING #ifdef CONFIG_JUMP_LABEL - STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 + STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_key, def=0#endif call enter_from_user_mode .Lafter_call_\@:
include/linux/context_tracking.h +10 -10
@@ -22,26 +22,26 @@ extern void context_tracking_user_exit(void);static inline void user_enter (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())context_tracking_enter (CONTEXT_USER ); }static inline void user_exit (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())context_tracking_exit (CONTEXT_USER ); }/* Called with interrupts disabled. */ static inline void user_enter_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_enter (CONTEXT_USER ); }static inline void user_exit_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_USER ); }
@@ -49,7 +49,7 @@ static inline enum ctx_state exception_enter(void) {enum ctx_state prev_ctx; -if (!context_tracking_is_enabled ()) +if (!context_tracking_enabled ())return 0 ; prev_ctx =this_cpu_read (context_tracking.state);
@@ -61,7 +61,7 @@ static inline enum ctx_state exception_enter(void)static inline void exception_exit (enum ctx_state prev_ctx) { -if (context_tracking_is_enabled ()) { +if (context_tracking_enabled ()) {if (prev_ctx !=CONTEXT_KERNEL )context_tracking_enter (prev_ctx); }
@@ -77,7 +77,7 @@ static inline void exception_exit(enum ctx_state prev_ctx)*/ static inline enum ctx_state ct_state (void ) { -return context_tracking_is_enabled () ? +return context_tracking_enabled () ?this_cpu_read (context_tracking.state) :CONTEXT_DISABLED ; }#else
@@ -90,7 +90,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) { }static inline enum ctx_state ct_state (void ) {return CONTEXT_DISABLED ; }#endif /* !CONFIG_CONTEXT_TRACKING */ -#define CT_WARN_ON (cond) WARN_ON(context_tracking_is_enabled() && (cond)) +#define CT_WARN_ON (cond) WARN_ON(context_tracking_enabled() && (cond))#ifdef CONFIG_CONTEXT_TRACKING_FORCE extern void context_tracking_init (void );
@@ -108,7 +108,7 @@ static inline void guest_enter_irqoff(void)else current->flags |=PF_VCPU ; -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_enter (CONTEXT_GUEST );/* KVM does not hold any references to rcu protected data when it
@@ -124,7 +124,7 @@ static inline void guest_enter_irqoff(void)static inline void guest_exit_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_GUEST );if (vtime_accounting_cpu_enabled ())
include/linux/context_tracking_state.h +4 -4
@@ -23,12 +23,12 @@ struct context_tracking { };#ifdef CONFIG_CONTEXT_TRACKING -extern struct static_key_false context_tracking_enabled; +extern struct static_key_false context_tracking_key; DECLARE_PER_CPU(struct context_tracking, context_tracking); -static inline bool context_tracking_is_enabled (void ) +static inline bool context_tracking_enabled (void ) { -return static_branch_unlikely (&context_tracking_enabled); +return static_branch_unlikely (&context_tracking_key); }static inline bool context_tracking_cpu_is_enabled (void )
@@ -42,7 +42,7 @@ static inline bool context_tracking_in_user(void) }#else static inline bool context_tracking_in_user (void ) {return false; } -static inline bool context_tracking_is_enabled (void ) {return false; } +static inline bool context_tracking_enabled (void ) {return false; }static inline bool context_tracking_cpu_is_enabled (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
include/linux/tick.h +1 -1
@@ -174,7 +174,7 @@ extern cpumask_var_t tick_nohz_full_mask;static inline bool tick_nohz_full_enabled (void ) { -if (!context_tracking_is_enabled ()) +if (!context_tracking_enabled ())return false;return tick_nohz_full_running;
include/linux/vtime.h +1 -1
@@ -28,7 +28,7 @@ extern void vtime_task_switch(struct task_struct *prev);*/ static inline bool vtime_accounting_enabled (void ) { -return context_tracking_is_enabled (); +return context_tracking_enabled (); }static inline bool vtime_accounting_cpu_enabled (void )
kernel/context_tracking.c +3 -3
@@ -25,8 +25,8 @@#define CREATE_TRACE_POINTS #include <trace/events/context_tracking.h> -DEFINE_STATIC_KEY_FALSE (context_tracking_enabled); -EXPORT_SYMBOL_GPL (context_tracking_enabled); +DEFINE_STATIC_KEY_FALSE (context_tracking_key); +EXPORT_SYMBOL_GPL (context_tracking_key); DEFINE_PER_CPU(struct context_tracking, context_tracking);EXPORT_SYMBOL_GPL (context_tracking);
@@ -192,7 +192,7 @@ void __init context_tracking_cpu_set(int cpu)if (!per_cpu (context_tracking.active, cpu)) {per_cpu (context_tracking.active, cpu) = true; -static_branch_inc (&context_tracking_enabled); +static_branch_inc (&context_tracking_key); }if (initialized)
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 6/14 context_tracking: Rename context_tracking_is_cpu_enabled() to context_tracking_enabled_this_cpu()
In reply to Frederic WeisbeckerStandardize the naming on top of the context_tracking_enabled_*() base. Also make it clear we are checking the context tracking state of the *current* CPU with this function. We'll need to add an API to check that state on remote CPUs as well, so we must disambiguate the naming.
--- include/linux/context_tracking.h | 2 +- include/linux/context_tracking_state.h | 4 ++-- include/linux/vtime.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
include/linux/context_tracking.h +1 -1
@@ -118,7 +118,7 @@ static inline void guest_enter_irqoff(void)* one time slice). Lets treat guest mode as quiescent state, just like * we do with user-mode execution. */ -if (!context_tracking_cpu_is_enabled ()) +if (!context_tracking_enabled_this_cpu ())rcu_virt_note_context_switch (smp_processor_id ()); }
include/linux/context_tracking_state.h +2 -2
@@ -31,7 +31,7 @@ static inline bool context_tracking_enabled(void)return static_branch_unlikely (&context_tracking_key); } -static inline bool context_tracking_cpu_is_enabled (void ) +static inline bool context_tracking_enabled_this_cpu (void ) {return __this_cpu_read (context_tracking.active); }
@@ -43,7 +43,7 @@ static inline bool context_tracking_in_user(void)#else static inline bool context_tracking_in_user (void ) {return false; }static inline bool context_tracking_enabled (void ) {return false; } -static inline bool context_tracking_cpu_is_enabled (void ) {return false; } +static inline bool context_tracking_enabled_this_cpu (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */ #endif
include/linux/vtime.h +1 -1
@@ -34,7 +34,7 @@ static inline bool vtime_accounting_enabled(void)static inline bool vtime_accounting_cpu_enabled (void ) {if (vtime_accounting_enabled ()) { -if (context_tracking_cpu_is_enabled ()) +if (context_tracking_enabled_this_cpu ())return true; }
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 7/14 context_tracking: Introduce context_tracking_enabled_cpu()
In reply to Frederic WeisbeckerThis allows us to check if a remote CPU runs context tracking (ie: is nohz_full). We'll need that to reliably support "nice" accounting on kcpustat.
--- include/linux/context_tracking_state.h | 6 ++++++ 1 file changed, 6 insertions(+)
include/linux/context_tracking_state.h +6 -0
@@ -31,6 +31,11 @@ static inline bool context_tracking_enabled(void)return static_branch_unlikely (&context_tracking_key); } +static inline bool context_tracking_enabled_cpu (int cpu) +{ +return per_cpu (context_tracking.active, cpu); +} +static inline bool context_tracking_enabled_this_cpu (void ) {return __this_cpu_read (context_tracking.active);
@@ -43,6 +48,7 @@ static inline bool context_tracking_in_user(void)#else static inline bool context_tracking_in_user (void ) {return false; }static inline bool context_tracking_enabled (void ) {return false; } +static inline bool context_tracking_enabled_cpu (int cpu) {return false; }static inline bool context_tracking_enabled_this_cpu (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 8/14 sched/vtime: Rename vtime_accounting_cpu_enabled() to vtime_accounting_enabled_this_cpu()
In reply to Frederic WeisbeckerStandardize the naming on top of the vtime_accounting_enabled_*() base. Also make it clear we are checking the vtime state of the *current* CPU with this function. We'll need to add an API to check that state on remote CPUs as well, so we must disambiguate the naming.
--- include/linux/context_tracking.h | 4 ++-- include/linux/vtime.h | 10 +++++----- kernel/sched/cputime.c | 2 +- kernel/time/tick-sched.c | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-)
include/linux/context_tracking.h +2 -2
@@ -103,7 +103,7 @@ static inline void context_tracking_init(void) { }/* must be called with irqs disabled */ static inline void guest_enter_irqoff (void ) { -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_guest_enter (current);else current->flags |=PF_VCPU ;
@@ -127,7 +127,7 @@ static inline void guest_exit_irqoff(void)if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_GUEST ); -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_guest_exit (current);else current->flags &= ~PF_VCPU ;
include/linux/vtime.h +5 -5
@@ -11,11 +11,11 @@struct task_struct ;/* -* vtime_accounting_cpu_enabled() definitions/declarations +* vtime_accounting_enabled_this_cpu() definitions/declarations */ #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) -static inline bool vtime_accounting_cpu_enabled (void ) {return true; } +static inline bool vtime_accounting_enabled_this_cpu (void ) {return true; }extern void vtime_task_switch (struct task_struct *prev);#elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN )
@@ -31,7 +31,7 @@ static inline bool vtime_accounting_enabled(void)return context_tracking_enabled (); } -static inline bool vtime_accounting_cpu_enabled (void ) +static inline bool vtime_accounting_enabled_this_cpu (void ) {if (vtime_accounting_enabled ()) {if (context_tracking_enabled_this_cpu ())
@@ -45,13 +45,13 @@ extern void vtime_task_switch_generic(struct task_struct *prev);static inline void vtime_task_switch (struct task_struct *prev) { -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_task_switch_generic (prev); }#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ -static inline bool vtime_accounting_cpu_enabled (void ) {return false; } +static inline bool vtime_accounting_enabled_this_cpu (void ) {return false; }static inline void vtime_task_switch (struct task_struct *prev) { }#endif
kernel/sched/cputime.c +1 -1
@@ -475,7 +475,7 @@ void account_process_tick(struct task_struct *p, int user_tick)u64 cputime, steal;struct rq *rq =this_rq (); -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())return ;if (sched_clock_irqtime) {
kernel/time/tick-sched.c +1 -1
@@ -1119,7 +1119,7 @@ static void tick_nohz_account_idle_ticks(struct tick_sched *ts)#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE unsigned long ticks; -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())return ;/* * We stopped the tick in idle. Update process times would miss the
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 9/14 sched/vtime: Introduce vtime_accounting_enabled_cpu()
In reply to Frederic WeisbeckerThis allows us to check if a remote CPU runs vtime accounting (ie: is nohz_full). We'll need that to reliably support reading kcpustat on nohz_full CPUs.
Also simplify a bit the condition in the local flavoured function while at it.
--- include/linux/vtime.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
include/linux/vtime.h +7 -6
@@ -31,14 +31,14 @@ static inline bool vtime_accounting_enabled(void)return context_tracking_enabled (); } +static inline bool vtime_accounting_enabled_cpu (int cpu) +{ +return (vtime_accounting_enabled () &&context_tracking_enabled_cpu (cpu)); +} +static inline bool vtime_accounting_enabled_this_cpu (void ) { -if (vtime_accounting_enabled ()) { -if (context_tracking_enabled_this_cpu ()) -return true; - } - -return false; +return (vtime_accounting_enabled () &&context_tracking_enabled_this_cpu ()); }extern void vtime_task_switch_generic (struct task_struct *prev);
@@ -51,6 +51,7 @@ static inline void vtime_task_switch(struct task_struct *prev)#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ +static inline bool vtime_accounting_enabled_cpu (int cpu) {return false; }static inline bool vtime_accounting_enabled_this_cpu (void ) {return false; }static inline void vtime_task_switch (struct task_struct *prev) { }
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 10/14 context_tracking: Check static key on context_tracking_enabled_*cpu()
In reply to Frederic Weisbeckerguest_enter() doesn't call context_tracking_enabled() before calling context_tracking_enabled_this_cpu(). Therefore the guest code doesn't benefit from the static key on the fast path.
Just make sure that context_tracking_enabled_*cpu() functions check the static key by themselves to propagate the optimization.
--- include/linux/context_tracking_state.h | 4 ++-- include/linux/vtime.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/context_tracking_state.h +2 -2
@@ -33,12 +33,12 @@ static inline bool context_tracking_enabled(void)static inline bool context_tracking_enabled_cpu (int cpu) { -return per_cpu (context_tracking.active, cpu); +return context_tracking_enabled () &&per_cpu (context_tracking.active, cpu); }static inline bool context_tracking_enabled_this_cpu (void ) { -return __this_cpu_read (context_tracking.active); +return context_tracking_enabled () &&__this_cpu_read (context_tracking.active); }static inline bool context_tracking_in_user (void )
include/linux/vtime.h +2 -2
@@ -33,12 +33,12 @@ static inline bool vtime_accounting_enabled(void)static inline bool vtime_accounting_enabled_cpu (int cpu) { -return (vtime_accounting_enabled () &&context_tracking_enabled_cpu (cpu)); +return context_tracking_enabled_cpu (cpu); }static inline bool vtime_accounting_enabled_this_cpu (void ) { -return (vtime_accounting_enabled () &&context_tracking_enabled_this_cpu ()); +return context_tracking_enabled_this_cpu (); }extern void vtime_task_switch_generic (struct task_struct *prev);
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 11/14 sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
In reply to Frederic WeisbeckerKcpustat is not correctly supported on nohz_full CPUs. The tick doesn't fire and the cputime therefore doesn't move forward. The issue has shown up after the vanishing of the remaining 1Hz which has made the stall visible.
We are solving that with checking the task running on a CPU through RCU and reading its vtime delta that we add to the raw kcpustat values.
We make sure that we fetch a coherent raw-kcpustat/vtime-delta couple sequence while checking that the CPU referred by the target vtime is the correct one, under the locked vtime seqcount.
Only CPUTIME_SYSTEM is handled here as a start because it's the trivial case. User and guest time will require more preparation work to correctly handle niceness.
--- include/linux/kernel_stat.h | 11 +++++ kernel/sched/cputime.c | 83 +++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+)
include/linux/kernel_stat.h +11 -0
@@ -78,6 +78,17 @@ static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu)return kstat_cpu (cpu).irqs_sum; } +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +extern u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu); +#else +static inline u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu) +{ +return kcpustat->cpustat[usage]; +} +#endif +extern void account_user_time (struct task_struct *,u64 );extern void account_guest_time (struct task_struct *,u64 );extern void account_system_time (struct task_struct *,int ,u64 );
kernel/sched/cputime.c +83 -0
87 lines not on this page · raw message
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 12/14 procfs: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerNow that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- fs/proc/stat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/proc/stat.c +2 -2
@@ -124,7 +124,7 @@ static int show_stat(struct seq_file *p, void *v) user += kcs->cpustat[CPUTIME_USER ]; nice += kcs->cpustat[CPUTIME_NICE ]; - system += kcs->cpustat[CPUTIME_SYSTEM ]; + system +=kcpustat_field (kcs,CPUTIME_SYSTEM , i); idle +=get_idle_time (kcs, i); iowait +=get_iowait_time (kcs, i); irq += kcs->cpustat[CPUTIME_IRQ ];
@@ -162,7 +162,7 @@ static int show_stat(struct seq_file *p, void *v)/* Copy values here to work around gcc-2.95.3, gcc-2.96 */ user = kcs->cpustat[CPUTIME_USER ]; nice = kcs->cpustat[CPUTIME_NICE ]; - system = kcs->cpustat[CPUTIME_SYSTEM ]; + system =kcpustat_field (kcs,CPUTIME_SYSTEM , i); idle =get_idle_time (kcs, i); iowait =get_iowait_time (kcs, i); irq = kcs->cpustat[CPUTIME_IRQ ];
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 14/14 leds: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerNow that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- drivers/leds/trigger/ledtrig-activity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/leds/trigger/ledtrig-activity.c +1 -1
@@ -59,7 +59,7 @@ static void led_activity_function(struct timer_list *t)for_each_possible_cpu (i) { curr_used +=kcpustat_cpu (i).cpustat[CPUTIME_USER ] +kcpustat_cpu (i).cpustat[CPUTIME_NICE ] - +kcpustat_cpu (i).cpustat[CPUTIME_SYSTEM ] + +kcpustat_field (&kcpustat_cpu (i),CPUTIME_SYSTEM , i) +kcpustat_cpu (i).cpustat[CPUTIME_SOFTIRQ ] +kcpustat_cpu (i).cpustat[CPUTIME_IRQ ]; cpus++;
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 13/14 cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerNow that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- drivers/cpufreq/cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/cpufreq/cpufreq.c +1 -1
@@ -120,7 +120,7 @@ static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) cur_wall_time =jiffies64_to_nsecs (get_jiffies_64 ()); busy_time =kcpustat_cpu (cpu).cpustat[CPUTIME_USER ]; - busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SYSTEM ]; + busy_time +=kcpustat_field (&kcpustat_cpu (cpu),CPUTIME_SYSTEM , cpu); busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_IRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SOFTIRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_STEAL ];
Signature · 1 line
2.23.0
Re: [PATCH 13/14] cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
Quoted 22 lines · Frederic Weisbecker
Now that we have a vtime safe kcpustat accessor for
CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values onnohz_fullCPUs.Reported-by Yauheni Kaliuta <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>Cc Rafael J. Wysocki <[email protected]>Cc Viresh Kumar <[email protected]>Cc Yauheni Kaliuta <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Rik van Riel <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Wanpeng Li <[email protected]>Cc Ingo Molnar <[email protected]>--- drivers/cpufreq/cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)drivers/cpufreq/cpufreq.c +0
-0
drivers/cpufreq/cpufreq.c static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) · line 126
Quoted 8 lines
drivers/cpufreq/cpufreq.c +1
-1@@ -120,7 +120,7 @@ static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) cur_wall_time =jiffies64_to_nsecs (get_jiffies_64 ()); busy_time =kcpustat_cpu (cpu).cpustat[CPUTIME_USER ]; - busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SYSTEM ]; + busy_time +=kcpustat_field (&kcpustat_cpu (cpu),CPUTIME_SYSTEM , cpu); busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_IRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SOFTIRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_STEAL ];
Signature · 1 line
viresh
Frederic Weisbecker
maintainer[GIT PULL] sched/nohz: Make kcpustat's CPUTIME_SYSTEM vtime aware
In reply to Frederic WeisbeckerIngo, Peter,
Please pull the nohz/kcpustat-for-tip branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
nohz/kcpustat-for-tip
HEAD: e179e89320c53a96c5d585af38126cfb124da789
Thanks,
Frederic
---
Frederic Weisbecker (14):
sched/vtime: Record CPU under seqcount for kcpustat needs
sched/cputime: Add vtime idle task state
sched/cputime: Add vtime guest task state
context_tracking: Remove context_tracking_active()
context_tracking: s/context_tracking_is_enabled/context_tracking_enabled()
context_tracking: Rename context_tracking_is_cpu_enabled() to context_tracking_enabled_this_cpu()
context_tracking: Introduce context_tracking_enabled_cpu()
sched/vtime: Rename vtime_accounting_cpu_enabled() to vtime_accounting_enabled_this_cpu() sched/vtime: Introduce vtime_accounting_enabled_cpu()
context_tracking: Check static key on context_tracking_enabled_*cpu() sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
procfs: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
leds: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
arch/x86/entry/calling.h | 2 +- drivers/cpufreq/cpufreq.c | 2 +- drivers/leds/trigger/ledtrig-activity.c | 2 +- fs/proc/stat.c | 4 +- include/linux/context_tracking.h | 26 +++---- include/linux/context_tracking_state.h | 21 +++--- include/linux/kernel_stat.h | 11 +++ include/linux/sched.h | 9 ++- include/linux/tick.h | 2 +- include/linux/vtime.h | 23 +++--- kernel/context_tracking.c | 6 +- kernel/sched/cputime.c | 119 ++++++++++++++++++++++++++++---- kernel/time/tick-sched.c | 2 +- 13 files changed, 172 insertions(+), 57 deletions(-)
Peter Zijlstra
maintainerRe: [PATCH 11/14] sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
In reply to Frederic WeisbeckerQuoted 8 lines · Frederic Weisbecker
kernel/sched/cputime.c +8
-0+static int kcpustat_field_vtime (u64 *cpustat, +struct vtime *vtime, +enum cpu_usage_stat usage, +int cpu,u64 *val) +{ +unsigned int seq; +int err; +
kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 932
Quoted 10 lines
kernel/sched/cputime.c +10
-0+ do { + seq = read_seqcount_begin(&vtime->seqcount); + + /* + * We raced against context switch, fetch the + * kcpustat task again. + */ + if (vtime->cpu != cpu && vtime->cpu != -1) { + err = -EAGAIN; + continue;
Did that want to be break?
Quoted 4 lines
kernel/sched/cputime.c +4
-0+ } + +/* +* Two possible things here:
kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 946
Quoted 10 lines
kernel/sched/cputime.c +10
-0+* 1) We are seeing the scheduling out task (prev) or any past one. +* 2) We are seeing the scheduling in task (next) but it hasn't +* passed though vtime_task_switch() yet so the pending +* cputime of the prev task may not be flushed yet. +* +* Case 1) is ok but 2) is not. So wait for a safe VTIME state. +*/ +if (vtime->state ==VTIME_INACTIVE ) { + err = -EAGAIN ; +continue ;
Idem.
If so, you can do return -EAGAIN here, and return 0 at the end and get rid of err.
Also, if you're spin-waiting here, there should probably be a cpu_relax() before the return -EAGAIN.
And in case that is so, you probably want the rcu_read_lock() section below _inside_ the do{}while loop, such that the RCU section doesn't cover the entire spin-wait.
Quoted 26 lines
kernel/sched/cputime.c +26
-0+ } + + err =0 ; + + *val = cpustat[usage]; + +if (vtime->state ==VTIME_SYS ) + *val += vtime->stime +vtime_delta (vtime); + + }while (read_seqcount_retry (&vtime->seqcount, seq)); + +return err; +} + +u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu) +{ +u64 val; +int err; +u64 *cpustat = kcpustat->cpustat; + +if (!vtime_accounting_enabled_cpu (cpu)) +return cpustat[usage]; + +/* Only support sys vtime for now */ +if (usage !=CPUTIME_SYSTEM )
kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 982
Quoted 10 lines
kernel/sched/cputime.c +10
-0+return cpustat[usage]; + +rcu_read_lock (); + + do { + struct rq *rq = cpu_rq(cpu); + struct task_struct *curr; + struct vtime *vtime; + + curr = rcu_dereference(rq->curr);
This is indeed safe now (relies on commit
5311a98fef7d ("tasks, sched/core: RCUify the assignment of rq->curr")
)
Quoted 18 lines
kernel/sched/cputime.c +14
-0+if (WARN_ON_ONCE (!curr)) { + val = cpustat[usage]; +break ; + } + + vtime = &curr->vtime; + err = kcpustat_field_vtime(cpustat, vtime, usage, cpu, &val); + }while (err == -EAGAIN ); + +rcu_read_unlock (); + +return val; +} +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerRe: [PATCH 11/14] sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
Quoted 23 lines · Peter Zijlstra
Quoted 8 lines · Frederic Weisbecker
kernel/sched/cputime.c +8
-0+static int kcpustat_field_vtime (u64 *cpustat, +struct vtime *vtime, +enum cpu_usage_stat usage, +int cpu,u64 *val) +{ +unsigned int seq; +int err; +kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 932
Quoted 10 lines
kernel/sched/cputime.c +10
-0+ do { + seq = read_seqcount_begin(&vtime->seqcount); + + /* + * We raced against context switch, fetch the + * kcpustat task again. + */ + if (vtime->cpu != cpu && vtime->cpu != -1) { + err = -EAGAIN; + continue;Did that want to be break?
Quoted 16 lines
Quoted 4 lines
kernel/sched/cputime.c +4
-0+ } + +/* +* Two possible things here: kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 946
Quoted 10 lines
kernel/sched/cputime.c +10
-0+* 1) We are seeing the scheduling out task (prev) or any past one. +* 2) We are seeing the scheduling in task (next) but it hasn't +* passed though vtime_task_switch() yet so the pending +* cputime of the prev task may not be flushed yet. +* +* Case 1) is ok but 2) is not. So wait for a safe VTIME state. +*/ +if (vtime->state ==VTIME_INACTIVE ) { + err = -EAGAIN ; +continue ;Idem.
Well, both were meant to be continue. Which means do the same as break but just in case we raced with the updater, try again with the same task.
Now as we are checking again, we may as well reload the task indeed so I'll turn those into break.
Quoted 10 lines
If so, you can do return
-EAGAINhere, and return 0 at the end and get rid of err.Also, if you're spin-waiting here, there should probably be a
cpu_relax()before the return-EAGAIN.And in case that is so, you probably want the
rcu_read_lock()section below _inside_ the do{}while loop, such that the RCU section doesn't cover the entire spin-wait.
Good point!
Quoted 11 lines
kernel/sched/cputime.c void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) · line 982
Quoted 7 lines
kernel/sched/cputime.c +7
-0+ + do { + struct rq *rq = cpu_rq(cpu); + struct task_struct *curr; + struct vtime *vtime; + + curr = rcu_dereference(rq->curr);This is indeed safe now (relies on commit
5311a98fef7d("tasks, sched/core: RCUify the assignment ofrq->curr")
Yeah and that has simplified the patchset a lot.
Thanks!
Frederic Weisbecker
maintainerPATCH v2 11/14 sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
In reply to Frederic WeisbeckerKcpustat is not correctly supported on nohz_full CPUs. The tick doesn't fire and the cputime therefore doesn't move forward. The issue has shown up after the vanishing of the remaining 1Hz which has made the stall visible.
We are solving that with checking the task running on a CPU through RCU and reading its vtime delta that we add to the raw kcpustat values.
We make sure that we fetch a coherent raw-kcpustat/vtime-delta couple sequence while checking that the CPU referred by the target vtime is the correct one, under the locked vtime seqcount.
Only CPUTIME_SYSTEM is handled here as a start because it's the trivial case. User and guest time will require more preparation work to correctly handle niceness.
--- include/linux/kernel_stat.h | 11 +++++ kernel/sched/cputime.c | 82 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+)
include/linux/kernel_stat.h +11 -0
@@ -78,6 +78,17 @@ static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu)return kstat_cpu (cpu).irqs_sum; } +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +extern u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu); +#else +static inline u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu) +{ +return kcpustat->cpustat[usage]; +} +#endif +extern void account_user_time (struct task_struct *,u64 );extern void account_guest_time (struct task_struct *,u64 );extern void account_system_time (struct task_struct *,int ,u64 );
kernel/sched/cputime.c +82 -0
86 lines not on this page · raw message
Signature · 1 line
2.23.0
[tip: sched/core] context_tracking: Introduce context_tracking_enabled_cpu()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 097f2541c6e51e0c1cdb1e6d46ef08a624336518 Gitweb: https://git.kernel.org/tip/097f2541c6e51e0c1cdb1e6d46ef08a624336518 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:53 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:13 +01:00
context_tracking: Introduce context_tracking_enabled_cpu()
This allows us to check if a remote CPU runs context tracking (ie: is nohz_full). We'll need that to reliably support "nice" accounting on kcpustat.
--- include/linux/context_tracking_state.h | 6 ++++++ 1 file changed, 6 insertions(+)
include/linux/context_tracking_state.h +6 -0
@@ -31,6 +31,11 @@ static inline bool context_tracking_enabled(void)return static_branch_unlikely (&context_tracking_key); } +static inline bool context_tracking_enabled_cpu (int cpu) +{ +return per_cpu (context_tracking.active, cpu); +} +static inline bool context_tracking_enabled_this_cpu (void ) {return __this_cpu_read (context_tracking.active);
@@ -43,6 +48,7 @@ static inline bool context_tracking_in_user(void)#else static inline bool context_tracking_in_user (void ) {return false; }static inline bool context_tracking_enabled (void ) {return false; } +static inline bool context_tracking_enabled_cpu (int cpu) {return false; }static inline bool context_tracking_enabled_this_cpu (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
[tip: sched/core] sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 64eea63c19a2c386a96638f4e54a1355510709e3 Gitweb: https://git.kernel.org/tip/64eea63c19a2c386a96638f4e54a1355510709e3 Author: Frederic Weisbecker <[email protected]> AuthorDate: Fri, 25 Oct 2019 04:03:03 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:17 +01:00
sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM
Kcpustat is not correctly supported on nohz_full CPUs. The tick doesn't fire and the cputime therefore doesn't move forward. The issue has shown up after the vanishing of the remaining 1Hz which has made the stall visible.
We are solving that with checking the task running on a CPU through RCU and reading its vtime delta that we add to the raw kcpustat values.
We make sure that we fetch a coherent raw-kcpustat/vtime-delta couple sequence while checking that the CPU referred by the target vtime is the correct one, under the locked vtime seqcount.
Only CPUTIME_SYSTEM is handled here as a start because it's the trivial case. User and guest time will require more preparation work to correctly handle niceness.
--- include/linux/kernel_stat.h | 11 +++++- kernel/sched/cputime.c | 82 ++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+)
include/linux/kernel_stat.h +11 -0
@@ -78,6 +78,17 @@ static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu)return kstat_cpu (cpu).irqs_sum; } +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +extern u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu); +#else +static inline u64 kcpustat_field (struct kernel_cpustat *kcpustat, +enum cpu_usage_stat usage,int cpu) +{ +return kcpustat->cpustat[usage]; +} +#endif +extern void account_user_time (struct task_struct *,u64 );extern void account_guest_time (struct task_struct *,u64 );extern void account_system_time (struct task_struct *,int ,u64 );
kernel/sched/cputime.c +82 -0
86 lines not on this page · raw message
[tip: sched/core] context_tracking: Rename context_tracking_is_enabled() => context_tracking_enabled()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 74c578759f15cb5a0d0107759bdad671d7b52ab9 Gitweb: https://git.kernel.org/tip/74c578759f15cb5a0d0107759bdad671d7b52ab9 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:51 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:12 +01:00
context_tracking: Rename context_tracking_is_enabled() => context_tracking_enabled()
Remove the superfluous "is" in the middle of the name. We want to standardize the naming so that it can be expanded through suffixes:
context_tracking_enabled()
context_tracking_enabled_cpu()
context_tracking_enabled_this_cpu()
--- arch/x86/entry/calling.h | 2 +- include/linux/context_tracking.h | 20 ++++++++++---------- include/linux/context_tracking_state.h | 8 ++++---- include/linux/tick.h | 2 +- include/linux/vtime.h | 2 +- kernel/context_tracking.c | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-)
arch/x86/entry/calling.h +1 -1
@@ -354,7 +354,7 @@ For 32-bit we have the following conventions - kernel is built with .macro CALL_enter_from_user_mode#ifdef CONFIG_CONTEXT_TRACKING #ifdef CONFIG_JUMP_LABEL - STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 + STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_key, def=0#endif call enter_from_user_mode .Lafter_call_\@:
include/linux/context_tracking.h +10 -10
@@ -22,26 +22,26 @@ extern void context_tracking_user_exit(void);static inline void user_enter (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())context_tracking_enter (CONTEXT_USER ); }static inline void user_exit (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())context_tracking_exit (CONTEXT_USER ); }/* Called with interrupts disabled. */ static inline void user_enter_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_enter (CONTEXT_USER ); }static inline void user_exit_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_USER ); }
@@ -49,7 +49,7 @@ static inline enum ctx_state exception_enter(void) {enum ctx_state prev_ctx; -if (!context_tracking_is_enabled ()) +if (!context_tracking_enabled ())return 0 ; prev_ctx =this_cpu_read (context_tracking.state);
@@ -61,7 +61,7 @@ static inline enum ctx_state exception_enter(void)static inline void exception_exit (enum ctx_state prev_ctx) { -if (context_tracking_is_enabled ()) { +if (context_tracking_enabled ()) {if (prev_ctx !=CONTEXT_KERNEL )context_tracking_enter (prev_ctx); }
@@ -77,7 +77,7 @@ static inline void exception_exit(enum ctx_state prev_ctx)*/ static inline enum ctx_state ct_state (void ) { -return context_tracking_is_enabled () ? +return context_tracking_enabled () ?this_cpu_read (context_tracking.state) :CONTEXT_DISABLED ; }#else
@@ -90,7 +90,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) { }static inline enum ctx_state ct_state (void ) {return CONTEXT_DISABLED ; }#endif /* !CONFIG_CONTEXT_TRACKING */ -#define CT_WARN_ON (cond) WARN_ON(context_tracking_is_enabled() && (cond)) +#define CT_WARN_ON (cond) WARN_ON(context_tracking_enabled() && (cond))#ifdef CONFIG_CONTEXT_TRACKING_FORCE extern void context_tracking_init (void );
@@ -108,7 +108,7 @@ static inline void guest_enter_irqoff(void)else current->flags |=PF_VCPU ; -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_enter (CONTEXT_GUEST );/* KVM does not hold any references to rcu protected data when it
@@ -124,7 +124,7 @@ static inline void guest_enter_irqoff(void)static inline void guest_exit_irqoff (void ) { -if (context_tracking_is_enabled ()) +if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_GUEST );if (vtime_accounting_cpu_enabled ())
include/linux/context_tracking_state.h +4 -4
@@ -23,12 +23,12 @@ struct context_tracking { };#ifdef CONFIG_CONTEXT_TRACKING -extern struct static_key_false context_tracking_enabled; +extern struct static_key_false context_tracking_key; DECLARE_PER_CPU(struct context_tracking, context_tracking); -static inline bool context_tracking_is_enabled (void ) +static inline bool context_tracking_enabled (void ) { -return static_branch_unlikely (&context_tracking_enabled); +return static_branch_unlikely (&context_tracking_key); }static inline bool context_tracking_cpu_is_enabled (void )
@@ -42,7 +42,7 @@ static inline bool context_tracking_in_user(void) }#else static inline bool context_tracking_in_user (void ) {return false; } -static inline bool context_tracking_is_enabled (void ) {return false; } +static inline bool context_tracking_enabled (void ) {return false; }static inline bool context_tracking_cpu_is_enabled (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
include/linux/tick.h +1 -1
@@ -174,7 +174,7 @@ extern cpumask_var_t tick_nohz_full_mask;static inline bool tick_nohz_full_enabled (void ) { -if (!context_tracking_is_enabled ()) +if (!context_tracking_enabled ())return false;return tick_nohz_full_running;
include/linux/vtime.h +1 -1
@@ -28,7 +28,7 @@ extern void vtime_task_switch(struct task_struct *prev);*/ static inline bool vtime_accounting_enabled (void ) { -return context_tracking_is_enabled (); +return context_tracking_enabled (); }static inline bool vtime_accounting_cpu_enabled (void )
kernel/context_tracking.c +3 -3
@@ -25,8 +25,8 @@#define CREATE_TRACE_POINTS #include <trace/events/context_tracking.h> -DEFINE_STATIC_KEY_FALSE (context_tracking_enabled); -EXPORT_SYMBOL_GPL (context_tracking_enabled); +DEFINE_STATIC_KEY_FALSE (context_tracking_key); +EXPORT_SYMBOL_GPL (context_tracking_key); DEFINE_PER_CPU(struct context_tracking, context_tracking);EXPORT_SYMBOL_GPL (context_tracking);
@@ -192,7 +192,7 @@ void __init context_tracking_cpu_set(int cpu)if (!per_cpu (context_tracking.active, cpu)) {per_cpu (context_tracking.active, cpu) = true; -static_branch_inc (&context_tracking_enabled); +static_branch_inc (&context_tracking_key); }if (initialized)
[tip: sched/core] context_tracking: Remove context_tracking_active()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 0ca167c056ea3d637b7959e6e94010fa526b9b8c Gitweb: https://git.kernel.org/tip/0ca167c056ea3d637b7959e6e94010fa526b9b8c Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:50 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:11 +01:00
context_tracking: Remove context_tracking_active()
This function is a leftover from old removal or rename. We can drop it.
--- include/linux/context_tracking_state.h | 1 - 1 file changed, 1 deletion(-)
include/linux/context_tracking_state.h +0 -1
@@ -42,7 +42,6 @@ static inline bool context_tracking_in_user(void) }#else static inline bool context_tracking_in_user (void ) {return false; } -static inline bool context_tracking_active (void ) {return false; }static inline bool context_tracking_is_enabled (void ) {return false; }static inline bool context_tracking_cpu_is_enabled (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */
[tip: sched/core] sched/vtime: Record CPU under seqcount for kcpustat needs
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 802f4a827f139f2581b3c50c69d20f8bf4c24af1 Gitweb: https://git.kernel.org/tip/802f4a827f139f2581b3c50c69d20f8bf4c24af1 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:47 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:08 +01:00
sched/vtime: Record CPU under seqcount for kcpustat needs
In order to compute the kcpustat delta on a nohz CPU, we'll need to fetch the task running on that target. Checking that its vtime state snapshot actually refers to the relevant target involves recording that CPU under the seqcount locked on task switch.
This is a step toward making kcpustat moving forward on full nohz CPUs.
--- include/linux/sched.h | 1 + kernel/sched/cputime.c | 3 +++ 2 files changed, 4 insertions(+)
include/linux/sched.h +1 -0
@@ -259,6 +259,7 @@ struct vtime {seqcount_t seqcount;unsigned long long starttime;enum vtime_state state; +unsigned int cpu;u64 utime;u64 stime;u64 gtime;
kernel/sched/cputime.c +3 -0
@@ -818,6 +818,7 @@ void vtime_task_switch_generic(struct task_struct *prev)else __vtime_account_kernel (prev, vtime); vtime->state =VTIME_INACTIVE ; + vtime->cpu =-1 ;write_seqcount_end (&vtime->seqcount); vtime = ¤t->vtime;
@@ -825,6 +826,7 @@ void vtime_task_switch_generic(struct task_struct *prev)write_seqcount_begin (&vtime->seqcount); vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); + vtime->cpu =smp_processor_id ();write_seqcount_end (&vtime->seqcount); }
@@ -837,6 +839,7 @@ void vtime_init_idle(struct task_struct *t, int cpu)write_seqcount_begin (&vtime->seqcount); vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); + vtime->cpu = cpu;write_seqcount_end (&vtime->seqcount);local_irq_restore (flags); }
[tip: sched/core] sched/cputime: Add vtime guest task state
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: e6d5bf3e321ca664d12eb00ceb40bd58987ce8a1 Gitweb: https://git.kernel.org/tip/e6d5bf3e321ca664d12eb00ceb40bd58987ce8a1 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:49 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:11 +01:00
sched/cputime: Add vtime guest task state
Record guest as a VTIME state instead of guessing it from VTIME_SYS and PF_VCPU. This is going to simplify the cputime read side especially as its state machine is going to further expand in order to fully support kcpustat on nohz_full.
--- include/linux/sched.h | 2 ++ kernel/sched/cputime.c | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-)
include/linux/sched.h +2 -0
@@ -255,6 +255,8 @@ enum vtime_state {VTIME_SYS ,/* Task runs in userspace in a CPU with VTIME active: */ VTIME_USER , +/* Task runs as guests in a CPU with VTIME active: */ + VTIME_GUEST, }; struct vtime {
kernel/sched/cputime.c +11 -7
@@ -733,7 +733,7 @@ static void __vtime_account_kernel(struct task_struct *tsk, struct vtime *vtime) {/* We might have scheduled out from guest path */ -if (tsk->flags &PF_VCPU ) +if (vtime->state ==VTIME_GUEST )vtime_account_guest (tsk, vtime);else vtime_account_system (tsk, vtime);
@@ -788,6 +788,7 @@ void vtime_guest_enter(struct task_struct *tsk)write_seqcount_begin (&vtime->seqcount);vtime_account_system (tsk, vtime); tsk->flags |=PF_VCPU ; + vtime->state =VTIME_GUEST ;write_seqcount_end (&vtime->seqcount); }EXPORT_SYMBOL_GPL (vtime_guest_enter);
@@ -799,6 +800,7 @@ void vtime_guest_exit(struct task_struct *tsk)write_seqcount_begin (&vtime->seqcount);vtime_account_guest (tsk, vtime); tsk->flags &= ~PF_VCPU ; + vtime->state =VTIME_SYS ;write_seqcount_end (&vtime->seqcount); }EXPORT_SYMBOL_GPL (vtime_guest_exit);
@@ -826,6 +828,8 @@ void vtime_task_switch_generic(struct task_struct *prev)write_seqcount_begin (&vtime->seqcount);if (is_idle_task (current)) vtime->state =VTIME_IDLE ; +else if (current->flags &PF_VCPU ) + vtime->state =VTIME_GUEST ;else vtime->state =VTIME_SYS ; vtime->starttime =sched_clock ();
@@ -860,7 +864,7 @@ u64 task_gtime(struct task_struct *t) seq =read_seqcount_begin (&vtime->seqcount); gtime = t->gtime; -if (vtime->state ==VTIME_SYS && t->flags &PF_VCPU ) +if (vtime->state ==VTIME_GUEST ) gtime += vtime->gtime +vtime_delta (vtime); }while (read_seqcount_retry (&vtime->seqcount, seq));
@@ -898,13 +902,13 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) delta =vtime_delta (vtime);/* -* Task runs either in user or kernel space, add pending nohz time to -* the right place. +* Task runs either in user (including guest) or kernel space, +* add pending nohz time to the right place. */ -if (vtime->state ==VTIME_USER || t->flags &PF_VCPU ) - *utime += vtime->utime + delta; -else if (vtime->state ==VTIME_SYS ) +if (vtime->state ==VTIME_SYS ) *stime += vtime->stime + delta; +else + *utime += vtime->utime + delta; }while (read_seqcount_retry (&vtime->seqcount, seq)); }#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
[tip: sched/core] sched/cputime: Add vtime idle task state
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 14faf6fcac4ba33f8fd8d9b2d0278010a9eb1742 Gitweb: https://git.kernel.org/tip/14faf6fcac4ba33f8fd8d9b2d0278010a9eb1742 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:48 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:10 +01:00
sched/cputime: Add vtime idle task state
Record idle as a VTIME state instead of guessing it from VTIME_SYS and is_idle_task(). This is going to simplify the cputime read side especially as its state machine is going to further expand in order to fully support kcpustat on nohz_full.
--- include/linux/sched.h | 6 ++++-- kernel/sched/cputime.c | 13 ++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-)
include/linux/sched.h +4 -2
@@ -249,10 +249,12 @@ struct prev_cputime {enum vtime_state {/* Task is sleeping or running in a CPU with VTIME inactive: */ VTIME_INACTIVE =0 , -/* Task runs in userspace in a CPU with VTIME active: */ -VTIME_USER , +/* Task is idle */ +VTIME_IDLE ,/* Task runs in kernelspace in a CPU with VTIME active: */ VTIME_SYS , +/* Task runs in userspace in a CPU with VTIME active: */ +VTIME_USER , };struct vtime {
kernel/sched/cputime.c +8 -5
@@ -813,7 +813,7 @@ void vtime_task_switch_generic(struct task_struct *prev)struct vtime *vtime = &prev->vtime;write_seqcount_begin (&vtime->seqcount); -if (is_idle_task (prev)) +if (vtime->state ==VTIME_IDLE )vtime_account_idle (prev);else __vtime_account_kernel (prev, vtime);
@@ -824,7 +824,10 @@ void vtime_task_switch_generic(struct task_struct *prev) vtime = ¤t->vtime;write_seqcount_begin (&vtime->seqcount); - vtime->state =VTIME_SYS ; +if (is_idle_task (current)) + vtime->state =VTIME_IDLE ; +else + vtime->state =VTIME_SYS ; vtime->starttime =sched_clock (); vtime->cpu =smp_processor_id ();write_seqcount_end (&vtime->seqcount);
@@ -837,7 +840,7 @@ void vtime_init_idle(struct task_struct *t, int cpu)local_irq_save (flags);write_seqcount_begin (&vtime->seqcount); - vtime->state =VTIME_SYS ; + vtime->state =VTIME_IDLE ; vtime->starttime =sched_clock (); vtime->cpu = cpu;write_seqcount_end (&vtime->seqcount);
@@ -888,8 +891,8 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime) *utime = t->utime; *stime = t->stime; -/* Task is sleeping, nothing to add */ -if (vtime->state ==VTIME_INACTIVE ||is_idle_task (t)) +/* Task is sleeping or idle, nothing to add */ +if (vtime->state <VTIME_SYS )continue ; delta =vtime_delta (vtime);
[tip: sched/core] sched/vtime: Introduce vtime_accounting_enabled_cpu()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 9adbb9dd4c4eb45e1129fc73d8de69ca72350f81 Gitweb: https://git.kernel.org/tip/9adbb9dd4c4eb45e1129fc73d8de69ca72350f81 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:55 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:15 +01:00
sched/vtime: Introduce vtime_accounting_enabled_cpu()
This allows us to check if a remote CPU runs vtime accounting (ie: is nohz_full). We'll need that to reliably support reading kcpustat on nohz_full CPUs.
Also simplify a bit the condition in the local flavoured function while at it.
--- include/linux/vtime.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
include/linux/vtime.h +7 -6
@@ -31,14 +31,14 @@ static inline bool vtime_accounting_enabled(void)return context_tracking_enabled (); } -static inline bool vtime_accounting_enabled_this_cpu (void ) +static inline bool vtime_accounting_enabled_cpu (int cpu) { -if (vtime_accounting_enabled ()) { -if (context_tracking_enabled_this_cpu ()) -return true; - } +return (vtime_accounting_enabled () &&context_tracking_enabled_cpu (cpu)); +} -return false; +static inline bool vtime_accounting_enabled_this_cpu (void ) +{ +return (vtime_accounting_enabled () &&context_tracking_enabled_this_cpu ()); }extern void vtime_task_switch_generic (struct task_struct *prev);
@@ -51,6 +51,7 @@ static inline void vtime_task_switch(struct task_struct *prev)#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ +static inline bool vtime_accounting_enabled_cpu (int cpu) {return false; }static inline bool vtime_accounting_enabled_this_cpu (void ) {return false; }static inline void vtime_task_switch (struct task_struct *prev) { }
[tip: sched/core] leds: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: e79b3ddad6790aabead5fb775456a5b0599e97ee Gitweb: https://git.kernel.org/tip/e79b3ddad6790aabead5fb775456a5b0599e97ee Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:57:00 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:19 +01:00
leds: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
Now that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- drivers/leds/trigger/ledtrig-activity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/leds/trigger/ledtrig-activity.c +1 -1
@@ -59,7 +59,7 @@ static void led_activity_function(struct timer_list *t)for_each_possible_cpu (i) { curr_used +=kcpustat_cpu (i).cpustat[CPUTIME_USER ] +kcpustat_cpu (i).cpustat[CPUTIME_NICE ] - +kcpustat_cpu (i).cpustat[CPUTIME_SYSTEM ] + +kcpustat_field (&kcpustat_cpu (i),CPUTIME_SYSTEM , i) +kcpustat_cpu (i).cpustat[CPUTIME_SOFTIRQ ] +kcpustat_cpu (i).cpustat[CPUTIME_IRQ ]; cpus++;
[tip: sched/core] procfs: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: ae37fe5c07508e1c3dcdd41c9127e5d50d31013d Gitweb: https://git.kernel.org/tip/ae37fe5c07508e1c3dcdd41c9127e5d50d31013d Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:58 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:17 +01:00
procfs: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
Now that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- fs/proc/stat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/proc/stat.c +2 -2
@@ -124,7 +124,7 @@ static int show_stat(struct seq_file *p, void *v) user += kcs->cpustat[CPUTIME_USER ]; nice += kcs->cpustat[CPUTIME_NICE ]; - system += kcs->cpustat[CPUTIME_SYSTEM ]; + system +=kcpustat_field (kcs,CPUTIME_SYSTEM , i); idle +=get_idle_time (kcs, i); iowait +=get_iowait_time (kcs, i); irq += kcs->cpustat[CPUTIME_IRQ ];
@@ -162,7 +162,7 @@ static int show_stat(struct seq_file *p, void *v)/* Copy values here to work around gcc-2.95.3, gcc-2.96 */ user = kcs->cpustat[CPUTIME_USER ]; nice = kcs->cpustat[CPUTIME_NICE ]; - system = kcs->cpustat[CPUTIME_SYSTEM ]; + system =kcpustat_field (kcs,CPUTIME_SYSTEM , i); idle =get_idle_time (kcs, i); iowait =get_iowait_time (kcs, i); irq = kcs->cpustat[CPUTIME_IRQ ];
[tip: sched/core] sched/vtime: Rename vtime_accounting_cpu_enabled() to vtime_accounting_enabled_this_cpu()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: e44fcb4b7a299602fb300b82a546c0b8a50d9d90 Gitweb: https://git.kernel.org/tip/e44fcb4b7a299602fb300b82a546c0b8a50d9d90 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:54 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:14 +01:00
sched/vtime: Rename vtime_accounting_cpu_enabled() to vtime_accounting_enabled_this_cpu()
Standardize the naming on top of the vtime_accounting_enabled_*() base. Also make it clear we are checking the vtime state of the *current* CPU with this function. We'll need to add an API to check that state on remote CPUs as well, so we must disambiguate the naming.
--- include/linux/context_tracking.h | 4 ++-- include/linux/vtime.h | 10 +++++----- kernel/sched/cputime.c | 2 +- kernel/time/tick-sched.c | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-)
include/linux/context_tracking.h +2 -2
@@ -103,7 +103,7 @@ static inline void context_tracking_init(void) { }/* must be called with irqs disabled */ static inline void guest_enter_irqoff (void ) { -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_guest_enter (current);else current->flags |=PF_VCPU ;
@@ -127,7 +127,7 @@ static inline void guest_exit_irqoff(void)if (context_tracking_enabled ())__context_tracking_exit (CONTEXT_GUEST ); -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_guest_exit (current);else current->flags &= ~PF_VCPU ;
include/linux/vtime.h +5 -5
@@ -11,11 +11,11 @@struct task_struct ;/* -* vtime_accounting_cpu_enabled() definitions/declarations +* vtime_accounting_enabled_this_cpu() definitions/declarations */ #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) -static inline bool vtime_accounting_cpu_enabled (void ) {return true; } +static inline bool vtime_accounting_enabled_this_cpu (void ) {return true; }extern void vtime_task_switch (struct task_struct *prev);#elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN )
@@ -31,7 +31,7 @@ static inline bool vtime_accounting_enabled(void)return context_tracking_enabled (); } -static inline bool vtime_accounting_cpu_enabled (void ) +static inline bool vtime_accounting_enabled_this_cpu (void ) {if (vtime_accounting_enabled ()) {if (context_tracking_enabled_this_cpu ())
@@ -45,13 +45,13 @@ extern void vtime_task_switch_generic(struct task_struct *prev);static inline void vtime_task_switch (struct task_struct *prev) { -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())vtime_task_switch_generic (prev); }#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ -static inline bool vtime_accounting_cpu_enabled (void ) {return false; } +static inline bool vtime_accounting_enabled_this_cpu (void ) {return false; }static inline void vtime_task_switch (struct task_struct *prev) { }#endif
kernel/sched/cputime.c +1 -1
@@ -475,7 +475,7 @@ void account_process_tick(struct task_struct *p, int user_tick)u64 cputime, steal;struct rq *rq =this_rq (); -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())return ;if (sched_clock_irqtime) {
kernel/time/tick-sched.c +1 -1
@@ -1119,7 +1119,7 @@ static void tick_nohz_account_idle_ticks(struct tick_sched *ts)#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE unsigned long ticks; -if (vtime_accounting_cpu_enabled ()) +if (vtime_accounting_enabled_this_cpu ())return ;/* * We stopped the tick in idle. Update process times would miss the
[tip: sched/core] context_tracking: Rename context_tracking_is_cpu_enabled() to context_tracking_enabled_this_cpu()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 84e0dacd0c347e9ee2531052013babd84683245f Gitweb: https://git.kernel.org/tip/84e0dacd0c347e9ee2531052013babd84683245f Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:52 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:12 +01:00
context_tracking: Rename context_tracking_is_cpu_enabled() to context_tracking_enabled_this_cpu()
Standardize the naming on top of the context_tracking_enabled_*() base. Also make it clear we are checking the context tracking state of the *current* CPU with this function. We'll need to add an API to check that state on remote CPUs as well, so we must disambiguate the naming.
--- include/linux/context_tracking.h | 2 +- include/linux/context_tracking_state.h | 4 ++-- include/linux/vtime.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
include/linux/context_tracking.h +1 -1
@@ -118,7 +118,7 @@ static inline void guest_enter_irqoff(void)* one time slice). Lets treat guest mode as quiescent state, just like * we do with user-mode execution. */ -if (!context_tracking_cpu_is_enabled ()) +if (!context_tracking_enabled_this_cpu ())rcu_virt_note_context_switch (smp_processor_id ()); }
include/linux/context_tracking_state.h +2 -2
@@ -31,7 +31,7 @@ static inline bool context_tracking_enabled(void)return static_branch_unlikely (&context_tracking_key); } -static inline bool context_tracking_cpu_is_enabled (void ) +static inline bool context_tracking_enabled_this_cpu (void ) {return __this_cpu_read (context_tracking.active); }
@@ -43,7 +43,7 @@ static inline bool context_tracking_in_user(void)#else static inline bool context_tracking_in_user (void ) {return false; }static inline bool context_tracking_enabled (void ) {return false; } -static inline bool context_tracking_cpu_is_enabled (void ) {return false; } +static inline bool context_tracking_enabled_this_cpu (void ) {return false; }#endif /* CONFIG_CONTEXT_TRACKING */ #endif
include/linux/vtime.h +1 -1
@@ -34,7 +34,7 @@ static inline bool vtime_accounting_enabled(void)static inline bool vtime_accounting_cpu_enabled (void ) {if (vtime_accounting_enabled ()) { -if (context_tracking_cpu_is_enabled ()) +if (context_tracking_enabled_this_cpu ())return true; }
[tip: sched/core] context_tracking: Check static key on context_tracking_enabled_*cpu()
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 023e9deb51c9e1aafacbd421e55beadcb8e87f53 Gitweb: https://git.kernel.org/tip/023e9deb51c9e1aafacbd421e55beadcb8e87f53 Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:56 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:16 +01:00
context_tracking: Check static key on context_tracking_enabled_*cpu()
guest_enter() doesn't call context_tracking_enabled() before calling context_tracking_enabled_this_cpu(). Therefore the guest code doesn't benefit from the static key on the fast path.
Just make sure that context_tracking_enabled_*cpu() functions check the static key by themselves to propagate the optimization.
--- include/linux/context_tracking_state.h | 4 ++-- include/linux/vtime.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/context_tracking_state.h +2 -2
@@ -33,12 +33,12 @@ static inline bool context_tracking_enabled(void)static inline bool context_tracking_enabled_cpu (int cpu) { -return per_cpu (context_tracking.active, cpu); +return context_tracking_enabled () &&per_cpu (context_tracking.active, cpu); }static inline bool context_tracking_enabled_this_cpu (void ) { -return __this_cpu_read (context_tracking.active); +return context_tracking_enabled () &&__this_cpu_read (context_tracking.active); }static inline bool context_tracking_in_user (void )
include/linux/vtime.h +2 -2
@@ -33,12 +33,12 @@ static inline bool vtime_accounting_enabled(void)static inline bool vtime_accounting_enabled_cpu (int cpu) { -return (vtime_accounting_enabled () &&context_tracking_enabled_cpu (cpu)); +return context_tracking_enabled_cpu (cpu); }static inline bool vtime_accounting_enabled_this_cpu (void ) { -return (vtime_accounting_enabled () &&context_tracking_enabled_this_cpu ()); +return context_tracking_enabled_this_cpu (); }extern void vtime_task_switch_generic (struct task_struct *prev);
[tip: sched/core] cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
In reply to Frederic WeisbeckerThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 49bb001e246d3a44692f8418c8e38d9ef4aa405f Gitweb: https://git.kernel.org/tip/49bb001e246d3a44692f8418c8e38d9ef4aa405f Author: Frederic Weisbecker <[email protected]> AuthorDate: Wed, 16 Oct 2019 04:56:59 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 29 Oct 2019 10:01:18 +01:00
cpufreq: Use vtime aware kcpustat accessor to fetch CPUTIME_SYSTEM
Now that we have a vtime safe kcpustat accessor for CPUTIME_SYSTEM, use it to start fixing frozen kcpustat values on nohz_full CPUs.
--- drivers/cpufreq/cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/cpufreq/cpufreq.c +1 -1
@@ -120,7 +120,7 @@ static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) cur_wall_time =jiffies64_to_nsecs (get_jiffies_64 ()); busy_time =kcpustat_cpu (cpu).cpustat[CPUTIME_USER ]; - busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SYSTEM ]; + busy_time +=kcpustat_field (&kcpustat_cpu (cpu),CPUTIME_SYSTEM , cpu); busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_IRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_SOFTIRQ ]; busy_time +=kcpustat_cpu (cpu).cpustat[CPUTIME_STEAL ];