vtime: Remove pair of seqcount on context switch
Subsystems (4)
Frederic Weisbecker
maintainercover vtime: Remove pair of seqcount on context switch
Extracted from a larger queue that fixes kcpustat on nohz_full, these two patches have value on their own as they remove two write barriers on nohz_full context switch.
Frederic Weisbecker (2):
vtime: Rename vtime_account_system() to vtime_account_kernel()
vtime: Spare a seqcount lock/unlock cycle on context switch
arch/ia64/kernel/time.c | 4 +-- arch/powerpc/kernel/time.c | 6 ++-- arch/s390/kernel/vtime.c | 4 +-- include/linux/context_tracking.h | 4 +-- include/linux/vtime.h | 38 ++++++++++++------------ kernel/sched/cputime.c | 50 ++++++++++++++++++-------------- 6 files changed, 57 insertions(+), 49 deletions(-)
Signature · 1 line
2.23.0
Frederic Weisbecker
maintainerPATCH 1/2 vtime: Rename vtime_account_system() to vtime_account_kernel()
vtime_account_system() decides if we need to account the time to the system (__vtime_account_system()) or to the guest (vtime_account_guest()).
So this function is a misnommer as we are on a higher level than "system". All we know when we call that function is that we are accounting kernel cputime. Whether it belongs to guest or system time is a lower level detail.
Rename this function to vtime_account_kernel(). This will clarify things and avoid too many underscored vtime_account_system() versions.
--- arch/ia64/kernel/time.c | 4 ++-- arch/powerpc/kernel/time.c | 6 +++--- arch/s390/kernel/vtime.c | 4 ++-- include/linux/context_tracking.h | 4 ++-- include/linux/vtime.h | 6 +++--- kernel/sched/cputime.c | 18 +++++++++--------- 6 files changed, 21 insertions(+), 21 deletions(-)
arch/ia64/kernel/time.c +2 -2
@@ -132,7 +132,7 @@ static __u64 vtime_delta(struct task_struct *tsk)return delta_stime; } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {struct thread_info *ti =task_thread_info (tsk);__u64 stime =vtime_delta (tsk);
@@ -146,7 +146,7 @@ void vtime_account_system(struct task_struct *tsk)else ti->stime += stime; } -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);void vtime_account_idle (struct task_struct *tsk) {
arch/powerpc/kernel/time.c +3 -3
@@ -338,7 +338,7 @@ static unsigned long vtime_delta(struct task_struct *tsk,return stime; } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {unsigned long stime, stime_scaled, steal_time;struct cpu_accounting_data *acct =get_accounting (tsk);
@@ -366,7 +366,7 @@ void vtime_account_system(struct task_struct *tsk)#endif } } -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);void vtime_account_idle (struct task_struct *tsk) {
@@ -395,7 +395,7 @@ static void vtime_flush_scaled(struct task_struct *tsk,/* * Account the whole cputime accumulated in the paca * Must be called with interrupts disabled. -* Assumes that vtime_account_system/idle() has been called +* Assumes that vtime_account_kernel/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */
arch/s390/kernel/vtime.c +2 -2
@@ -247,9 +247,9 @@ void vtime_account_irq_enter(struct task_struct *tsk) }EXPORT_SYMBOL_GPL (vtime_account_irq_enter); -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) __attribute__((alias ("vtime_account_irq_enter" ))); -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);/* * Sorted add to a list. List is linear searched until first bigger
include/linux/context_tracking.h +2 -2
@@ -141,7 +141,7 @@ static inline void guest_enter_irqoff(void)* to assume that it's the stime pending cputime * to flush. */ -vtime_account_system (current); +vtime_account_kernel (current); current->flags |=PF_VCPU ;rcu_virt_note_context_switch (smp_processor_id ()); }
@@ -149,7 +149,7 @@ static inline void guest_enter_irqoff(void)static inline void guest_exit_irqoff (void ) {/* Flush the guest cputime we spent on the guest */ -vtime_account_system (current); +vtime_account_kernel (current); current->flags &= ~PF_VCPU ; }#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
include/linux/vtime.h +3 -3
@@ -57,13 +57,13 @@ static inline void vtime_task_switch(struct task_struct *prev) }#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ -extern void vtime_account_system (struct task_struct *tsk); +extern void vtime_account_kernel (struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk);#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ static inline void vtime_task_switch (struct task_struct *prev) { } -static inline void vtime_account_system (struct task_struct *tsk) { } +static inline void vtime_account_kernel (struct task_struct *tsk) { }#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
@@ -86,7 +86,7 @@ extern void vtime_account_irq_enter(struct task_struct *tsk);static inline void vtime_account_irq_exit (struct task_struct *tsk) {/* On hard|softirq exit we always account to hard|softirq cputime */ -vtime_account_system (tsk); +vtime_account_kernel (tsk); }extern void vtime_flush (struct task_struct *tsk);#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
kernel/sched/cputime.c +9 -9
@@ -412,7 +412,7 @@ void vtime_common_task_switch(struct task_struct *prev)if (is_idle_task (prev))vtime_account_idle (prev);else -vtime_account_system (prev); +vtime_account_kernel (prev);vtime_flush (prev);arch_vtime_task_switch (prev);
@@ -425,7 +425,7 @@ void vtime_common_task_switch(struct task_struct *prev)/* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement -* vtime_account_system() and vtime_account_idle(). Archs that +* vtime_account_kernel() and vtime_account_idle(). Archs that * have other meaning of the idle time (s390 only includes the * time spent by the CPU when it's in low power mode) must override * vtime_account().
@@ -436,7 +436,7 @@ void vtime_account_irq_enter(struct task_struct *tsk)if (!in_interrupt () &&is_idle_task (tsk))vtime_account_idle (tsk);else - vtime_account_system(tsk); + vtime_account_kernel(tsk); }EXPORT_SYMBOL_GPL (vtime_account_irq_enter);#endif /* __ARCH_HAS_VTIME_ACCOUNT */
@@ -711,8 +711,8 @@ static u64 get_vtime_delta(struct vtime *vtime)return delta - other; } -static void __vtime_account_system (struct task_struct *tsk, -struct vtime *vtime) +static void vtime_account_system (struct task_struct *tsk, +struct vtime *vtime) { vtime->stime +=get_vtime_delta (vtime);if (vtime->stime >=TICK_NSEC ) {
@@ -731,7 +731,7 @@ static void vtime_account_guest(struct task_struct *tsk, } } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {struct vtime *vtime = &tsk->vtime;
@@ -743,7 +743,7 @@ void vtime_account_system(struct task_struct *tsk)if (tsk->flags &PF_VCPU )vtime_account_guest (tsk, vtime);else -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime);write_seqcount_end (&vtime->seqcount); }
@@ -752,7 +752,7 @@ void vtime_user_enter(struct task_struct *tsk)struct vtime *vtime = &tsk->vtime;write_seqcount_begin (&vtime->seqcount); -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime); vtime->state =VTIME_USER ;write_seqcount_end (&vtime->seqcount); }
@@ -782,7 +782,7 @@ void vtime_guest_enter(struct task_struct *tsk)* that can thus safely catch up with a tickless delta. */ write_seqcount_begin (&vtime->seqcount); -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime); tsk->flags |=PF_VCPU ;write_seqcount_end (&vtime->seqcount); }
Signature · 1 line
2.23.0
[tip: sched/core] sched/cputime: Rename vtime_account_system() to vtime_account_kernel()
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: f83eeb1a01689b2691f6f56629ac9f66de8d41c2 Gitweb: https://git.kernel.org/tip/f83eeb1a01689b2691f6f56629ac9f66de8d41c2 Author: Frederic Weisbecker <[email protected]> AuthorDate: Thu, 03 Oct 2019 18:17:44 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Wed, 09 Oct 2019 12:39:25 +02:00
sched/cputime: Rename vtime_account_system() to vtime_account_kernel()
vtime_account_system() decides if we need to account the time to the system (__vtime_account_system()) or to the guest (vtime_account_guest()).
So this function is a misnomer as we are on a higher level than "system". All we know when we call that function is that we are accounting kernel cputime. Whether it belongs to guest or system time is a lower level detail.
Rename this function to vtime_account_kernel(). This will clarify things and avoid too many underscored vtime_account_system() versions.
--- arch/ia64/kernel/time.c | 4 ++-- arch/powerpc/kernel/time.c | 6 +++--- arch/s390/kernel/vtime.c | 4 ++-- include/linux/context_tracking.h | 4 ++-- include/linux/vtime.h | 6 +++--- kernel/sched/cputime.c | 18 +++++++++--------- 6 files changed, 21 insertions(+), 21 deletions(-)
arch/ia64/kernel/time.c +2 -2
@@ -132,7 +132,7 @@ static __u64 vtime_delta(struct task_struct *tsk)return delta_stime; } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {struct thread_info *ti =task_thread_info (tsk);__u64 stime =vtime_delta (tsk);
@@ -146,7 +146,7 @@ void vtime_account_system(struct task_struct *tsk)else ti->stime += stime; } -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);void vtime_account_idle (struct task_struct *tsk) {
arch/powerpc/kernel/time.c +3 -3
@@ -338,7 +338,7 @@ static unsigned long vtime_delta(struct task_struct *tsk,return stime; } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {unsigned long stime, stime_scaled, steal_time;struct cpu_accounting_data *acct =get_accounting (tsk);
@@ -366,7 +366,7 @@ void vtime_account_system(struct task_struct *tsk)#endif } } -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);void vtime_account_idle (struct task_struct *tsk) {
@@ -395,7 +395,7 @@ static void vtime_flush_scaled(struct task_struct *tsk,/* * Account the whole cputime accumulated in the paca * Must be called with interrupts disabled. -* Assumes that vtime_account_system/idle() has been called +* Assumes that vtime_account_kernel/idle() has been called * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */
arch/s390/kernel/vtime.c +2 -2
@@ -247,9 +247,9 @@ void vtime_account_irq_enter(struct task_struct *tsk) }EXPORT_SYMBOL_GPL (vtime_account_irq_enter); -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) __attribute__((alias ("vtime_account_irq_enter" ))); -EXPORT_SYMBOL_GPL (vtime_account_system); +EXPORT_SYMBOL_GPL (vtime_account_kernel);/* * Sorted add to a list. List is linear searched until first bigger
include/linux/context_tracking.h +2 -2
@@ -141,7 +141,7 @@ static inline void guest_enter_irqoff(void)* to assume that it's the stime pending cputime * to flush. */ -vtime_account_system (current); +vtime_account_kernel (current); current->flags |=PF_VCPU ;rcu_virt_note_context_switch (smp_processor_id ()); }
@@ -149,7 +149,7 @@ static inline void guest_enter_irqoff(void)static inline void guest_exit_irqoff (void ) {/* Flush the guest cputime we spent on the guest */ -vtime_account_system (current); +vtime_account_kernel (current); current->flags &= ~PF_VCPU ; }#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
include/linux/vtime.h +3 -3
@@ -57,13 +57,13 @@ static inline void vtime_task_switch(struct task_struct *prev) }#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ -extern void vtime_account_system (struct task_struct *tsk); +extern void vtime_account_kernel (struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk);#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ static inline void vtime_task_switch (struct task_struct *prev) { } -static inline void vtime_account_system (struct task_struct *tsk) { } +static inline void vtime_account_kernel (struct task_struct *tsk) { }#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
@@ -86,7 +86,7 @@ extern void vtime_account_irq_enter(struct task_struct *tsk);static inline void vtime_account_irq_exit (struct task_struct *tsk) {/* On hard|softirq exit we always account to hard|softirq cputime */ -vtime_account_system (tsk); +vtime_account_kernel (tsk); }extern void vtime_flush (struct task_struct *tsk);#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
kernel/sched/cputime.c +9 -9
@@ -412,7 +412,7 @@ void vtime_common_task_switch(struct task_struct *prev)if (is_idle_task (prev))vtime_account_idle (prev);else -vtime_account_system (prev); +vtime_account_kernel (prev);vtime_flush (prev);arch_vtime_task_switch (prev);
@@ -425,7 +425,7 @@ void vtime_common_task_switch(struct task_struct *prev)/* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement -* vtime_account_system() and vtime_account_idle(). Archs that +* vtime_account_kernel() and vtime_account_idle(). Archs that * have other meaning of the idle time (s390 only includes the * time spent by the CPU when it's in low power mode) must override * vtime_account().
@@ -436,7 +436,7 @@ void vtime_account_irq_enter(struct task_struct *tsk)if (!in_interrupt () &&is_idle_task (tsk))vtime_account_idle (tsk);else - vtime_account_system(tsk); + vtime_account_kernel(tsk); }EXPORT_SYMBOL_GPL (vtime_account_irq_enter);#endif /* __ARCH_HAS_VTIME_ACCOUNT */
@@ -711,8 +711,8 @@ static u64 get_vtime_delta(struct vtime *vtime)return delta - other; } -static void __vtime_account_system (struct task_struct *tsk, -struct vtime *vtime) +static void vtime_account_system (struct task_struct *tsk, +struct vtime *vtime) { vtime->stime +=get_vtime_delta (vtime);if (vtime->stime >=TICK_NSEC ) {
@@ -731,7 +731,7 @@ static void vtime_account_guest(struct task_struct *tsk, } } -void vtime_account_system (struct task_struct *tsk) +void vtime_account_kernel (struct task_struct *tsk) {struct vtime *vtime = &tsk->vtime;
@@ -743,7 +743,7 @@ void vtime_account_system(struct task_struct *tsk)if (tsk->flags &PF_VCPU )vtime_account_guest (tsk, vtime);else -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime);write_seqcount_end (&vtime->seqcount); }
@@ -752,7 +752,7 @@ void vtime_user_enter(struct task_struct *tsk)struct vtime *vtime = &tsk->vtime;write_seqcount_begin (&vtime->seqcount); -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime); vtime->state =VTIME_USER ;write_seqcount_end (&vtime->seqcount); }
@@ -782,7 +782,7 @@ void vtime_guest_enter(struct task_struct *tsk)* that can thus safely catch up with a tickless delta. */ write_seqcount_begin (&vtime->seqcount); -__vtime_account_system (tsk, vtime); +vtime_account_system (tsk, vtime); tsk->flags |=PF_VCPU ;write_seqcount_end (&vtime->seqcount); }
Frederic Weisbecker
maintainerPATCH 2/2 vtime: Spare a seqcount lock/unlock cycle on context switch
In reply to Frederic WeisbeckerOn context switch we are locking the vtime seqcount of the scheduling-out task twice:
* On vtime_task_switch_common(), when we flush the pending vtime through vtime_account_system()
* On arch_vtime_task_switch() to reset the vtime state.
This is pointless as these actions can be performed without the need to unlock/lock in the middle. The reason these steps are separated is to consolidate a very small amount of common code between CONFIG_VIRT_CPU_ACCOUNTING_GEN and CONFIG_VIRT_CPU_ACCOUNTING_NATIVE.
Performance in this fast path is definetly a priority over artificial code factorization so split the task switch code between GEN and NATIVE and mutualize the parts than can run under a single seqcount locked block.
As a side effect, vtime_account_idle() becomes included in the seqcount protection. This happens to be a welcome preparation in order to properly support kcpustat under vtime in the future and fetch CPUTIME_IDLE without race.
--- include/linux/vtime.h | 32 ++++++++++++++++---------------- kernel/sched/cputime.c | 34 +++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 29 deletions(-)
include/linux/vtime.h +16 -16
@@ -14,8 +14,12 @@ struct task_struct;* vtime_accounting_cpu_enabled() definitions/declarations */ #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) +static inline bool vtime_accounting_cpu_enabled (void ) {return true; } +extern void vtime_task_switch (struct task_struct *prev); +#elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN ) +/* * Checks if vtime is enabled on some CPU. Cputime readers want to be careful * in that case and compute the tickless cputime.
@@ -36,33 +40,29 @@ static inline bool vtime_accounting_cpu_enabled(void) return false; } + +extern void vtime_task_switch_generic (struct task_struct *prev); + +static inline void vtime_task_switch (struct task_struct *prev) +{ +if (vtime_accounting_cpu_enabled ()) +vtime_task_switch_generic (prev); +} +#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ +static inline bool vtime_accounting_cpu_enabled (void ) {return false; } -#endif +static inline void vtime_task_switch (struct task_struct *prev) { } +#endif /* * Common vtime APIs */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING - -#ifdef __ARCH_HAS_VTIME_TASK_SWITCH -extern void vtime_task_switch (struct task_struct *prev); -#else -extern void vtime_common_task_switch (struct task_struct *prev); -static inline void vtime_task_switch (struct task_struct *prev) -{ -if (vtime_accounting_cpu_enabled ()) -vtime_common_task_switch (prev); -} -#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ -extern void vtime_account_kernel (struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk); -#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ - -static inline void vtime_task_switch (struct task_struct *prev) { }static inline void vtime_account_kernel (struct task_struct *tsk) { }#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
kernel/sched/cputime.c +21 -13
@@ -405,9 +405,10 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_/* * Use precise platform statistics if available: */ -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE + # ifndef __ARCH_HAS_VTIME_TASK_SWITCH -void vtime_common_task_switch(struct task_struct *prev) +void vtime_task_switch(struct task_struct *prev) { if (is_idle_task(prev)) vtime_account_idle(prev);
@@ -418,10 +419,7 @@ void vtime_common_task_switch(struct task_struct *prev) arch_vtime_task_switch(prev); }# endif -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ - -#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement
@@ -731,19 +729,25 @@ static void vtime_account_guest(struct task_struct *tsk, } } -void vtime_account_kernel (struct task_struct *tsk) +static void __vtime_account_kernel (struct task_struct *tsk, +struct vtime *vtime) { -struct vtime *vtime = &tsk->vtime; - -if (!vtime_delta (vtime)) -return ; - -write_seqcount_begin (&vtime->seqcount);/* We might have scheduled out from guest path */ if (tsk->flags &PF_VCPU )vtime_account_guest (tsk, vtime);else vtime_account_system (tsk, vtime); +} + +void vtime_account_kernel (struct task_struct *tsk) +{ +struct vtime *vtime = &tsk->vtime; + +if (!vtime_delta (vtime)) +return ; + +write_seqcount_begin (&vtime->seqcount); +__vtime_account_kernel (tsk, vtime);write_seqcount_end (&vtime->seqcount); }
@@ -804,11 +808,15 @@ void vtime_account_idle(struct task_struct *tsk)account_idle_time (get_vtime_delta (&tsk->vtime)); } -void arch_vtime_task_switch (struct task_struct *prev) +void vtime_task_switch_generic (struct task_struct *prev) {struct vtime *vtime = &prev->vtime;write_seqcount_begin (&vtime->seqcount); +if (is_idle_task (prev)) +vtime_account_idle (prev); +else +__vtime_account_kernel (prev, vtime); vtime->state =VTIME_INACTIVE ;write_seqcount_end (&vtime->seqcount);
Signature · 1 line
2.23.0
[tip: sched/core] sched/cputime: Spare a seqcount lock/unlock cycle on context switch
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 8d495477d62e4397207f22a432fcaa86d9f2bc2d Gitweb: https://git.kernel.org/tip/8d495477d62e4397207f22a432fcaa86d9f2bc2d Author: Frederic Weisbecker <[email protected]> AuthorDate: Thu, 03 Oct 2019 18:17:45 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Wed, 09 Oct 2019 12:39:26 +02:00
sched/cputime: Spare a seqcount lock/unlock cycle on context switch
On context switch we are locking the vtime seqcount of the scheduling-out task twice:
* On vtime_task_switch_common(), when we flush the pending vtime through vtime_account_system()
* On arch_vtime_task_switch() to reset the vtime state.
This is pointless as these actions can be performed without the need to unlock/lock in the middle. The reason these steps are separated is to consolidate a very small amount of common code between CONFIG_VIRT_CPU_ACCOUNTING_GEN and CONFIG_VIRT_CPU_ACCOUNTING_NATIVE.
Performance in this fast path is definitely a priority over artificial code factorization so split the task switch code between GEN and NATIVE and mutualize the parts than can run under a single seqcount locked block.
As a side effect, vtime_account_idle() becomes included in the seqcount protection. This happens to be a welcome preparation in order to properly support kcpustat under vtime in the future and fetch CPUTIME_IDLE without race.
--- include/linux/vtime.h | 32 ++++++++++++++++---------------- kernel/sched/cputime.c | 30 +++++++++++++++++++----------- 2 files changed, 35 insertions(+), 27 deletions(-)
include/linux/vtime.h +16 -16
@@ -14,8 +14,12 @@ struct task_struct;* vtime_accounting_cpu_enabled() definitions/declarations */ #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) +static inline bool vtime_accounting_cpu_enabled (void ) {return true; } +extern void vtime_task_switch (struct task_struct *prev); +#elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN ) +/* * Checks if vtime is enabled on some CPU. Cputime readers want to be careful * in that case and compute the tickless cputime.
@@ -36,33 +40,29 @@ static inline bool vtime_accounting_cpu_enabled(void) return false; } + +extern void vtime_task_switch_generic (struct task_struct *prev); + +static inline void vtime_task_switch (struct task_struct *prev) +{ +if (vtime_accounting_cpu_enabled ()) +vtime_task_switch_generic (prev); +} +#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ +static inline bool vtime_accounting_cpu_enabled (void ) {return false; } -#endif +static inline void vtime_task_switch (struct task_struct *prev) { } +#endif /* * Common vtime APIs */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING - -#ifdef __ARCH_HAS_VTIME_TASK_SWITCH -extern void vtime_task_switch (struct task_struct *prev); -#else -extern void vtime_common_task_switch (struct task_struct *prev); -static inline void vtime_task_switch (struct task_struct *prev) -{ -if (vtime_accounting_cpu_enabled ()) -vtime_common_task_switch (prev); -} -#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ -extern void vtime_account_kernel (struct task_struct *tsk);extern void vtime_account_idle (struct task_struct *tsk); -#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ - -static inline void vtime_task_switch (struct task_struct *prev) { }static inline void vtime_account_kernel (struct task_struct *tsk) { }#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
kernel/sched/cputime.c +19 -11
@@ -405,9 +405,10 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_/* * Use precise platform statistics if available: */ -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE + # ifndef __ARCH_HAS_VTIME_TASK_SWITCH -void vtime_common_task_switch(struct task_struct *prev) +void vtime_task_switch(struct task_struct *prev) { if (is_idle_task(prev)) vtime_account_idle(prev);
@@ -418,10 +419,7 @@ void vtime_common_task_switch(struct task_struct *prev) arch_vtime_task_switch(prev); }# endif -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ - -#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement
@@ -731,6 +729,16 @@ static void vtime_account_guest(struct task_struct *tsk, } } +static void __vtime_account_kernel (struct task_struct *tsk, +struct vtime *vtime) +{ +/* We might have scheduled out from guest path */ +if (tsk->flags &PF_VCPU ) +vtime_account_guest (tsk, vtime); +else +vtime_account_system (tsk, vtime); +} +void vtime_account_kernel (struct task_struct *tsk) {struct vtime *vtime = &tsk->vtime;
@@ -739,11 +747,7 @@ void vtime_account_kernel(struct task_struct *tsk)return ;write_seqcount_begin (&vtime->seqcount); -/* We might have scheduled out from guest path */ -if (tsk->flags &PF_VCPU ) -vtime_account_guest (tsk, vtime); -else -vtime_account_system (tsk, vtime); +__vtime_account_kernel (tsk, vtime);write_seqcount_end (&vtime->seqcount); }
@@ -804,11 +808,15 @@ void vtime_account_idle(struct task_struct *tsk)account_idle_time (get_vtime_delta (&tsk->vtime)); } -void arch_vtime_task_switch (struct task_struct *prev) +void vtime_task_switch_generic (struct task_struct *prev) {struct vtime *vtime = &prev->vtime;write_seqcount_begin (&vtime->seqcount); +if (is_idle_task (prev)) +vtime_account_idle (prev); +else +__vtime_account_kernel (prev, vtime); vtime->state =VTIME_INACTIVE ;write_seqcount_end (&vtime->seqcount);
Quoted 8 lines · Frederic Weisbecker
Extracted from a larger queue that fixes kcpustat on
nohz_full, these two patches have value on their own as they remove two write barriers onnohz_fullcontext switch.Frederic Weisbecker (2):
vtime: Renamevtime_account_system()tovtime_account_kernel()
vtime: Spare a seqcount lock/unlock cycle on context switch
Quoted 7 lines
arch/ia64/kernel/time.c | 4 +-- arch/powerpc/kernel/time.c | 6 ++-- arch/s390/kernel/vtime.c | 4 +-- include/linux/context_tracking.h | 4 +-- include/linux/vtime.h | 38 ++++++++++++------------ kernel/sched/cputime.c | 50 ++++++++++++++++++-------------- 6 files changed, 57 insertions(+), 49 deletions(-)
Which tree is this against? Doesn't apply cleanly to v5.4-rc2 nor v5.3. Does it have any prereqs perhaps?
Thanks,
Ingo
Frederic Weisbecker
maintainerQuoted 19 lines · Ingo Molnar
Quoted 8 lines · Frederic Weisbecker
Extracted from a larger queue that fixes kcpustat on
nohz_full, these two patches have value on their own as they remove two write barriers onnohz_fullcontext switch.Frederic Weisbecker (2):
vtime: Renamevtime_account_system()tovtime_account_kernel()
vtime: Spare a seqcount lock/unlock cycle on context switchQuoted 7 lines
arch/ia64/kernel/time.c | 4 +-- arch/powerpc/kernel/time.c | 6 ++-- arch/s390/kernel/vtime.c | 4 +-- include/linux/context_tracking.h | 4 +-- include/linux/vtime.h | 38 ++++++++++++------------ kernel/sched/cputime.c | 50 ++++++++++++++++++-------------- 6 files changed, 57 insertions(+), 49 deletions(-)
Quoted 2 lines
Which tree is this against? Doesn't apply cleanly to v5.4-rc2 nor v5.3. Does it have any prereqs perhaps?
Indeed, you need to apply this fix first: https://lore.kernel.org/lkml/[email protected]/
"[PATCH] sched/vtime: Fix guest/system mis-accounting on task switch"
Thanks!