sched/cputime: correct account of irqtime
PATCH 1/1 sched/cputime: correct account of irqtime
__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.
By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.
--- include/linux/hardirq.h | 4 ++-- include/linux/vtime.h | 12 ++++++------ kernel/sched/cputime.c | 4 ++-- kernel/softirq.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-)
include/linux/hardirq.h +2 -2
@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)*/ #define __irq_enter () \ do { \ - account_irq_enter_time(current); \ + account_irq_enter_time(current, true); \ preempt_count_add(HARDIRQ_OFFSET); \ lockdep_hardirq_enter(); \ } while (0)
@@ -63,7 +63,7 @@ void irq_enter_rcu(void);#define __irq_exit () \ do { \ lockdep_hardirq_exit(); \ - account_irq_exit_time(current); \ + account_irq_exit_time(current, true); \ preempt_count_sub(HARDIRQ_OFFSET); \ } while (0)
include/linux/vtime.h +6 -6
@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }#ifdef CONFIG_IRQ_TIME_ACCOUNTING -extern void irqtime_account_irq (struct task_struct *tsk); +extern void irqtime_account_irq (struct task_struct *tsk,bool hardirq);#else -static inline void irqtime_account_irq (struct task_struct *tsk) { } +static inline void irqtime_account_irq (struct task_struct *tsk,bool hardirq) { }#endif -static inline void account_irq_enter_time (struct task_struct *tsk) +static inline void account_irq_enter_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_enter (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); } -static inline void account_irq_exit_time (struct task_struct *tsk) +static inline void account_irq_exit_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_exit (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); }#endif /* _LINUX_KERNEL_VTIME_H */
kernel/sched/cputime.c +2 -2
@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,* Called before incrementing preempt_count on {soft,}irq_enter * and before decrementing preempt_count on {soft,}irq_exit. */ -void irqtime_account_irq (struct task_struct *curr) +void irqtime_account_irq (struct task_struct *curr,bool hardirq) {struct irqtime *irqtime =this_cpu_ptr (&cpu_irqtime);s64 delta;
@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)*/ if (hardirq_count ())irqtime_account_delta (irqtime, delta,CPUTIME_IRQ ); -else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd ()) +else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd () && !hardirq)irqtime_account_delta (irqtime, delta,CPUTIME_SOFTIRQ ); }EXPORT_SYMBOL_GPL (irqtime_account_irq);
kernel/softirq.c +3 -3
@@ -270,7 +270,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) current->flags &= ~PF_MEMALLOC ; pending =local_softirq_pending (); -account_irq_enter_time (current); +account_irq_enter_time (current, false);__local_bh_disable_ip (_RET_IP_,SOFTIRQ_OFFSET ); in_hardirq =lockdep_softirq_start ();
@@ -321,7 +321,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) }lockdep_softirq_end (in_hardirq); -account_irq_exit_time (current); +account_irq_exit_time (current, false);__local_bh_enable (SOFTIRQ_OFFSET );WARN_ON_ONCE (in_interrupt ());current_restore_flags (old_flags,PF_MEMALLOC );
@@ -417,7 +417,7 @@ static inline void __irq_exit_rcu(void)#else lockdep_assert_irqs_disabled ();#endif -account_irq_exit_time (current); +account_irq_exit_time (current, true);preempt_count_sub (HARDIRQ_OFFSET );if (!in_interrupt () &&local_softirq_pending ())invoke_softirq ();
Signature · 1 line
2.7.5
Quoted 100 lines · Pingfan Liu <[email protected]> 于2020年10月12日周一 下午9:54写道:
__do_softirq()may be interrupted by hardware interrupts. In this case,irqtime_account_irq()will account the time slice asCPUTIME_SOFTIRQby mistake.By passing
irqtime_account_irq()an extra param about either hardirq or softirq,irqtime_account_irq()can handle the above case.Signed-off-by Pingfan Liu <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Juri Lelli <[email protected]>Cc Vincent Guittot <[email protected]>Cc Dietmar Eggemann <[email protected]>Cc Steven Rostedt <[email protected]>Cc Ben Segall <[email protected]>Cc Mel Gorman <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Andy Lutomirski <[email protected]>Cc Will Deacon <[email protected]>Cc "Paul E. McKenney" <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Allen Pais <[email protected]>Cc Romain Perier <[email protected]>--- include/linux/hardirq.h | 4 ++-- include/linux/vtime.h | 12 ++++++------ kernel/sched/cputime.c | 4 ++-- kernel/softirq.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-)include/linux/hardirq.h +2
-2@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)*/ #define __irq_enter () \ do { \ - account_irq_enter_time(current); \ + account_irq_enter_time(current, true); \ preempt_count_add(HARDIRQ_OFFSET); \ lockdep_hardirq_enter(); \ } while (0)@@ -63,7 +63,7 @@ void irq_enter_rcu(void);#define __irq_exit () \ do { \ lockdep_hardirq_exit(); \ - account_irq_exit_time(current); \ + account_irq_exit_time(current, true); \ preempt_count_sub(HARDIRQ_OFFSET); \ } while (0)include/linux/vtime.h +6
-6@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }#ifdef CONFIG_IRQ_TIME_ACCOUNTING -extern void irqtime_account_irq (struct task_struct *tsk); +extern void irqtime_account_irq (struct task_struct *tsk,bool hardirq);#else -static inline void irqtime_account_irq (struct task_struct *tsk) { } +static inline void irqtime_account_irq (struct task_struct *tsk,bool hardirq) { }#endif -static inline void account_irq_enter_time (struct task_struct *tsk) +static inline void account_irq_enter_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_enter (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); } -static inline void account_irq_exit_time (struct task_struct *tsk) +static inline void account_irq_exit_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_exit (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); }#endif /* _LINUX_KERNEL_VTIME_H */ kernel/sched/cputime.c +1
-1@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,* Called before incrementing preempt_count on {soft,}irq_enter * and before decrementing preempt_count on {soft,}irq_exit. */ -void irqtime_account_irq (struct task_struct *curr) +void irqtime_account_irq (struct task_struct *curr,bool hardirq) {struct irqtime *irqtime =this_cpu_ptr (&cpu_irqtime);s64 delta;
kernel/sched/cputime.c void irqtime_account_irq(struct task_struct *curr) · line 73
Quoted 7 lines
kernel/sched/cputime.c +1
-1@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)*/ if (hardirq_count ())irqtime_account_delta (irqtime, delta,CPUTIME_IRQ ); -else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd ()) +else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd () && !hardirq)irqtime_account_delta (irqtime, delta,CPUTIME_SOFTIRQ ); }
In my opinion, we don't need to use the hardirq flag, the code: if (hardirq_count())
already tell us that where the delt time is from.
Thanks
Quoted 35 lines
kernel/sched/cputime.c +0
-0EXPORT_SYMBOL_GPL (irqtime_account_irq);kernel/softirq.c +3
-4@@ -270,7 +270,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) current->flags &= ~PF_MEMALLOC ; pending =local_softirq_pending (); -account_irq_enter_time (current); +account_irq_enter_time (current, false);__local_bh_disable_ip (_RET_IP_,SOFTIRQ_OFFSET ); in_hardirq =lockdep_softirq_start ();@@ -321,7 +321,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) }lockdep_softirq_end (in_hardirq); -account_irq_exit_time (current); +account_irq_exit_time (current, false);__local_bh_enable (SOFTIRQ_OFFSET );WARN_ON_ONCE (in_interrupt ());current_restore_flags (old_flags,PF_MEMALLOC );@@ -417,7 +417,7 @@ static inline void __irq_exit_rcu(void)#else lockdep_assert_irqs_disabled ();#endif -account_irq_exit_time (current); +account_irq_exit_time (current, true);preempt_count_sub (HARDIRQ_OFFSET );if (!in_interrupt () &&local_softirq_pending ())invoke_softirq (); --2.7.5
Quoted 105 lines · jun qian
Quoted 103 lines · Pingfan Liu <[email protected]> 于2020年10月12日周一 下午9:54写道:
__do_softirq()may be interrupted by hardware interrupts. In this case,irqtime_account_irq()will account the time slice asCPUTIME_SOFTIRQby mistake.By passing
irqtime_account_irq()an extra param about either hardirq or softirq,irqtime_account_irq()can handle the above case.Signed-off-by Pingfan Liu <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Juri Lelli <[email protected]>Cc Vincent Guittot <[email protected]>Cc Dietmar Eggemann <[email protected]>Cc Steven Rostedt <[email protected]>Cc Ben Segall <[email protected]>Cc Mel Gorman <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Andy Lutomirski <[email protected]>Cc Will Deacon <[email protected]>Cc "Paul E. McKenney" <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Allen Pais <[email protected]>Cc Romain Perier <[email protected]>--- include/linux/hardirq.h | 4 ++-- include/linux/vtime.h | 12 ++++++------ kernel/sched/cputime.c | 4 ++-- kernel/softirq.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-)include/linux/hardirq.h +2
-2@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)*/ #define __irq_enter () \ do { \ - account_irq_enter_time(current); \ + account_irq_enter_time(current, true); \ preempt_count_add(HARDIRQ_OFFSET); \ lockdep_hardirq_enter(); \ } while (0)@@ -63,7 +63,7 @@ void irq_enter_rcu(void);#define __irq_exit () \ do { \ lockdep_hardirq_exit(); \ - account_irq_exit_time(current); \ + account_irq_exit_time(current, true); \ preempt_count_sub(HARDIRQ_OFFSET); \ } while (0)include/linux/vtime.h +6
-6@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }#ifdef CONFIG_IRQ_TIME_ACCOUNTING -extern void irqtime_account_irq (struct task_struct *tsk); +extern void irqtime_account_irq (struct task_struct *tsk,bool hardirq);#else -static inline void irqtime_account_irq (struct task_struct *tsk) { } +static inline void irqtime_account_irq (struct task_struct *tsk,bool hardirq) { }#endif -static inline void account_irq_enter_time (struct task_struct *tsk) +static inline void account_irq_enter_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_enter (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); } -static inline void account_irq_exit_time (struct task_struct *tsk) +static inline void account_irq_exit_time (struct task_struct *tsk,bool hardirq) {vtime_account_irq_exit (tsk); -irqtime_account_irq (tsk); +irqtime_account_irq (tsk, hardirq); }#endif /* _LINUX_KERNEL_VTIME_H */ kernel/sched/cputime.c +1
-1@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,* Called before incrementing preempt_count on {soft,}irq_enter * and before decrementing preempt_count on {soft,}irq_exit. */ -void irqtime_account_irq (struct task_struct *curr) +void irqtime_account_irq (struct task_struct *curr,bool hardirq) {struct irqtime *irqtime =this_cpu_ptr (&cpu_irqtime);s64 delta;@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)*/ if (hardirq_count ())irqtime_account_delta (irqtime, delta,CPUTIME_IRQ );
Quoted 8 lines
kernel/sched/cputime.c void irqtime_account_irq(struct task_struct *curr) · line 73
Quoted 4 lines
kernel/sched/cputime.c +1
-1-else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd ()) +else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd () && !hardirq)irqtime_account_delta (irqtime, delta,CPUTIME_SOFTIRQ ); }In my opinion, we don't need to use the hardirq flag, the code: if (
hardirq_count())
already tell us that where the delt time is from.
Considering the scenario in which hardirq happens immediately after __do_softirq()->local_irq_enable(). The following code shows that hardirq_count() can not help.
#define __irq_enter() \
do { \account_irq_enter_time(current); \preempt_count_add(HARDIRQ_OFFSET); \lockdep_hardirq_enter(); \
} while (0)
Anything I missed?
Thanks,
Pingfan
Peter Zijlstra
maintainerQuoted 6 lines · Pingfan Liu
__do_softirq()may be interrupted by hardware interrupts. In this case,irqtime_account_irq()will account the time slice asCPUTIME_SOFTIRQby mistake.By passing
irqtime_account_irq()an extra param about either hardirq or softirq,irqtime_account_irq()can handle the above case.
I'm not sure I see the scenario in which it goes wrong.
irqtime_account_irq() is designed such that we're called with the old preempt_count on enter and the new preempt_count on exit. This way we'll accumuate the delta to the previous context.
Quoted 8 lines
kernel/sched/cputime.c +1
-1@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)*/ if (hardirq_count ())irqtime_account_delta (irqtime, delta,CPUTIME_IRQ ); -else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd ()) +else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd () && !hardirq)irqtime_account_delta (irqtime, delta,CPUTIME_SOFTIRQ ); }
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · Pingfan Liu
__do_softirq()may be interrupted by hardware interrupts. In this case,irqtime_account_irq()will account the time slice asCPUTIME_SOFTIRQby mistake.
Quoted 9 lines
Quoted 2 lines
By passing
irqtime_account_irq()an extra param about either hardirq or softirq,irqtime_account_irq()can handle the above case.I'm not sure I see the scenario in which it goes wrong.
irqtime_account_irq()is designed such that we're called with the oldpreempt_counton enter and the newpreempt_counton exit. This way we'll accumuate the delta to the previous context.
Oops! You are right, the time delta between a softirq and a interrupting hardirq should be accounted into the softrq.
Thanks for your clear explanation.
Regards,
Pingfan