Skip to contentSkip to conversation

sched/cputime: correct account of irqtime

PATCH In review Pingfan Liu 3 participants 5 messages Oct 12, 2020 – Oct 15, 2020 Scheduler

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 sched/cputime: correct account of irqtime

__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.

By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Juri Lelli <[email protected]>
Cc Vincent Guittot <[email protected]>
Cc Dietmar Eggemann <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ben Segall <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Will Deacon <[email protected]>
Cc "Paul E. McKenney" <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Allen Pais <[email protected]>
Cc Romain Perier <[email protected]>

To: [email protected]

---
 include/linux/hardirq.h |  4 ++--
 include/linux/vtime.h   | 12 ++++++------
 kernel/sched/cputime.c  |  4 ++--
 kernel/softirq.c        |  6 +++---
 4 files changed, 13 insertions(+), 13 deletions(-)
include/linux/hardirq.h +2 -2
index 754f67a..56e7bb5 100644
@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)
  */
 #define __irq_enter()					\
 	do {						\
-		account_irq_enter_time(current);	\
+		account_irq_enter_time(current, true);	\
 		preempt_count_add(HARDIRQ_OFFSET);	\
 		lockdep_hardirq_enter();		\
 	} while (0)
@@ -63,7 +63,7 @@ void irq_enter_rcu(void);
 #define __irq_exit()					\
 	do {						\
 		lockdep_hardirq_exit();			\
-		account_irq_exit_time(current);		\
+		account_irq_exit_time(current, true);	\
 		preempt_count_sub(HARDIRQ_OFFSET);	\
 	} while (0)
 
include/linux/vtime.h +6 -6
index 2cdeca0..294188ae1 100644
@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }
 
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
-extern void irqtime_account_irq(struct task_struct *tsk);
+extern void irqtime_account_irq(struct task_struct *tsk, bool hardirq);
 #else
-static inline void irqtime_account_irq(struct task_struct *tsk) { }
+static inline void irqtime_account_irq(struct task_struct *tsk, bool hardirq) { }
 #endif
 
-static inline void account_irq_enter_time(struct task_struct *tsk)
+static inline void account_irq_enter_time(struct task_struct *tsk, bool hardirq)
 {
 	vtime_account_irq_enter(tsk);
-	irqtime_account_irq(tsk);
+	irqtime_account_irq(tsk, hardirq);
 }
 
-static inline void account_irq_exit_time(struct task_struct *tsk)
+static inline void account_irq_exit_time(struct task_struct *tsk, bool hardirq)
 {
 	vtime_account_irq_exit(tsk);
-	irqtime_account_irq(tsk);
+	irqtime_account_irq(tsk, hardirq);
 }
 
 #endif /* _LINUX_KERNEL_VTIME_H */
kernel/sched/cputime.c +2 -2
index 5a55d23..166f1d7 100644
@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
-void irqtime_account_irq(struct task_struct *curr)
+void irqtime_account_irq(struct task_struct *curr, bool hardirq)
 {
 	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
 	s64 delta;
@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)
 	 */
 	if (hardirq_count())
 		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
-	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
+	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd() && !hardirq)
 		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
kernel/softirq.c +3 -3
index bf88d7f6..da59ea39 100644
@@ -270,7 +270,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
 	current->flags &= ~PF_MEMALLOC;
 
 	pending = local_softirq_pending();
-	account_irq_enter_time(current);
+	account_irq_enter_time(current, false);
 
 	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 	in_hardirq = lockdep_softirq_start();
@@ -321,7 +321,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
 	}
 
 	lockdep_softirq_end(in_hardirq);
-	account_irq_exit_time(current);
+	account_irq_exit_time(current, false);
 	__local_bh_enable(SOFTIRQ_OFFSET);
 	WARN_ON_ONCE(in_interrupt());
 	current_restore_flags(old_flags, PF_MEMALLOC);
@@ -417,7 +417,7 @@ static inline void __irq_exit_rcu(void)
 #else
 	lockdep_assert_irqs_disabled();
 #endif
-	account_irq_exit_time(current);
+	account_irq_exit_time(current, true);
 	preempt_count_sub(HARDIRQ_OFFSET);
 	if (!in_interrupt() && local_softirq_pending())
 		invoke_softirq();
Signature · 1 line
2.7.5
#1 lore raw
Quoted 100 lines · Pingfan Liu <[email protected]> 于2020年10月12日周一 下午9:54写道:

__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.

By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Juri Lelli <[email protected]>
Cc Vincent Guittot <[email protected]>
Cc Dietmar Eggemann <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ben Segall <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Will Deacon <[email protected]>
Cc "Paul E. McKenney" <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Allen Pais <[email protected]>
Cc Romain Perier <[email protected]>

To: [email protected]

---
 include/linux/hardirq.h |  4 ++--
 include/linux/vtime.h   | 12 ++++++------
 kernel/sched/cputime.c  |  4 ++--
 kernel/softirq.c        |  6 +++---
 4 files changed, 13 insertions(+), 13 deletions(-)
include/linux/hardirq.h +2 -2
index 754f67a..56e7bb5 100644
@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)
  */
 #define __irq_enter()                                  \
        do {                                            \
-               account_irq_enter_time(current);        \
+               account_irq_enter_time(current, true);  \
                preempt_count_add(HARDIRQ_OFFSET);      \
                lockdep_hardirq_enter();                \
        } while (0)
@@ -63,7 +63,7 @@ void irq_enter_rcu(void);
 #define __irq_exit()                                   \
        do {                                            \
                lockdep_hardirq_exit();                 \
-               account_irq_exit_time(current);         \
+               account_irq_exit_time(current, true);   \
                preempt_count_sub(HARDIRQ_OFFSET);      \
        } while (0)
 
include/linux/vtime.h +6 -6
index 2cdeca0..294188ae1 100644
@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }
 
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
-extern void irqtime_account_irq(struct task_struct *tsk);
+extern void irqtime_account_irq(struct task_struct *tsk, bool hardirq);
 #else
-static inline void irqtime_account_irq(struct task_struct *tsk) { }
+static inline void irqtime_account_irq(struct task_struct *tsk, bool hardirq) { }
 #endif
 
-static inline void account_irq_enter_time(struct task_struct *tsk)
+static inline void account_irq_enter_time(struct task_struct *tsk, bool hardirq)
 {
        vtime_account_irq_enter(tsk);
-       irqtime_account_irq(tsk);
+       irqtime_account_irq(tsk, hardirq);
 }
 
-static inline void account_irq_exit_time(struct task_struct *tsk)
+static inline void account_irq_exit_time(struct task_struct *tsk, bool hardirq)
 {
        vtime_account_irq_exit(tsk);
-       irqtime_account_irq(tsk);
+       irqtime_account_irq(tsk, hardirq);
 }
 
 #endif /* _LINUX_KERNEL_VTIME_H */
kernel/sched/cputime.c +1 -1
index 5a55d23..166f1d7 100644
@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
-void irqtime_account_irq(struct task_struct *curr)
+void irqtime_account_irq(struct task_struct *curr, bool hardirq)
 {
        struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
        s64 delta;

kernel/sched/cputime.c void irqtime_account_irq(struct task_struct *curr) · line 73

Quoted 7 lines
kernel/sched/cputime.c +1 -1
@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)
         */
        if (hardirq_count())
                irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
-       else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
+       else if (in_serving_softirq() && curr != this_cpu_ksoftirqd() && !hardirq)
                irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
 }

In my opinion, we don't need to use the hardirq flag, the code: if (hardirq_count())
already tell us that where the delt time is from.

Thanks

Quoted 35 lines
kernel/sched/cputime.c +0 -0
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
kernel/softirq.c +3 -4
index bf88d7f6..da59ea39 100644
@@ -270,7 +270,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
        current->flags &= ~PF_MEMALLOC;
 
        pending = local_softirq_pending();
-       account_irq_enter_time(current);
+       account_irq_enter_time(current, false);
 
        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
        in_hardirq = lockdep_softirq_start();
@@ -321,7 +321,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
        }
 
        lockdep_softirq_end(in_hardirq);
-       account_irq_exit_time(current);
+       account_irq_exit_time(current, false);
        __local_bh_enable(SOFTIRQ_OFFSET);
        WARN_ON_ONCE(in_interrupt());
        current_restore_flags(old_flags, PF_MEMALLOC);
@@ -417,7 +417,7 @@ static inline void __irq_exit_rcu(void)
 #else
        lockdep_assert_irqs_disabled();
 #endif
-       account_irq_exit_time(current);
+       account_irq_exit_time(current, true);
        preempt_count_sub(HARDIRQ_OFFSET);
        if (!in_interrupt() && local_softirq_pending())
                invoke_softirq();
--

2.7.5

#2 lore raw
Quoted 105 lines · jun qian
Quoted 103 lines · Pingfan Liu <[email protected]> 于2020年10月12日周一 下午9:54写道:

__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.

By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Juri Lelli <[email protected]>
Cc Vincent Guittot <[email protected]>
Cc Dietmar Eggemann <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ben Segall <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Will Deacon <[email protected]>
Cc "Paul E. McKenney" <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Allen Pais <[email protected]>
Cc Romain Perier <[email protected]>

To: [email protected]

---
 include/linux/hardirq.h |  4 ++--
 include/linux/vtime.h   | 12 ++++++------
 kernel/sched/cputime.c  |  4 ++--
 kernel/softirq.c        |  6 +++---
 4 files changed, 13 insertions(+), 13 deletions(-)
include/linux/hardirq.h +2 -2
index 754f67a..56e7bb5 100644
@@ -32,7 +32,7 @@ static __always_inline void rcu_irq_enter_check_tick(void)
  */
 #define __irq_enter()                                  \
        do {                                            \
-               account_irq_enter_time(current);        \
+               account_irq_enter_time(current, true);  \
                preempt_count_add(HARDIRQ_OFFSET);      \
                lockdep_hardirq_enter();                \
        } while (0)
@@ -63,7 +63,7 @@ void irq_enter_rcu(void);
 #define __irq_exit()                                   \
        do {                                            \
                lockdep_hardirq_exit();                 \
-               account_irq_exit_time(current);         \
+               account_irq_exit_time(current, true);   \
                preempt_count_sub(HARDIRQ_OFFSET);      \
        } while (0)
 
include/linux/vtime.h +6 -6
index 2cdeca0..294188ae1 100644
@@ -98,21 +98,21 @@ static inline void vtime_flush(struct task_struct *tsk) { }
 
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
-extern void irqtime_account_irq(struct task_struct *tsk);
+extern void irqtime_account_irq(struct task_struct *tsk, bool hardirq);
 #else
-static inline void irqtime_account_irq(struct task_struct *tsk) { }
+static inline void irqtime_account_irq(struct task_struct *tsk, bool hardirq) { }
 #endif
 
-static inline void account_irq_enter_time(struct task_struct *tsk)
+static inline void account_irq_enter_time(struct task_struct *tsk, bool hardirq)
 {
        vtime_account_irq_enter(tsk);
-       irqtime_account_irq(tsk);
+       irqtime_account_irq(tsk, hardirq);
 }
 
-static inline void account_irq_exit_time(struct task_struct *tsk)
+static inline void account_irq_exit_time(struct task_struct *tsk, bool hardirq)
 {
        vtime_account_irq_exit(tsk);
-       irqtime_account_irq(tsk);
+       irqtime_account_irq(tsk, hardirq);
 }
 
 #endif /* _LINUX_KERNEL_VTIME_H */
kernel/sched/cputime.c +1 -1
index 5a55d23..166f1d7 100644
@@ -47,7 +47,7 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
-void irqtime_account_irq(struct task_struct *curr)
+void irqtime_account_irq(struct task_struct *curr, bool hardirq)
 {
        struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
        s64 delta;
@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)
         */
        if (hardirq_count())
                irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
Quoted 8 lines

kernel/sched/cputime.c void irqtime_account_irq(struct task_struct *curr) · line 73

Quoted 4 lines
kernel/sched/cputime.c +1 -1
-       else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
+       else if (in_serving_softirq() && curr != this_cpu_ksoftirqd() && !hardirq)
                irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
 }

In my opinion, we don't need to use the hardirq flag, the code: if (hardirq_count())
already tell us that where the delt time is from.

Considering the scenario in which hardirq happens immediately after __do_softirq()->local_irq_enable(). The following code shows that hardirq_count() can not help.
#define __irq_enter() \
do { \
account_irq_enter_time(current); \
preempt_count_add(HARDIRQ_OFFSET); \
lockdep_hardirq_enter(); \
} while (0)

Anything I missed?

Thanks,
Pingfan

maintainer
#3 lore raw
In reply to Pingfan Liu
Quoted 6 lines · Pingfan Liu

__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.

By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.

I'm not sure I see the scenario in which it goes wrong.

irqtime_account_irq() is designed such that we're called with the old preempt_count on enter and the new preempt_count on exit. This way we'll accumuate the delta to the previous context.

Quoted 8 lines
kernel/sched/cputime.c +1 -1
@@ -68,7 +68,7 @@ void irqtime_account_irq(struct task_struct *curr)
 	 */
 	if (hardirq_count())
 		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
-	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
+	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd() && !hardirq)
 		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
 }
#4 lore raw
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · Pingfan Liu

__do_softirq() may be interrupted by hardware interrupts. In this case, irqtime_account_irq() will account the time slice as CPUTIME_SOFTIRQ by mistake.

Quoted 9 lines
Quoted 2 lines

By passing irqtime_account_irq() an extra param about either hardirq or softirq, irqtime_account_irq() can handle the above case.

I'm not sure I see the scenario in which it goes wrong.

irqtime_account_irq() is designed such that we're called with the old preempt_count on enter and the new preempt_count on exit. This way we'll accumuate the delta to the previous context.

Oops! You are right, the time delta between a softirq and a interrupting hardirq should be accounted into the softrq.

Thanks for your clear explanation.

Regards,
Pingfan