sched/core: fix inaccurate idle stat in /proc/stat when force idle
PATCH 1/1 sched/core: fix inaccurate idle stat in /proc/stat when force idle
When force idle, rq->curr == rq->idle but rq->nr_running > 0, results that idle_cpu() returns 0. In function tick_irq_exit(), if idle_cpu() is 0, tick_nohz_irq_exit() will not be called, and ts->idle_active will not become 1, which became 0 in tick_nohz_irq_enter(). ts->idle_sleeptime won't update in function update_ts_time_stats(), if ts->idle_active is 0, which should be 1. And this bug will result that ts->idle_sleeptime is less than the actual value, and finally will result that the idle time in /proc/stat is less than the actual value.
To fix this problem, we introduce sched_core_idle_cpu(), which returns true when force idle, and we change the judgment criteria from idle_cpu() to sched_core_idle_cpu() in function tick_irq_exit().
--- include/linux/sched.h | 2 ++ kernel/sched/core.c | 13 +++++++++++++ kernel/softirq.c | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-)
include/linux/sched.h +2 -0
@@ -2430,9 +2430,11 @@ extern void sched_core_free(struct task_struct *tsk);extern void sched_core_fork (struct task_struct *p);extern int sched_core_share_pid (unsigned int cmd,pid_t pid,enum pid_type type,unsigned long uaddr); +extern int sched_core_idle_cpu (int cpu);#else static inline void sched_core_free (struct task_struct *tsk) { }static inline void sched_core_fork (struct task_struct *p) { } +static inline int sched_core_idle_cpu (int cpu) {return idle_cpu (cpu); }#endif extern void sched_set_stop_task (int cpu,struct task_struct *stop);
kernel/sched/core.c +13 -0
@@ -7286,6 +7286,19 @@ struct task_struct *idle_task(int cpu)return cpu_rq (cpu)->idle; } +#ifdef CONFIG_SCHED_CORE +int sched_core_idle_cpu (int cpu) +{ +struct rq *rq =cpu_rq (cpu); + +if (sched_core_enabled (rq) && rq->curr == rq->idle) +return 1 ; + +return idle_cpu (cpu); +} + +#endif +#ifdef CONFIG_SMP /* * This function computes an effective utilization for the given CPU, to be
kernel/softirq.c +1 -1
@@ -630,7 +630,7 @@ static inline void tick_irq_exit(void)int cpu =smp_processor_id ();/* Make sure that timer wheel updates are propagated */ -if ((idle_cpu (cpu) && !need_resched ()) ||tick_nohz_full_cpu (cpu)) { +if ((sched_core_idle_cpu (cpu) && !need_resched ()) ||tick_nohz_full_cpu (cpu)) {if (!in_hardirq ())tick_nohz_irq_exit (); }
Signature · 1 line
2.27.0
Peter Zijlstra
maintainerQuoted 65 lines · Cruz Zhao
When force idle,
rq->curr==rq->idlebutrq->nr_running> 0, results thatidle_cpu()returns 0. In functiontick_irq_exit(), ifidle_cpu()is 0,tick_nohz_irq_exit()will not be called, andts->idle_activewill not become 1, which became 0 intick_nohz_irq_enter().ts->idle_sleeptimewon't update in functionupdate_ts_time_stats(), ifts->idle_activeis 0, which should be 1. And this bug will result thatts->idle_sleeptimeis less than the actual value, and finally will result that the idle time in/proc/statis less than the actual value.To fix this problem, we introduce
sched_core_idle_cpu(), which returns true when force idle, and we change the judgment criteria fromidle_cpu()tosched_core_idle_cpu()in functiontick_irq_exit().Signed-off-by Cruz Zhao <[email protected]>--- include/linux/sched.h | 2 ++ kernel/sched/core.c | 13 +++++++++++++ kernel/softirq.c | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-)include/linux/sched.h +2
-0@@ -2430,9 +2430,11 @@ extern void sched_core_free(struct task_struct *tsk);extern void sched_core_fork (struct task_struct *p);extern int sched_core_share_pid (unsigned int cmd,pid_t pid,enum pid_type type,unsigned long uaddr); +extern int sched_core_idle_cpu (int cpu);#else static inline void sched_core_free (struct task_struct *tsk) { }static inline void sched_core_fork (struct task_struct *p) { } +static inline int sched_core_idle_cpu (int cpu) {return idle_cpu (cpu); }#endif extern void sched_set_stop_task (int cpu,struct task_struct *stop);kernel/sched/core.c +13
-0@@ -7286,6 +7286,19 @@ struct task_struct *idle_task(int cpu)return cpu_rq (cpu)->idle; } +#ifdef CONFIG_SCHED_CORE +int sched_core_idle_cpu (int cpu) +{ +struct rq *rq =cpu_rq (cpu); + +if (sched_core_enabled (rq) && rq->curr == rq->idle) +return 1 ; + +return idle_cpu (cpu); +} + +#endif +#ifdef CONFIG_SMP /* * This function computes an effective utilization for the given CPU, to be kernel/softirq.c +0
-0
kernel/softirq.c static inline void tick_irq_exit(void) · line 636
Quoted 8 lines
kernel/softirq.c +1
-1@@ -630,7 +630,7 @@ static inline void tick_irq_exit(void)int cpu =smp_processor_id ();/* Make sure that timer wheel updates are propagated */ -if ((idle_cpu (cpu) && !need_resched ()) ||tick_nohz_full_cpu (cpu)) { +if ((sched_core_idle_cpu (cpu) && !need_resched ()) ||tick_nohz_full_cpu (cpu)) {if (!in_hardirq ())tick_nohz_irq_exit (); }
Urgh...
did you audit all idle_cpu() users?
Or so we want something like so instead?
---
kernel/sched/core.c +5 -0
@@ -7303,6 +7303,11 @@ int idle_cpu(int cpu)if (rq->curr != rq->idle)return 0 ; +#ifdef CONFIG_SCHED_CORE +if (sched_core_enabled () && rq->core->core_forceidle_count) +return 1 ; +#endif +if (rq->nr_running)return 0 ;