gFrom 9667d5ddbb1dc230653e5f8cedb778e9c562d46c Mon Sep 17 00:00:00 2001
Instead of periodically resetting watchdogs from thread context, this patch simply forces resched and checks rq->sched_count. Watchdog is reset if the sched count increases. If the same thread is picked by pick_next_task during resched, there is no context switch.
With the new method we lose coverage on: a migration/n thread actually gets picked and we actually context switch to the migration/n thread. These steps are unlikely to silently fail. The change would provide nearly the same level of protection with less latency / jitter.
v3:
- Removed the old method and boot option
- Still need to check resched
v2:
- Use sched_count instead of having sched calling into watchdog code
- Remove the sysctl and add a boot option, which can be removed later
- Changed the subject line
Xi Wang (1):
sched: watchdog: Touch kernel watchdog with sched count
include/linux/sched.h | 4 ++++ kernel/sched/core.c | 23 +++++++++++++++++++-- kernel/sched/sched.h | 6 +++++- kernel/watchdog.c | 47 +++++++++++++------------------------------ 4 files changed, 44 insertions(+), 36 deletions(-)
Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
PATCH v3 1/1 sched: watchdog: Touch kernel watchdog with sched count
Instead of periodically resetting watchdogs from thread context, this patch simply forces resched and checks rq->sched_count. Watchdog is reset if the sched count increases. If the same thread is picked by pick_next_task during resched, there is no context switch.
With the new method we lose coverage on: a migration/n thread actually gets picked and we actually context switch to the migration/n thread. These steps are unlikely to silently fail. The change would provide nearly the same level of protection with less latency / jitter.
--- include/linux/sched.h | 4 ++++ kernel/sched/core.c | 23 +++++++++++++++++++-- kernel/sched/sched.h | 6 +++++- kernel/watchdog.c | 47 +++++++++++++------------------------------ 4 files changed, 44 insertions(+), 36 deletions(-)
include/linux/sched.h +4 -0
@@ -1662,6 +1662,10 @@ extern int sched_setattr(struct task_struct *, const struct sched_attr *);extern int sched_setattr_nocheck (struct task_struct *,const struct sched_attr *);extern struct task_struct *idle_task (int cpu); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +extern unsigned int sched_get_count (int cpu); +#endif +/** * is_idle_task - is the specified task an idle task? * @p: the task in question.
kernel/sched/core.c +21 -2
@@ -4293,8 +4293,6 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)rcu_sleep_check ();profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); - -schedstat_inc (this_rq ()->sched_count); } static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
@@ -4492,6 +4490,12 @@ static void __sched notrace __schedule(bool preempt)clear_tsk_need_resched (prev);clear_preempt_need_resched (); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +this_rq ()->sched_count++;/* sched count is also used by watchdog */ +#else +schedstat_inc (this_rq ()->sched_count); +#endif +if (likely (prev != next)) { rq->nr_switches++;/*
@@ -5117,6 +5121,21 @@ struct task_struct *idle_task(int cpu)return cpu_rq (cpu)->idle; } +#ifdef CONFIG_SOFTLOCKUP_DETECTOR + +/** +* sched_get_count - get the sched count of a CPU. +* @cpu: the CPU in question. +* +* Return: sched count. +*/ +unsigned int sched_get_count (int cpu) +{ +return cpu_rq (cpu)->sched_count; +} + +#endif +/** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question.
kernel/sched/sched.h +5 -1
@@ -959,6 +959,11 @@ struct rq {u64 clock_pelt;unsigned long lost_idle_time; +#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_SOFTLOCKUP_DETECTOR ) +/* Also used by watchdog - no longer grouping with other sched stats */ +unsigned int sched_count; +#endif +atomic_t nr_iowait;#ifdef CONFIG_MEMBARRIER
@@ -1036,7 +1041,6 @@ struct rq {unsigned int yld_count;/* schedule() stats */ -unsigned int sched_count;unsigned int sched_goidle;/* try_to_wake_up() stats */
kernel/watchdog.c +14 -33
92 lines not on this page · raw message
Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
Vincent Guittot
maintainerRe: [PATCH v3 1/1] sched: watchdog: Touch kernel watchdog with sched count
Quoted 7 lines · Xi Wang
Instead of periodically resetting watchdogs from thread context, this patch simply forces resched and checks
rq->sched_count. Watchdog is reset if the sched count increases. If the same thread is picked bypick_next_taskduring resched, there is no context switch.
Quoted 5 lines
With the new method we lose coverage on: a migration/n thread actually gets picked and we actually context switch to the migration/n thread. These steps are unlikely to silently fail. The change would provide nearly the same level of protection with less latency / jitter.
When a patch provides an improvement, it's usually good to give figures that show the improvement
Quoted 204 lines
Suggested-by Paul Turner <[email protected]>Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Xi Wang <[email protected]>--- include/linux/sched.h | 4 ++++ kernel/sched/core.c | 23 +++++++++++++++++++-- kernel/sched/sched.h | 6 +++++- kernel/watchdog.c | 47 +++++++++++++------------------------------ 4 files changed, 44 insertions(+), 36 deletions(-)include/linux/sched.h +4
-0@@ -1662,6 +1662,10 @@ extern int sched_setattr(struct task_struct *, const struct sched_attr *);extern int sched_setattr_nocheck (struct task_struct *,const struct sched_attr *);extern struct task_struct *idle_task (int cpu); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +extern unsigned int sched_get_count (int cpu); +#endif +/** * is_idle_task - is the specified task an idle task? * @p: the task in question. kernel/sched/core.c +21
-2@@ -4293,8 +4293,6 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)rcu_sleep_check ();profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); - -schedstat_inc (this_rq ()->sched_count); } static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,@@ -4492,6 +4490,12 @@ static void __sched notrace __schedule(bool preempt)clear_tsk_need_resched (prev);clear_preempt_need_resched (); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +this_rq ()->sched_count++;/* sched count is also used by watchdog */ +#else +schedstat_inc (this_rq ()->sched_count); +#endif +if (likely (prev != next)) { rq->nr_switches++;/* @@ -5117,6 +5121,21 @@ struct task_struct *idle_task(int cpu)return cpu_rq (cpu)->idle; } +#ifdef CONFIG_SOFTLOCKUP_DETECTOR + +/** +* sched_get_count - get the sched count of a CPU. +* @cpu: the CPU in question. +* +* Return: sched count. +*/ +unsigned int sched_get_count (int cpu) +{ +return cpu_rq (cpu)->sched_count; +} + +#endif +/** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. kernel/sched/sched.h +5
-1@@ -959,6 +959,11 @@ struct rq {u64 clock_pelt;unsigned long lost_idle_time; +#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_SOFTLOCKUP_DETECTOR ) +/* Also used by watchdog - no longer grouping with other sched stats */ +unsigned int sched_count; +#endif +atomic_t nr_iowait;#ifdef CONFIG_MEMBARRIER @@ -1036,7 +1041,6 @@ struct rq {unsigned int yld_count;/* schedule() stats */ -unsigned int sched_count;unsigned int sched_goidle;/* try_to_wake_up() stats */ kernel/watchdog.c +14
-34@@ -170,6 +170,7 @@ static bool softlockup_initialized __read_mostly;static u64 __read_mostly sample_period; static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); +static DEFINE_PER_CPU(unsigned int, watchdog_sched_prev); static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); static DEFINE_PER_CPU(bool, softlockup_touch_sync); static DEFINE_PER_CPU(bool, soft_watchdog_warn);@@ -239,6 +240,7 @@ static void set_sample_period(void)static void __touch_watchdog (void ) {__this_cpu_write (watchdog_touch_ts,get_timestamp ()); +__this_cpu_write (watchdog_sched_prev,sched_get_count (smp_processor_id ())); }/** @@ -318,25 +320,6 @@ static void watchdog_interrupt_count(void)__this_cpu_inc (hrtimer_interrupts); } -static DEFINE_PER_CPU(struct completion, softlockup_completion); -static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); - -/* -* The watchdog thread function - touches the timestamp. -* -* It only runs once every sample_period seconds (4 seconds by -* default) to reset the softlockup timestamp. If this gets delayed -* for more than 2*watchdog_thresh seconds then the debug-printout -* triggers in watchdog_timer_fn(). -*/ -static int softlockup_fn (void *data) -{ -__touch_watchdog (); -complete (this_cpu_ptr (&softlockup_completion)); - -return 0 ; -} -/* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn (struct hrtimer *hrtimer) {@@ -351,15 +334,6 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)/* kick the hardlockup detector */ watchdog_interrupt_count (); -/* kick the softlockup detector */ -if (completion_done (this_cpu_ptr (&softlockup_completion))) { -reinit_completion (this_cpu_ptr (&softlockup_completion)); -stop_one_cpu_nowait (smp_processor_id (), - softlockup_fn,NULL , -this_cpu_ptr (&softlockup_stop_work)); - } - -/* .. and repeat */ hrtimer_forward_now (hrtimer,ns_to_ktime (sample_period));if (touch_ts ==SOFTLOCKUP_RESET ) {@@ -378,6 +352,18 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)return HRTIMER_RESTART ; } +/* Trigger reschedule for the next round */ +set_tsk_need_resched (current); +set_preempt_need_resched (); + +/* sched_count increase in __schedule is taken as watchdog touched */ +if (sched_get_count (smp_processor_id ()) - +__this_cpu_read (watchdog_sched_prev)) { +__touch_watchdog (); +__this_cpu_write (soft_watchdog_warn, false); +return HRTIMER_RESTART ; + } +/* check for a softlockup * This is done by making sure a high priority task is * being scheduled. The task touches the watchdog to @@ -443,13 +429,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)static void watchdog_enable (unsigned int cpu) {struct hrtimer *hrtimer =this_cpu_ptr (&watchdog_hrtimer); -struct completion *done =this_cpu_ptr (&softlockup_completion);WARN_ON_ONCE (cpu !=smp_processor_id ()); -init_completion (done); -complete (done); -/* * Start the timer first to prevent the NMI watchdog triggering * before the timer has a chance to fire. @@ -479,7 +461,6 @@ static void watchdog_disable(unsigned int cpu)*/ watchdog_nmi_disable (cpu);hrtimer_cancel (hrtimer); -wait_for_completion (this_cpu_ptr (&softlockup_completion)); }static int softlockup_stop_fn (void *data) --2.29.0.rc2.309.g374f81d7ae-goog
Re: [PATCH v3 1/1] sched: watchdog: Touch kernel watchdog with sched count
Quoted 9 lines · Vincent Guittot
Quoted 7 lines · Xi Wang
Instead of periodically resetting watchdogs from thread context, this patch simply forces resched and checks
rq->sched_count. Watchdog is reset if the sched count increases. If the same thread is picked bypick_next_taskduring resched, there is no context switch.
Quoted 8 lines
Quoted 5 lines
With the new method we lose coverage on: a migration/n thread actually gets picked and we actually context switch to the migration/n thread. These steps are unlikely to silently fail. The change would provide nearly the same level of protection with less latency / jitter.
When a patch provides an improvement, it's usually good to give figures that show the improvement
This change would reduce jitters for a continuously running thread. The difference is likely too small to tell for sched latency benchmarks.
will-it-scale mmap1 reported 15.8% improvement for the v1 patch:
https://lkml.org/lkml/2020/3/10/129
The performance change is actually unexpected. If it's not noise there might be contentions related to watchdog thread wake up or context switch.
-Xi
Quoted 205 lines
Quoted 205 lines
Suggested-by Paul Turner <[email protected]>Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Xi Wang <[email protected]>--- include/linux/sched.h | 4 ++++ kernel/sched/core.c | 23 +++++++++++++++++++-- kernel/sched/sched.h | 6 +++++- kernel/watchdog.c | 47 +++++++++++++------------------------------ 4 files changed, 44 insertions(+), 36 deletions(-)include/linux/sched.h +4
-0@@ -1662,6 +1662,10 @@ extern int sched_setattr(struct task_struct *, const struct sched_attr *);extern int sched_setattr_nocheck (struct task_struct *,const struct sched_attr *);extern struct task_struct *idle_task (int cpu); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +extern unsigned int sched_get_count (int cpu); +#endif +/** * is_idle_task - is the specified task an idle task? * @p: the task in question. kernel/sched/core.c +21
-2@@ -4293,8 +4293,6 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)rcu_sleep_check ();profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); - -schedstat_inc (this_rq ()->sched_count); } static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,@@ -4492,6 +4490,12 @@ static void __sched notrace __schedule(bool preempt)clear_tsk_need_resched (prev);clear_preempt_need_resched (); +#ifdef CONFIG_SOFTLOCKUP_DETECTOR +this_rq ()->sched_count++;/* sched count is also used by watchdog */ +#else +schedstat_inc (this_rq ()->sched_count); +#endif +if (likely (prev != next)) { rq->nr_switches++;/* @@ -5117,6 +5121,21 @@ struct task_struct *idle_task(int cpu)return cpu_rq (cpu)->idle; } +#ifdef CONFIG_SOFTLOCKUP_DETECTOR + +/** +* sched_get_count - get the sched count of a CPU. +* @cpu: the CPU in question. +* +* Return: sched count. +*/ +unsigned int sched_get_count (int cpu) +{ +return cpu_rq (cpu)->sched_count; +} + +#endif +/** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. kernel/sched/sched.h +5
-1@@ -959,6 +959,11 @@ struct rq {u64 clock_pelt;unsigned long lost_idle_time; +#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_SOFTLOCKUP_DETECTOR ) +/* Also used by watchdog - no longer grouping with other sched stats */ +unsigned int sched_count; +#endif +atomic_t nr_iowait;#ifdef CONFIG_MEMBARRIER @@ -1036,7 +1041,6 @@ struct rq {unsigned int yld_count;/* schedule() stats */ -unsigned int sched_count;unsigned int sched_goidle;/* try_to_wake_up() stats */ kernel/watchdog.c +14
-34@@ -170,6 +170,7 @@ static bool softlockup_initialized __read_mostly;static u64 __read_mostly sample_period; static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); +static DEFINE_PER_CPU(unsigned int, watchdog_sched_prev); static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); static DEFINE_PER_CPU(bool, softlockup_touch_sync); static DEFINE_PER_CPU(bool, soft_watchdog_warn);@@ -239,6 +240,7 @@ static void set_sample_period(void)static void __touch_watchdog (void ) {__this_cpu_write (watchdog_touch_ts,get_timestamp ()); +__this_cpu_write (watchdog_sched_prev,sched_get_count (smp_processor_id ())); }/** @@ -318,25 +320,6 @@ static void watchdog_interrupt_count(void)__this_cpu_inc (hrtimer_interrupts); } -static DEFINE_PER_CPU(struct completion, softlockup_completion); -static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); - -/* -* The watchdog thread function - touches the timestamp. -* -* It only runs once every sample_period seconds (4 seconds by -* default) to reset the softlockup timestamp. If this gets delayed -* for more than 2*watchdog_thresh seconds then the debug-printout -* triggers in watchdog_timer_fn(). -*/ -static int softlockup_fn (void *data) -{ -__touch_watchdog (); -complete (this_cpu_ptr (&softlockup_completion)); - -return 0 ; -} -/* watchdog kicker functions */ static enum hrtimer_restart watchdog_timer_fn (struct hrtimer *hrtimer) {@@ -351,15 +334,6 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)/* kick the hardlockup detector */ watchdog_interrupt_count (); -/* kick the softlockup detector */ -if (completion_done (this_cpu_ptr (&softlockup_completion))) { -reinit_completion (this_cpu_ptr (&softlockup_completion)); -stop_one_cpu_nowait (smp_processor_id (), - softlockup_fn,NULL , -this_cpu_ptr (&softlockup_stop_work)); - } - -/* .. and repeat */ hrtimer_forward_now (hrtimer,ns_to_ktime (sample_period));if (touch_ts ==SOFTLOCKUP_RESET ) {@@ -378,6 +352,18 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)return HRTIMER_RESTART ; } +/* Trigger reschedule for the next round */ +set_tsk_need_resched (current); +set_preempt_need_resched (); + +/* sched_count increase in __schedule is taken as watchdog touched */ +if (sched_get_count (smp_processor_id ()) - +__this_cpu_read (watchdog_sched_prev)) { +__touch_watchdog (); +__this_cpu_write (soft_watchdog_warn, false); +return HRTIMER_RESTART ; + } +/* check for a softlockup * This is done by making sure a high priority task is * being scheduled. The task touches the watchdog to @@ -443,13 +429,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)static void watchdog_enable (unsigned int cpu) {struct hrtimer *hrtimer =this_cpu_ptr (&watchdog_hrtimer); -struct completion *done =this_cpu_ptr (&softlockup_completion);WARN_ON_ONCE (cpu !=smp_processor_id ()); -init_completion (done); -complete (done); -/* * Start the timer first to prevent the NMI watchdog triggering * before the timer has a chance to fire. @@ -479,7 +461,6 @@ static void watchdog_disable(unsigned int cpu)*/ watchdog_nmi_disable (cpu);hrtimer_cancel (hrtimer); -wait_for_completion (this_cpu_ptr (&softlockup_completion)); }static int softlockup_stop_fn (void *data) --2.29.0.rc2.309.g374f81d7ae-goog