cobalt/sched: improve watchdog accuracy
PATCH 1/1 cobalt/sched: improve watchdog accuracy
The original watchdog mechanism was based on a sampling method: every second (built-in value), it used to check the runtime mode of the current task preempted on the ticking CPU. A per-cpu counter was increased by one every time rt/primary mode was detected, then checked against the trigger limit (CONFIG_XENO_OPT_WATCHDOG_TIMEOUT). Otherwise, the counter was reset to zero.
With this fairly naive approach, it only takes a single hit with CONFIG_XENO_OPT_WATCHDOG_TIMEOUT=1 to trigger the watchdog, i.e. if the system-fixed 1s watchdog tick preempts any Xenomai task when it is running in primary mode on the current CPU, the watchdog fires.
The default value of 4s papered over the inherent imprecision of such a coarse-grained method, lengthening the odds of observing false positive triggers.
To improve the accuracy of the watchdog, arm the watchdog timer to fire at the final trigger date directly, right before switching the CPU to primary mode (leave_root()), disarming it when the CPU is about to switch back to secondary mode (enter_root()).
Better accuracy comes at the expense of slightly more overhead when transitioning between primary and secondary modes, which should be acceptable for a debug feature which is not affecting the hot path anyway (i.e. there is no added cost for strictly rt context switches).
--- include/cobalt/kernel/sched.h | 13 ------------- kernel/cobalt/sched.c | 41 ++++++++++++++++++++++++++--------------- kernel/cobalt/timer.c | 7 ------- 3 files changed, 26 insertions(+), 35 deletions(-)
include/cobalt/kernel/sched.h +0 -13
@@ -104,8 +104,6 @@ struct xnsched {#ifdef CONFIG_XENO_OPT_WATCHDOG /*!< Watchdog timer object. */ struct xntimer wdtimer; -/*!< Watchdog tick count. */ -int wdcount;#endif #ifdef CONFIG_XENO_OPT_STATS /*!< Last account switch date (ticks). */
@@ -361,17 +359,6 @@ xnsched_maybe_resched_after_unlocked_switch(struct xnsched *sched)#endif /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */ -#ifdef CONFIG_XENO_OPT_WATCHDOG -static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ - sched->wdcount =0 ; -} -#else /* !CONFIG_XENO_OPT_WATCHDOG */ -static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ -} -#endif /* CONFIG_XENO_OPT_WATCHDOG */ -#include <cobalt/kernel/sched-idle.h> #include <cobalt/kernel/sched-rt.h>
kernel/cobalt/sched.c +26 -15
83 lines not on this page · raw message
kernel/cobalt/timer.c +0 -7
@@ -893,10 +893,6 @@ int xntimer_grab_hardware(void)else if (ret ==1 )xntimer_start (&sched->htimer,0 ,0 ,XN_RELATIVE ); -#ifdef CONFIG_XENO_OPT_WATCHDOG -xntimer_start (&sched->wdtimer,1000000000UL ,1000000000UL ,XN_RELATIVE ); -xnsched_reset_watchdog (sched); -#endif xnlock_put_irqrestore (&nklock, s); }
@@ -908,9 +904,6 @@ fail:xnlock_get_irqsave (&nklock, s); sched =xnsched_struct (cpu);xntimer_stop (&sched->htimer); -#ifdef CONFIG_XENO_OPT_WATCHDOG -xntimer_stop (&sched->wdtimer); -#endif xnlock_put_irqrestore (&nklock, s);ipipe_timer_stop (_cpu); }
Signature · 1 line
2.14.4
Quoted 102 lines · Philippe Gerum
The original watchdog mechanism was based on a sampling method: every second (built-in value), it used to check the runtime mode of the current task preempted on the ticking CPU. A per-cpu counter was increased by one every time rt/primary mode was detected, then checked against the trigger limit (
CONFIG_XENO_OPT_WATCHDOG_TIMEOUT). Otherwise, the counter was reset to zero.With this fairly naive approach, it only takes a single hit with
CONFIG_XENO_OPT_WATCHDOG_TIMEOUT=1to trigger the watchdog, i.e. if the system-fixed 1s watchdog tick preempts any Xenomai task when it is running in primary mode on the current CPU, the watchdog fires.The default value of 4s papered over the inherent imprecision of such a coarse-grained method, lengthening the odds of observing false positive triggers.
To improve the accuracy of the watchdog, arm the watchdog timer to fire at the final trigger date directly, right before switching the CPU to primary mode (
leave_root()), disarming it when the CPU is about to switch back to secondary mode (enter_root()).Better accuracy comes at the expense of slightly more overhead when transitioning between primary and secondary modes, which should be acceptable for a debug feature which is not affecting the hot path anyway (i.e. there is no added cost for strictly rt context switches).
Signed-off-by Philippe Gerum <[email protected]>--- include/cobalt/kernel/sched.h | 13 ------------- kernel/cobalt/sched.c | 41 ++++++++++++++++++++++++++--------------- kernel/cobalt/timer.c | 7 ------- 3 files changed, 26 insertions(+), 35 deletions(-)include/cobalt/kernel/sched.h +0
-13@@ -104,8 +104,6 @@ struct xnsched {#ifdef CONFIG_XENO_OPT_WATCHDOG /*!< Watchdog timer object. */ struct xntimer wdtimer; -/*!< Watchdog tick count. */ -int wdcount;#endif #ifdef CONFIG_XENO_OPT_STATS /*!< Last account switch date (ticks). */ @@ -361,17 +359,6 @@ xnsched_maybe_resched_after_unlocked_switch(struct xnsched *sched)#endif /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */ -#ifdef CONFIG_XENO_OPT_WATCHDOG -static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ - sched->wdcount =0 ; -} -#else /* !CONFIG_XENO_OPT_WATCHDOG */ -static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ -} -#endif /* CONFIG_XENO_OPT_WATCHDOG */ -#include <cobalt/kernel/sched-idle.h> #include <cobalt/kernel/sched-rt.h> kernel/cobalt/sched.c +9
-9@@ -93,14 +93,20 @@ void xnsched_register_classes(void)static unsigned long wd_timeout_arg =CONFIG_XENO_OPT_WATCHDOG_TIMEOUT ;module_param_named (watchdog_timeout, wd_timeout_arg, ulong,0644 ); +static inline xnticks_t get_watchdog_timeout (void ) +{ +return wd_timeout_arg *1000000000ULL ; +} +/** * @internal * @fn void watchdog_handler(struct xntimer *timer) * @brief Process watchdog ticks. * -* This internal routine handles incoming watchdog ticks to detect -* software lockups. It kills any offending thread which is found to -* monopolize the CPU so as to starve the Linux kernel for too long. +* This internal routine handles incoming watchdog triggers to detect +* software lockups. It forces the offending thread to stop +* monopolizing the CPU, either by kicking it out of primary mode if +* running in user space, or cancelling it if kernel-based. * * @coretags{coreirq-only, atomic-entry} */ @@ -109,14 +115,15 @@ static void watchdog_handler(struct xntimer *timer)struct xnsched *sched =xnsched_current ();struct xnthread *curr = sched->curr; -if (likely (xnthread_test_state (curr,XNROOT ))) { -xnsched_reset_watchdog (sched); -return ; - } - -if (likely (++sched->wdcount < wd_timeout_arg))
kernel/cobalt/sched.c static void watchdog_handler(struct xntimer *timer) · line 126
Quoted 10 lines
kernel/cobalt/sched.c +8
-1+/* +* CAUTION: The watchdog tick might have been delayed while we +* were busy switching the CPU to secondary mode at the +* trigger date eventually. Make sure that we are not about to +* kick the incoming root thread. +*/ +if (xnthread_test_state (curr,XNROOT ))return ; - +
Stray whitespace change. Could fix that on merge but...
Quoted 62 lines
kernel/cobalt/sched.c +9
-5trace_cobalt_watchdog_signal (curr);if (xnthread_test_state (curr,XNUSER )) {@@ -137,8 +144,6 @@ static void watchdog_handler(struct xntimer *timer)*/ xnthread_set_info(curr, XNKICKED|XNCANCELD); } - -xnsched_reset_watchdog (sched); }#endif /* CONFIG_XENO_OPT_WATCHDOG */ @@ -818,6 +823,10 @@ static inline void enter_root(struct xnthread *root) {struct xnarchtcb *rootcb__maybe_unused =xnthread_archtcb (root); +#ifdef CONFIG_XENO_OPT_WATCHDOG +xntimer_stop (&root->sched->wdtimer); +#endif +#ifdef CONFIG_XENO_ARCH_UNLOCKED_SWITCH if (rootcb->core.mm ==NULL )set_ti_thread_flag (rootcb->core.tip,TIF_MMSWITCH_INT );@@ -840,6 +849,11 @@ static inline void leave_root(struct xnthread *root) rootcb->core.tip =task_thread_info (p);#endif xnarch_leave_root (root); + +#ifdef CONFIG_XENO_OPT_WATCHDOG +xntimer_start (&root->sched->wdtimer,get_watchdog_timeout (), +XN_INFINITE ,XN_RELATIVE ); +#endif }void __xnsched_run_handler (void )/* hw interrupts off. */ @@ -889,9 +903,6 @@ reschedule:trace_cobalt_switch_context (prev, next); -if (xnthread_test_state (next,XNROOT )) -xnsched_reset_watchdog (sched); - sched->curr = next; shadow =1 ;kernel/cobalt/timer.c +0
-4@@ -893,10 +893,6 @@ int xntimer_grab_hardware(void)else if (ret ==1 )xntimer_start (&sched->htimer,0 ,0 ,XN_RELATIVE ); -#ifdef CONFIG_XENO_OPT_WATCHDOG -xntimer_start (&sched->wdtimer,1000000000UL ,1000000000UL ,XN_RELATIVE ); -xnsched_reset_watchdog (sched); -#endif xnlock_put_irqrestore (&nklock, s); }
kernel/cobalt/timer.c fail: · line 909
Quoted 9 lines
kernel/cobalt/timer.c +0
-3@@ -908,9 +904,6 @@ fail:xnlock_get_irqsave (&nklock, s); sched =xnsched_struct (cpu);xntimer_stop (&sched->htimer); -#ifdef CONFIG_XENO_OPT_WATCHDOG -xntimer_stop (&sched->wdtimer); -#endif xnlock_put_irqrestore (&nklock, s);ipipe_timer_stop (_cpu); }
... it does not apply to next. There must be something between this patch and current next (and it's not "cobalt/sched: group high-level init/cleanup code") which breaks things.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux
Quoted 173 lines · Jan Kiszka
Quoted 100 lines · Philippe Gerum
The original watchdog mechanism was based on a sampling method: every second (built-in value), it used to check the runtime mode of the current task preempted on the ticking CPU. A per-cpu counter was increased by one every time rt/primary mode was detected, then checked against the trigger limit (
CONFIG_XENO_OPT_WATCHDOG_TIMEOUT). Otherwise, the counter was reset to zero.With this fairly naive approach, it only takes a single hit with
CONFIG_XENO_OPT_WATCHDOG_TIMEOUT=1to trigger the watchdog, i.e. if the system-fixed 1s watchdog tick preempts any Xenomai task when it is running in primary mode on the current CPU, the watchdog fires.The default value of 4s papered over the inherent imprecision of such a coarse-grained method, lengthening the odds of observing false positive triggers.
To improve the accuracy of the watchdog, arm the watchdog timer to fire at the final trigger date directly, right before switching the CPU to primary mode (
leave_root()), disarming it when the CPU is about to switch back to secondary mode (enter_root()).Better accuracy comes at the expense of slightly more overhead when transitioning between primary and secondary modes, which should be acceptable for a debug feature which is not affecting the hot path anyway (i.e. there is no added cost for strictly rt context switches).
Signed-off-by Philippe Gerum <[email protected]>---
include/cobalt/kernel/sched.h| 13 ------------- kernel/cobalt/sched.c | 41
++++++++++++++++++++++++++---------------
kernel/cobalt/timer.c | 7 -------3 files changed, 26 insertions(+), 35 deletions(-)include/cobalt/kernel/sched.h +0
-2@@ -104,8 +104,6 @@ struct xnsched { #ifdef CONFIG_XENO_OPT_WATCHDOG /*!< Watchdog timer object. */ struct xntimer wdtimer; - /*!< Watchdog tick count. */ - int wdcount; #endif #ifdef CONFIG_XENO_OPT_STATS /*!< Last account switch date (ticks). */@@ -361,17 +359,6 @@ xnsched_maybe_resched_after_unlocked_switch(struct xnsched *sched) #endif /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */-#ifdef
CONFIG_XENO_OPT_WATCHDOGinclude/cobalt/kernel/sched.h +0
-10-static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ - sched->wdcount = 0; -} -#else /* !CONFIG_XENO_OPT_WATCHDOG */ -static inline void xnsched_reset_watchdog (struct xnsched *sched) -{ -} -#endif /* CONFIG_XENO_OPT_WATCHDOG */ -#include <
cobalt/kernel/sched-idle.h>
#include <cobalt/kernel/sched-rt.h>
diff --gita/kernel/cobalt/sched.cb/kernel/cobalt/sched.cindex76e3bc5d7..dd14d4975100644kernel/cobalt/sched.c +8
-8@@ -93,14 +93,20 @@ void xnsched_register_classes(void) static unsigned long wd_timeout_arg = CONFIG_XENO_OPT_WATCHDOG_TIMEOUT; module_param_named(watchdog_timeout, wd_timeout_arg, ulong, 0644); +static inline xnticks_t get_watchdog_timeout(void) +{ + return wd_timeout_arg * 1000000000ULL; +} + /*** @internal * @fn void watchdog_handler(struct xntimer *timer) * @brief Process watchdog ticks. * -* This internal routine handles incoming watchdog ticks to detect -* software lockups. It kills any offending thread which is found to -* monopolize the CPU so as to starve the Linux kernel for too long. +* This internal routine handles incoming watchdog triggers to detect +* software lockups. It forces the offending thread to stop +* monopolizing the CPU, either by kicking it out of primary mode if +* running in user space, or cancelling it if kernel-based. * * @coretags{coreirq-only, atomic-entry} */ @@ -109,14 +115,15 @@ static void watchdog_handler(struct xntimer *timer) struct xnsched *sched = xnsched_current(); struct xnthread *curr = sched->curr; - if (likely(xnthread_test_state(curr, XNROOT))) { - xnsched_reset_watchdog(sched); - return; - } - - if (likely(++sched->wdcount < wd_timeout_arg))kernel/cobalt/sched.c static void watchdog_handler(struct xntimer *timer) · line 126
Quoted 10 lines
kernel/cobalt/sched.c +8
-1+ /* + * CAUTION: The watchdog tick might have been delayed while we + * were busy switching the CPU to secondary mode at the + * trigger date eventually. Make sure that we are not about to + * kick the incoming root thread. + */ + if (xnthread_test_state(curr, XNROOT)) return; - +Stray whitespace change. Could fix that on merge but...
Quoted 59 lines
kernel/cobalt/sched.c +0
-2trace_cobalt_watchdog_signal(curr); if (xnthread_test_state(curr, XNUSER)) {@@ -137,8 +144,6 @@ static void watchdog_handler(struct xntimer *timer)*/ xnthread_set_info(curr, XNKICKED|XNCANCELD); } - - xnsched_reset_watchdog(sched); } #endif /* CONFIG_XENO_OPT_WATCHDOG */@@ -818,6 +823,10 @@ static inline void enter_root(struct xnthread *root) { struct xnarchtcb *rootcb __maybe_unused = xnthread_archtcb(root);+#ifdef
CONFIG_XENO_OPT_WATCHDOG
+xntimer_stop(&root->sched->wdtimer);kernel/cobalt/sched.c +2
-0+#endif +#ifdef
CONFIG_XENO_ARCH_UNLOCKED_SWITCH
if (rootcb->core.mm== NULL)set_ti_thread_flag(rootcb->core.tip, TIF_MMSWITCH_INT);kernel/cobalt/sched.c +5
-0@@ -840,6 +849,11 @@ static inline void leave_root(struct xnthread *root) rootcb->core.tip = task_thread_info(p); #endif xnarch_leave_root(root); + +#ifdef CONFIG_XENO_OPT_WATCHDOG + xntimer_start(&root->sched->wdtimer, get_watchdog_timeout(), + XN_INFINITE, XN_RELATIVE); +#endif }void
__xnsched_run_handler(void)/* hw interrupts off. */kernel/cobalt/sched.c +0
-2@@ -889,9 +903,6 @@ reschedule: trace_cobalt_switch_context(prev, next); - if (xnthread_test_state(next, XNROOT)) - xnsched_reset_watchdog(sched); - sched->curr = next; shadow = 1;diff --git
a/kernel/cobalt/timer.cb/kernel/cobalt/timer.cindex6d985a995..5d8b3bc76100644kernel/cobalt/timer.c +0
-0@@ -893,10 +893,6 @@ int xntimer_grab_hardware(void) else if (ret == 1) xntimer_start(&sched->htimer, 0, 0, XN_RELATIVE);-#ifdef
CONFIG_XENO_OPT_WATCHDOG
-xntimer_start(&sched->wdtimer, 1000000000UL, 1000000000UL, XN_RELATIVE);
-xnsched_reset_watchdog(sched);
-#endif
xnlock_put_irqrestore(&nklock, s);
}
@@ -908,9 +904,6 @@ fail:
xnlock_get_irqsave(&nklock, s);
sched =xnsched_struct(cpu);xntimer_stop(&sched->htimer);kernel/cobalt/timer.c +0
-3-#ifdef CONFIG_XENO_OPT_WATCHDOG - xntimer_stop(&sched->wdtimer); -#endif
Quoted 9 lines
Quoted 4 lines
xnlock_put_irqrestore(&nklock, s);
ipipe_timer_stop(_cpu);
}... it does not apply to next. There must be something between this patch and current next (and it's not "cobalt/sched: group high-level init/cleanup code") which breaks things.
Same answer as previous patch. It also applies to the stable/future-master-base branch, mainly aimed at starting the review process.
Signature · 1 line
Philippe.