Skip to contentSkip to conversation

watchdog: avoid extra sys_info dumps for all_bt

PATCH In review Bradley Morgan 3 participants 6 messages Jun 20 – Jun 23

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 watchdog: avoid extra sys_info dumps for all_bt

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Skip sys_info() for that case.

Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info on system lockup")
---
 kernel/watchdog.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
kernel/watchdog.c +18 -4
index 87dd5e0f6968..bad390a9b59e 100644
@@ -54,6 +54,16 @@ static int __read_mostly watchdog_hardlockup_available;
 struct cpumask watchdog_cpumask __read_mostly;
 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
 
+static void watchdog_sys_info(unsigned long si_mask)
+{
+	unsigned long dump_mask = si_mask & ~SYS_INFO_ALL_BT;
+
+	if (si_mask && !dump_mask)
+		return;
+
+	sys_info(dump_mask);
+}
+
 #ifdef CONFIG_HARDLOCKUP_DETECTOR
 
 # ifdef CONFIG_SMP
@@ -208,6 +218,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 {
 	int hardlockup_all_cpu_backtrace;
 	unsigned int this_cpu;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (per_cpu(watchdog_hardlockup_touched, cpu)) {
@@ -216,7 +227,8 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 		return;
 	}
 
-	hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(hardlockup_si_mask);
+	hardlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_hardlockup_all_cpu_backtrace;
 	/*
 	 * Check for a hardlockup by making sure the CPU's timer
@@ -286,7 +298,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 			clear_bit_unlock(0, &hard_lockup_nmi_warn);
 	}
 
-	sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
+	watchdog_sys_info(si_mask);
 	if (hardlockup_panic)
 		nmi_panic(regs, "Hard LOCKUP");
 
@@ -798,6 +810,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	struct pt_regs *regs = get_irq_regs();
 	int softlockup_all_cpu_backtrace;
 	int duration, thresh_count;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (!watchdog_enabled)
@@ -809,7 +822,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	if (panic_in_progress())
 		return HRTIMER_NORESTART;
 
-	softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(softlockup_si_mask);
+	softlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_softlockup_all_cpu_backtrace;
 
 	watchdog_hardlockup_kick();
@@ -900,7 +914,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 		}
 
 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
-		sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);
+		watchdog_sys_info(si_mask);
 		thresh_count = duration / get_softlockup_thresh();
 
 		if (softlockup_panic && thresh_count >= softlockup_panic)
Signature · 1 line
2.53.0
#1 lore raw

Hi,

Quoted 4 lines · Bradley Morgan

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Skip sys_info() for that case.

Thanks for the patch!

I would explain some about he intention of the global 'kernel_sys_info' which was suggested by Andrew. One use case is, for kernel stability issues, it could be panic, soft/hard lockup, task-hung etc, and we now have different conrol knobs for them each, but with similar 'xxx_sys_info' capability, and they actually need similar system info like all-cpu dump, blocked processes dump, debug ftrace dump, using one central 'kernel_sys_info' could be very handy by avoiding setting many knobs. And for debugging random issues, you can just add sys_info(0) everywhere, and controll it by the existing 'kernel_si_mask'

btw, did you meet some issues with current code? If yes, could you agive some more details ? IIUC, when 'kernel_si_mask' is not set specifically, sys_info(0) is a nop.

Thanks,
Feng

Quoted 83 lines
Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info on system lockup")
---
 kernel/watchdog.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
kernel/watchdog.c +18 -4
index 87dd5e0f6968..bad390a9b59e 100644
@@ -54,6 +54,16 @@ static int __read_mostly watchdog_hardlockup_available;
 struct cpumask watchdog_cpumask __read_mostly;
 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
 
+static void watchdog_sys_info(unsigned long si_mask)
+{
+	unsigned long dump_mask = si_mask & ~SYS_INFO_ALL_BT;
+
+	if (si_mask && !dump_mask)
+		return;
+
+	sys_info(dump_mask);
+}
+
 #ifdef CONFIG_HARDLOCKUP_DETECTOR
 
 # ifdef CONFIG_SMP
@@ -208,6 +218,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 {
 	int hardlockup_all_cpu_backtrace;
 	unsigned int this_cpu;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (per_cpu(watchdog_hardlockup_touched, cpu)) {
@@ -216,7 +227,8 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 		return;
 	}
 
-	hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(hardlockup_si_mask);
+	hardlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_hardlockup_all_cpu_backtrace;
 	/*
 	 * Check for a hardlockup by making sure the CPU's timer
@@ -286,7 +298,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 			clear_bit_unlock(0, &hard_lockup_nmi_warn);
 	}
 
-	sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
+	watchdog_sys_info(si_mask);
 	if (hardlockup_panic)
 		nmi_panic(regs, "Hard LOCKUP");
 
@@ -798,6 +810,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	struct pt_regs *regs = get_irq_regs();
 	int softlockup_all_cpu_backtrace;
 	int duration, thresh_count;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (!watchdog_enabled)
@@ -809,7 +822,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	if (panic_in_progress())
 		return HRTIMER_NORESTART;
 
-	softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(softlockup_si_mask);
+	softlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_softlockup_all_cpu_backtrace;
 
 	watchdog_hardlockup_kick();
@@ -900,7 +914,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 		}
 
 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
-		sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);
+		watchdog_sys_info(si_mask);
 		thresh_count = duration / get_softlockup_thresh();
 
 		if (softlockup_panic && thresh_count >= softlockup_panic)
Signature · 1 line
2.53.0
#2 lore raw
Quoted 20 lines · Feng Tang

Hi,

Quoted 4 lines · Bradley Morgan

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Skip sys_info() for that case.

Thanks for the patch!

I would explain some about he intention of the global 'kernel_sys_info' which was suggested by Andrew. One use case is, for kernel stability issues, it could be panic, soft/hard lockup, task-hung etc, and we now have different conrol knobs for them each, but with similar 'xxx_sys_info' capability, and they actually need similar system info like all-cpu dump, blocked processes dump, debug ftrace dump, using one central 'kernel_sys_info' could be very handy by avoiding setting many knobs. And for debugging random issues, you can just add sys_info(0) everywhere, and controll it by the existing 'kernel_si_mask'

Quoted 6 lines

btw, did you meet some issues with current code? If yes, could you agive some more details ? IIUC, when 'kernel_si_mask' is not set specifically, sys_info(0) is a nop.

Thanks,
Feng

Feng!

I see what you are saying, but there is a "double ended sword"! :)

Unfortunately. The issue is where the watchdog specific mask is explicitly set to ONLY all_bt for instance.

The watchdog only handles SYS_INFO_ALL_BT.

↓

so the later call becomes sys_info(0).

At that point sys_info() cannot tell that the caller had a nonzero watchdog mask, and it falls back to kernel_sys_info.

Here is a example where this would be reached:

kernel.kernel_sys_info=tasks,mem kernel.hardlockup_sys_info=all_bt

a hard lockup still dumps tasks and memory through the global default.

I expected hardlockup_sys_info to override the global default once it is explicitly set IMHO.

IF hardlockup_sys_info is empty..

This patch keeps the existing fallback! :)

So the **bug** is not sys_info(0) itself.

but the watchdog turning an explicit
all_bt ONLY mask into zero after handling all_bt locally.

What do you think Feng? Is this good to you? :)

Cheers!

Quoted 92 lines
Quoted 1 line
Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info on

system lockup")

kernel/watchdog.c static int __read_mostly

Quoted 10 lines
---
 kernel/watchdog.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
kernel/watchdog.c +0 -0
index 87dd5e0f6968..bad390a9b59e 100644
@@ -54,6 +54,16 @@ static int __read_mostly

watchdog_hardlockup_available;

kernel/watchdog.c void watchdog_hardlockup_check(unsigned int cpu,

Quoted 17 lines
kernel/watchdog.c +10 -0
 struct cpumask watchdog_cpumask __read_mostly;
 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
 
+static void watchdog_sys_info(unsigned long si_mask)
+{
+	unsigned long dump_mask = si_mask & ~SYS_INFO_ALL_BT;
+
+	if (si_mask && !dump_mask)
+		return;
+
+	sys_info(dump_mask);
+}
+
 #ifdef CONFIG_HARDLOCKUP_DETECTOR
 
 # ifdef CONFIG_SMP
@@ -208,6 +218,7 @@ void watchdog_hardlockup_check(unsigned int cpu,

struct pt_regs *regs)

kernel/watchdog.c void watchdog_hardlockup_check(unsigned int cpu,

Quoted 8 lines
kernel/watchdog.c +1 -0
 {
 	int hardlockup_all_cpu_backtrace;
 	unsigned int this_cpu;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (per_cpu(watchdog_hardlockup_touched, cpu)) {
@@ -216,7 +227,8 @@ void watchdog_hardlockup_check(unsigned int cpu,

struct pt_regs *regs)

kernel/watchdog.c void watchdog_hardlockup_check(unsigned int cpu,

Quoted 10 lines
kernel/watchdog.c +2 -1
 		return;
 	}
 
-	hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(hardlockup_si_mask);
+	hardlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_hardlockup_all_cpu_backtrace;
 	/*
 	 * Check for a hardlockup by making sure the CPU's timer
@@ -286,7 +298,7 @@ void watchdog_hardlockup_check(unsigned int cpu,

struct pt_regs *regs)

kernel/watchdog.c static enum hrtimer_restart watchdog_timer_fn(struct

Quoted 9 lines
kernel/watchdog.c +1 -1
 			clear_bit_unlock(0, &hard_lockup_nmi_warn);
 	}
 
-	sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
+	watchdog_sys_info(si_mask);
 	if (hardlockup_panic)
 		nmi_panic(regs, "Hard LOCKUP");
 
@@ -798,6 +810,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct

hrtimer *hrtimer)

kernel/watchdog.c static enum hrtimer_restart watchdog_timer_fn(struct

Quoted 8 lines
kernel/watchdog.c +1 -0
 	struct pt_regs *regs = get_irq_regs();
 	int softlockup_all_cpu_backtrace;
 	int duration, thresh_count;
+	unsigned long si_mask;
 	unsigned long flags;
 
 	if (!watchdog_enabled)
@@ -809,7 +822,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct

hrtimer *hrtimer)

kernel/watchdog.c static enum hrtimer_restart watchdog_timer_fn(struct

Quoted 10 lines
kernel/watchdog.c +2 -1
 	if (panic_in_progress())
 		return HRTIMER_NORESTART;
 
-	softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ?
+	si_mask = READ_ONCE(softlockup_si_mask);
+	softlockup_all_cpu_backtrace = (si_mask & SYS_INFO_ALL_BT) ?
 					1 : sysctl_softlockup_all_cpu_backtrace;
 
 	watchdog_hardlockup_kick();
@@ -900,7 +914,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct

hrtimer *hrtimer)

Quoted 10 lines
kernel/watchdog.c +1 -1
 		}
 
 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
-		sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);
+		watchdog_sys_info(si_mask);
 		thresh_count = duration / get_softlockup_thresh();
 
 		if (softlockup_panic && thresh_count >= softlockup_panic)
Signature · 1 line
2.53.0
#3 lore raw
Quoted 46 lines · Bradley Morgan
Quoted 20 lines · Feng Tang

Hi,

Quoted 4 lines · Bradley Morgan

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Skip sys_info() for that case.

Thanks for the patch!

I would explain some about he intention of the global 'kernel_sys_info' which was suggested by Andrew. One use case is, for kernel stability issues, it could be panic, soft/hard lockup, task-hung etc, and we now have different conrol knobs for them each, but with similar 'xxx_sys_info' capability, and they actually need similar system info like all-cpu dump, blocked processes dump, debug ftrace dump, using one central 'kernel_sys_info' could be very handy by avoiding setting many knobs. And for debugging random issues, you can just add sys_info(0) everywhere, and controll it by the existing 'kernel_si_mask'

Quoted 6 lines

btw, did you meet some issues with current code? If yes, could you agive some more details ? IIUC, when 'kernel_si_mask' is not set specifically, sys_info(0) is a nop.

Thanks,
Feng

Feng!

I see what you are saying, but there is a "double ended sword"! :)

Unfortunately. The issue is where the watchdog specific mask is explicitly set to ONLY all_bt for instance.

The watchdog only handles SYS_INFO_ALL_BT.

↓

so the later call becomes sys_info(0).

At that point sys_info() cannot tell that the caller had a nonzero watchdog mask, and it falls back to kernel_sys_info.

Quoted 5 lines

Here is a example where this would be reached:

kernel.kernel_sys_info=tasks,mem kernel.hardlockup_sys_info=all_bt

a hard lockup still dumps tasks and memory through the global default.

Thanks for providing the case. I cannot say this is not a valid case :), so I'm not against your patch, and would listen to other developers' opinions.

Thanks,
Feng

Quoted 20 lines

I expected hardlockup_sys_info to override the global default once it is explicitly set IMHO.

IF hardlockup_sys_info is empty..

This patch keeps the existing fallback! :)

So the **bug** is not sys_info(0) itself.

but the watchdog turning an explicit
all_bt ONLY mask into zero after handling all_bt locally.

What do you think Feng? Is this good to you? :)

Cheers!

Quoted 3 lines
Quoted 1 line
Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info on

system lockup")

Quoted 1 line
#4 lore raw
Quoted 52 lines · Feng Tang
Quoted 8 lines · Bradley Morgan
Quoted 6 lines · Feng Tang

Hi,

Quoted 3 lines · Bradley Morgan

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Quoted 8 lines
Quoted 8 lines
Quoted 1 line

Skip sys_info() for that case.

Thanks for the patch!

I would explain some about he intention of the global 'kernel_sys_info' which was suggested by Andrew. One use case is, for kernel stability issues, it could be panic, soft/hard lockup, task-hung etc, and we now have different conrol knobs for them each, but with similar

'xxx_sys_info'

Quoted 34 lines
Quoted 6 lines

capability, and they actually need similar system info like all-cpu dump, blocked processes dump, debug ftrace dump, using one central 'kernel_sys_info' could be very handy by avoiding setting many knobs. And for debugging random issues, you can just add sys_info(0) everywhere, and controll it by the existing 'kernel_si_mask'

Quoted 6 lines

btw, did you meet some issues with current code? If yes, could you agive some more details ? IIUC, when 'kernel_si_mask' is not set specifically, sys_info(0) is a nop.

Thanks,
Feng

Feng!

I see what you are saying, but there is a "double ended sword"! :)

Unfortunately. The issue is where the watchdog specific mask is explicitly set to ONLY all_bt for instance.

The watchdog only handles SYS_INFO_ALL_BT.

↓

so the later call becomes sys_info(0).

At that point sys_info() cannot tell that the caller had a nonzero watchdog mask, and it falls back to kernel_sys_info.

Here is a example where this would be reached:

kernel.kernel_sys_info=tasks,mem kernel.hardlockup_sys_info=all_bt

Quoted 8 lines
Quoted 1 line

a hard lockup still dumps tasks and memory through the global default.

Thanks for providing the case. I cannot say this is not a valid case :), so I'm not against your patch, and would listen to other developers' opinions.

Thanks,
Feng

Thanks Feng!

Quoted 13 lines
Quoted 13 lines

I expected hardlockup_sys_info to override the global default once it is explicitly set IMHO.

IF hardlockup_sys_info is empty..

This patch keeps the existing fallback! :)

So the **bug** is not sys_info(0) itself.

but the watchdog turning an explicit
all_bt ONLY mask into zero after handling all_bt locally.

Quoted 9 lines
Quoted 5 lines

What do you think Feng? Is this good to you? :)

Cheers!

Quoted 1 line
Quoted 1 line
Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info

on

Quoted 2 lines
Quoted 2 lines

system lockup")

Quoted 1 line

Thanks!

#5 lore raw
In reply to Bradley Morgan
Quoted 2 lines · Bradley Morgan

The watchdog handles SYS_INFO_ALL_BT itself. When that is the only watchdog specific bit, sys_info(0) falls back to kernel_sys_info.

Great catch!

kernel/watchdog.c static int __read_mostly watchdog_hardlockup_available; · line 56

Quoted 11 lines

Skip sys_info() for that case.

Fixes a9af76a78760 ("watchdog: add sys_info sysctls to dump sys info on system lockup")
kernel/watchdog.c +0 -0
@@ -54,6 +54,16 @@ static int __read_mostly watchdog_hardlockup_available;
 struct cpumask watchdog_cpumask __read_mostly;
 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
 

Similar problem exists also in kernel/panic.c. It would make sense to put this variant into lib/sys_info.c and use a generic name, e.g. sys_info_without_all_bt().

kernel/watchdog.c static int __read_mostly watchdog_hardlockup_available; · line 60

Quoted 4 lines
kernel/watchdog.c +4 -0
+static void watchdog_sys_info(unsigned long si_mask)
+{
+	unsigned long dump_mask = si_mask & ~SYS_INFO_ALL_BT;
+

This looks like an optimization. The real reason is not obvious. It would deserve a comment, something like:

/*
* Do not call sys_info() when the caller context required
* only backtraces from all CPUs. Otherwise, sys_info()
* would fall back to the generic "kernel_si_mask". */

kernel/watchdog.c static int __read_mostly watchdog_hardlockup_available; · line 65

Quoted 5 lines
kernel/watchdog.c +5 -0
+	if (si_mask && !dump_mask)
+		return;
+
+	sys_info(dump_mask);
+}

Alternative solution would be to remove the custom trigger_all*_cpu_backtrace() calls from both watchdog and panic code and rely on sys_info() API. But it is not easy, especially in panic().

Otherwise, the changes make sense.

Best Regards,
Petr