Skip to contentSkip to conversation

: Fix bogus softlockup warning with sysrq-t

PATCH In review Prarit Bhargava 4 participants 8 messages Mar 15, 2007 – Mar 27, 2007

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 : Fix bogus softlockup warning with sysrq-t

There are some situations when soft lockup warnings are expected in the kernel. For example, when doing an alt-sysrq-t on a large number of processes, the dump to console can take a long time and the tasklist_lock is held over that period. This results in a bogus soft lockup warning.

This patch reworks touch_softlockup_watchdog to touch ALL cpu's touch_timestamp. It also introduces touch_cpu_softlockup_watchdog to touch a single cpu's touch_timestamp. This makes it functionally equivalent to touch_nmi_watchdog.

touch_nmi_watchdog is not modified -- AFAICT it was attempting to touch all cpu's softlockup watchdogs, not just a specific cpu.

/drivers/ide/ide-iops.c does not need to call touch_softlockup_watchdog as it is done in the call to touch_nmi_watchdog.

The EXPORT_SYMBOL for touch_softlockup_watchdog is needed by drivers/scsi/ips.ko

arch/ia64/kernel/uncached.c +1 -1
index c58e933..db514bb 100644
@@ -254,7 +254,7 @@ static int __init uncached_build_memmap(unsigned long uc_start,
 	struct gen_pool *pool = uncached_pools[nid].pool;
 	size_t size = uc_end - uc_start;
 
-	touch_softlockup_watchdog();
+	touch_cpu_softlockup_watchdog();
 
 	if (pool != NULL) {
 		memset((char *)uc_start, 0, size);
drivers/ide/ide-iops.c +0 -1
index bd513f5..176c97b 100644
@@ -1225,7 +1225,6 @@ int ide_wait_not_busy(ide_hwif_t *hwif, unsigned long timeout)
 		 */
 		if (stat == 0xff)
 			return -ENODEV;
-		touch_softlockup_watchdog();
 		touch_nmi_watchdog();
 	}
 	return -EBUSY;
drivers/ide/ide-taskfile.c +1 -1
index 30175c7..5fb2dce 100644
@@ -313,7 +313,7 @@ static void ide_pio_datablock(ide_drive_t *drive, struct request *rq,
 	if (rq->bio)	/* fs request */
 		rq->errors = 0;
 
-	touch_softlockup_watchdog();
+	touch_cpu_softlockup_watchdog();
 
 	switch (drive->hwif->data_phase) {
 	case TASKFILE_MULTI_IN:
drivers/mtd/nand/nand_base.c +1 -1
index 6af37b8..9669d5f 100644
@@ -425,7 +425,7 @@ void nand_wait_ready(struct mtd_info *mtd)
 	do {
 		if (chip->dev_ready(mtd))
 			break;
-		touch_softlockup_watchdog();
+		touch_cpu_softlockup_watchdog();
 	} while (time_before(jiffies, timeo));
 	led_trigger_event(nand_led_trigger, LED_OFF);
 }
include/linux/sched.h +4 -0
index 49fe299..113421e 100644
@@ -225,6 +225,7 @@ extern void scheduler_tick(void);
 #ifdef CONFIG_DETECT_SOFTLOCKUP
 extern void softlockup_tick(void);
 extern void spawn_softlockup_task(void);
+extern void touch_cpu_softlockup_watchdog(void);
 extern void touch_softlockup_watchdog(void);
 #else
 static inline void softlockup_tick(void)
@@ -233,6 +234,9 @@ static inline void softlockup_tick(void)
 static inline void spawn_softlockup_task(void)
 {
 }
+static inline void touch_cpu_softlockup_watchdog(void)
+{
+}
 static inline void touch_softlockup_watchdog(void)
 {
 }
kernel/panic.c +1 -1
index 623d182..9e4565c 100644
@@ -132,7 +132,7 @@ NORET_TYPE void panic(const char * fmt, ...)
 #endif
 	local_irq_enable();
 	for (i = 0;;) {
-		touch_softlockup_watchdog();
+		touch_cpu_softlockup_watchdog();
 		i += panic_blink(i);
 		mdelay(1);
 		i++;
kernel/power/swsusp.c +1 -1
index 7fb8343..d259b12 100644
@@ -323,7 +323,7 @@ int swsusp_resume(void)
 	 */
 	swsusp_free();
 	restore_processor_state();
-	touch_softlockup_watchdog();
+	touch_cpu_softlockup_watchdog();
 	device_power_up();
 	local_irq_enable();
 	return error;
kernel/sched.c +1 -0
index a4ca632..243f558 100644
@@ -4783,6 +4783,7 @@ void show_state_filter(unsigned long state_filter)
 		if (p->state & state_filter)
 			show_task(p);
 	} while_each_thread(g, p);
+	touch_softlockup_watchdog();
 
 	read_unlock(&tasklist_lock);
 	/*
kernel/softlockup.c +13 -3
index 50afeb8..05b66db 100644
@@ -8,6 +8,7 @@
  */
 #include <linux/mm.h>
 #include <linux/cpu.h>
+#include <linux/cpumask.h>
 #include <linux/init.h>
 #include <linux/delay.h>
 #include <linux/kthread.h>
@@ -34,10 +35,19 @@ static struct notifier_block panic_block = {
 	.notifier_call = softlock_panic,
 };
 
-void touch_softlockup_watchdog(void)
+void touch_cpu_softlockup_watchdog(void)
 {
 	__raw_get_cpu_var(touch_timestamp) = jiffies;
 }
+EXPORT_SYMBOL(touch_cpu_softlockup_watchdog);
+
+void touch_softlockup_watchdog(void)
+{
+	int cpu;
+
+	for_each_online_cpu(cpu)
+		per_cpu(touch_timestamp, cpu) = jiffies;
+}
 EXPORT_SYMBOL(touch_softlockup_watchdog);
 
 /*
@@ -57,7 +67,7 @@ void softlockup_tick(void)
 
 	/* do not print during early bootup: */
 	if (unlikely(system_state != SYSTEM_RUNNING)) {
-		touch_softlockup_watchdog();
+		touch_cpu_softlockup_watchdog();
 		return;
 	}
 
@@ -94,7 +104,7 @@ static int watchdog(void * __bind_cpu)
 	 */
 	while (!kthread_should_stop()) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		touch_softlockup_watchdog();
+		touch_cpu_softlockup_watchdog();
 		schedule();
 	}
 
#2 lore raw
In reply to Prarit Bhargava
Quoted 5 lines · Prarit Bhargava

There are some situations when soft lockup warnings are expected in the kernel. For example, when doing an alt-sysrq-t on a large number of processes, the dump to console can take a long time and the tasklist_lock is held over that period. This results in a bogus soft lockup warning.

Wouldn't it be better to just temporarily disable softlockups for the duration?

Quoted 3 lines

This patch reworks touch_softlockup_watchdog to touch ALL cpu's touch_timestamp. It also introduces touch_cpu_softlockup_watchdog to touch a single cpu's touch_timestamp.

Doesn't this mean that if one CPU gets locked up, it will be undetected so long as some other CPU is making progress?

I have another pair of softlockup patches in which I try to address:

* ignoring time stolen by hypervisors
* threads going to sleep tickless for long periods of time

I could easy add a "global disable" function, which would allow long sysrq messages, and it would help Thilo with his long flash update freezes.

J

#3 lore raw
Quoted 2 lines

I could easy add a "global disable" function, which would allow long sysrq messages, and it would help Thilo with his long flash update freezes.

A "global disable" and "reenable" functions pair which works during irq disabled, would be a perfect solution for me. Thx Jeremy for your effort :)

Ciao Thilo

#4 lore raw
In reply to Jeremy Fitzhardinge
Quoted 7 lines

I have another pair of softlockup patches in which I try to address:

* ignoring time stolen by hypervisors
* threads going to sleep tickless for long periods of time

I'm looking at the code now. Your solution is definately better :)

Quoted 4 lines

I could easy add a "global disable" function, which would allow long sysrq messages, and it would help Thilo with his long flash update freezes.

I think that's a good idea -- I'll propose an add on patch to fix the sysrq-t case ...

P.

Quoted 2 lines

J

#5 lore raw
Quoted 2 lines · Prarit Bhargava

I think that's a good idea -- I'll propose an add on patch to fix the sysrq-t case ...

I'm working on this patch at the moment. I'm just wondering what happens if you do a global re-enable while a CPU is locally disabled. I think it won't matter; it will end up in the "enabled but need to update timestamp" state, and the next time it gets a timer tick, it will simply update the timestamp and carry on.

(This is relative to the other two softlockup patches, but modified since I posted them.)

J

drivers/char/sysrq.c +4 -0
diff -r 4c81d8cafb67 drivers/char/sysrq.c
@@ -408,6 +408,8 @@ void __handle_sysrq(int key, struct tty_
 	int i;
 	unsigned long flags;
 
+	softlockup_global_disable();
+
 	spin_lock_irqsave(&sysrq_key_table_lock, flags);
 	orig_log_level = console_loglevel;
 	console_loglevel = 7;
@@ -445,6 +447,8 @@ void __handle_sysrq(int key, struct tty_
 		console_loglevel = orig_log_level;
 	}
 	spin_unlock_irqrestore(&sysrq_key_table_lock, flags);
+
+	softlockup_global_enable();
 }
 
 /*
include/linux/sched.h +8 -0
diff -r 4c81d8cafb67 include/linux/sched.h
@@ -235,6 +235,8 @@ extern void softlockup_tick(void);
 extern void softlockup_tick(void);
 extern void softlockup_enable(void);
 extern void softlockup_disable(void);
+extern void softlockup_global_enable(void);
+extern void softlockup_global_disable(void);
 extern void spawn_softlockup_task(void);
 extern void touch_softlockup_watchdog(void);
 #else
@@ -245,6 +247,12 @@ static inline void softlockup_enable(voi
 {
 }
 static inline void softlockup_disable(void)
+{
+}
+static inline void softlockup_global_enable(void)
+{
+}
+static inline void softlockup_global_disable(void)
 {
 }
 static inline void spawn_softlockup_task(void)
kernel/softlockup.c +39 -7

77 lines not on this page · raw message

#6 lore raw
Quoted 30 lines · Jeremy Fitzhardinge
Quoted 3 lines · Prarit Bhargava

I think that's a good idea -- I'll propose an add on patch to fix the sysrq-t case ...

I'm working on this patch at the moment. I'm just wondering what happens if you do a global re-enable while a CPU is locally disabled. I think it won't matter; it will end up in the "enabled but need to update timestamp" state, and the next time it gets a timer tick, it will simply update the timestamp and carry on.

(This is relative to the other two softlockup patches, but modified since I posted them.)

J

diff -r 4c81d8cafb67 drivers/char/sysrq.c

drivers/char/sysrq.c +2 -0
@@ -408,6 +408,8 @@ void __handle_sysrq(int key, struct tty_
 	int i;
 	unsigned long flags;
 
+	softlockup_global_disable();
+
 	spin_lock_irqsave(&sysrq_key_table_lock, flags);
 	orig_log_level = console_loglevel;
 	console_loglevel = 7;

drivers/char/sysrq.c void __handle_sysrq(int key, struct tty_ · line 452

Quoted 6 lines
drivers/char/sysrq.c +2 -0
@@ -445,6 +447,8 @@ void __handle_sysrq(int key, struct tty_
 		console_loglevel = orig_log_level;
 	}
 	spin_unlock_irqrestore(&sysrq_key_table_lock, flags);
+
+	softlockup_global_enable();
 

I think that works -- I'll test it out on a big honkin' ia64 box ;)

Shouldn't you also do softlockup_disable/softlockup_enable instead of touch_softlockup_watchdog? Why do we have both? I can't see why we would have two exported methods to stop/reset the softlockup timer...

P.

#7 lore raw
In reply to Jeremy Fitzhardinge
Quoted 14 lines · Jeremy Fitzhardinge

diff -r 4c81d8cafb67 drivers/char/sysrq.c

drivers/char/sysrq.c +2 -0
@@ -408,6 +408,8 @@ void __handle_sysrq(int key, struct tty_
 	int i;
 	unsigned long flags;
 
+	softlockup_global_disable();
+
 	spin_lock_irqsave(&sysrq_key_table_lock, flags);
 	orig_log_level = console_loglevel;
 	console_loglevel = 7;

drivers/char/sysrq.c void __handle_sysrq(int key, struct tty_ · line 454

Quoted 8 lines
drivers/char/sysrq.c +2 -0
@@ -445,6 +447,8 @@ void __handle_sysrq(int key, struct tty_
 		console_loglevel = orig_log_level;
 	}
 	spin_unlock_irqrestore(&sysrq_key_table_lock, flags);
+
+	softlockup_global_enable();
 }
 
 

I wonder if that's too strong of a fix. It's only the sysrq-t case that is the problem. AFAIK, none of the other sysrq-t operations hold the tasklist_lock for a long time. I'd move these around show_state.

P.