removal of oops->printk deadlocks

This patch is designed to remove the things which can cause loss of oops information due to the SMP deadlocks which can occur during the oops processing itself. It's fairly x86-specific, but the infrastructure is in place for other SMP-capable architectures to use.

It also changes the x86 NMI oopser so that it no longer shuts the console up after the first NMI oops. Instead, each CPU is allowed to print out NMI diagnostics a single time per reboot.

This patch only addresses console-on-vgacon. If the console is on a serial device then there may still be deadlocks. The serial drivers can individually test `oops_in_progress' to avoid this.

The call to poke_blanked_console() is resurrected, but it's no longer deadlocky. This is fairly important - without this change a machine could be wedged with oops diagnostics on its screen, but the screen would remain blanked.

The patch addresses oopses, die(), NMI oopses and BUG(). The changes to BUG() result in 3 kbytes less .rodata.

There's a tiny kernel module at
http://www.uow.edu.au/~andrewm/linux/ddeadlock.c which may be used to test the NMI oopser changes.

Comments are welcome!

@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \

@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 

@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c

@@ -260,6 +261,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */

@@ -308,7 +311,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 

@@ -77,17 +77,40 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
+ */
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give syslogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+/*
+ * Called from the BUG() macro.
  */
-void bust_spinlocks(void)
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);

@@ -264,8 +287,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else

@@ -283,6 +305,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*

@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {

@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }

@@ -414,8 +415,9 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
+	static unsigned char	printed_message [NR_CPUS];
 
 	/*
 	 * Since current-> is always on the stack, and we always switch

@@ -432,17 +434,19 @@
 		 */
 		alert_counter[cpu]++;
 		if (alert_counter[cpu] == 5*HZ) {
-			spin_lock(&nmi_print_lock);
-			/*
-			 * We are in trouble anyway, lets at least try
-			 * to get a message out.
-			 */
-			bust_spinlocks();
-			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
-			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
-			spin_unlock(&nmi_print_lock);
+			if (printed_message[cpu] == 0) {
+				printed_message[cpu] = 1;
+				spin_lock(&nmi_print_lock);
+				/*
+				 * We are in trouble anyway, lets at least try
+				 * to get a message out.
+				 */
+				bust_spinlocks(1);
+				printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
+				show_registers(regs);
+				spin_unlock(&nmi_print_lock);
+				bust_spinlocks(0);
+			}
 			do_exit(SIGSEGV);
 		}
 	} else {

@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);

@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);

@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 

@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
-

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Hi,

The whole oops produced by the 2.4.0-test11pre4 with this patch is in the attachment. Other system info is tehere as well.

Oops is produced when runnning big-tables test from mysql sql-bench. It's reproducable. The machine is running only mysqld.

regards
pkot

mailto:[email protected]
http://urtica.linuxnews.pl/~pkot/
http://newsreader.linuxnews.pl/

ksymoops 2.3.4 on i686 2.4.0-test11. Options used
-V (default)
-k /proc/ksyms (default)
-l /proc/modules (default)
-o /lib/modules/2.4.0-test11/ (default) -m /boot/System.map-2.4.0-test11 (default)

Warning: You did not tell me where to find symbol information. I will assume that the log matches the kernel and modules that are running right now and I'll use the default options above for symbol resolution. If the current kernel and/or modules do not match the log, you can get more accurate output by telling me the kernel version and where to find map, modules, ksyms etc. ksymoops -h explains the options.

Error (regular_file): read_ksyms stat /proc/ksyms failed No modules in ksyms, skipping objects
No ksyms, skipping lsmod

CPU:    0
EIP:    0010:[<c010b5ee>]

Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00000097
eax: c0285420 ebx: f6f08000 ecx: 00000000 edx: bf1fff2c
esi: bf1fff2c edi: f6f09fa8 ebp: f6f09fa8 esp: f6f09f90
ds: 0018 es: 0018 ss: 0018
Process mysqld (pid: 2386, stackpage=f6f09000)

Stack: c02026f7 f6f08000 00000000 bf1fff24 fffffff2 00000000 f6f09fbc c0120404 
       00000952 00000000 bf1fff28 bf1ffe34 c010c66b 00000952 00000000 bf1fff28 
       00000000 bf1fff24 bf1ffe34 0000009c 0000002b 0000002b 0000009c 081313e4 
Call Trace: [<c02026f7>] [<c0120404>] [<c010c66b>] 
Code: 78 fb f0 ff 08 78 f3 c3 90 90 90 90 90 90 90 90 90 90 55 57 

Trace; c02026f7 <stext_lock+474b/8d34>
Trace; c0120404 <sys_nanosleep+16c/178>
Trace; c010c66b <__global_cli+b/124>
Code; c010b5ee <show_registers+12e/13c>
00000000 <_EIP>:
Code; c010b5ee <show_registers+12e/13c> <=====
0: 78 fb js fffffffd <_EIP+0xfffffffd> c010b5eb <show_registers+12b/13c> <=====
Code; c010b5f0 <show_registers+130/13c>
2: f0 ff 08 lock decl (%eax)
Code; c010b5f3 <show_registers+133/13c>
5: 78 f3 js fffffffa <_EIP+0xfffffffa> c010b5e8 <show_registers+128/13c>
Code; c010b5f5 <show_registers+135/13c>
7: c3 ret
Code; c010b5f6 <show_registers+136/13c>
8: 90 nop
Code; c010b5f7 <show_registers+137/13c>
9: 90 nop
Code; c010b5f8 <show_registers+138/13c>
a: 90 nop
Code; c010b5f9 <show_registers+139/13c>
b: 90 nop
Code; c010b5fa <show_registers+13a/13c>
c: 90 nop
Code; c010b5fb <show_registers+13b/13c>
d: 90 nop
Code; c010b5fc <die+0/58>
e: 90 nop
Code; c010b5fd <die+1/58>
f: 90 nop
Code; c010b5fe <die+2/58>
10: 90 nop
Code; c010b5ff <die+3/58>
11: 90 nop
Code; c010b600 <die+4/58>
12: 55 push %ebp
Code; c010b601 <die+5/58>
13: 57 push %edi

NMI Watchdog detected LOCKUP on CPU1, registers:

CPU:    1
EIP:    0010:[<c0202590>]
EFLAGS: 00000082
eax: c028e7ec   ebx: 00000282   ecx: c4df0000   edx: c028e7ec
esi: c011f550   edi: 00000000   ebp: c4d59ef4   esp: c4d59ef0
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d59000)
Stack: c4df0000 c0285c40 c0128592 c4df0000 00000000 00000020 00000000 c0285c40 
       f70082a0 f7c06800 00000086 c0124de5 c02a1220 00000020 c0124cc7 00000000 
       00000001 c0286040 00000020 0000000e c0124b6c c0286040 c02a1580 c0283800 
Call Trace: [<c0128592>] [<c0124de5>] [<c0124cc7>] [<c0124b6c>] [<c010e225>] [<c010a810>] [<c010a810>] 
       [<c010c72c>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a83d>] [<c010a8a2>] [<c0116c84>] [<c0124b6c>] 
       [<c010e225>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 ba cf f1 ff 80 bb 74 06 

Trace; c0128592 <vmfree_area_pages+122/174>
Trace; c0124de5 <do_generic_file_read+279/544>
Trace; c0124cc7 <do_generic_file_read+15b/544>
Trace; c0124b6c <do_generic_file_read+0/544>
Trace; c010e225 <do_vm86_irq_handling+22d/2a8>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010c72c <__global_cli+cc/124>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a83d <setup_sigcontext+7d/128>
Trace; c010a8a2 <setup_sigcontext+e2/128>
Trace; c0116c84 <getrusage+168/1ec>
Trace; c0124b6c <do_generic_file_read+0/544>
Trace; c010e225 <do_vm86_irq_handling+22d/2a8>
Code; c0202590 <stext_lock+45e4/8d34>
00000000 <_EIP>:
Code; c0202590 <stext_lock+45e4/8d34> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c0202597 <stext_lock+45eb/8d34>
7: f3 90 repz nop
Code; c0202599 <stext_lock+45ed/8d34>
9: 7e f5 jle 0 <_EIP>
Code; c020259b <stext_lock+45ef/8d34>
b: e9 ba cf f1 ff jmp fff1cfca <_EIP+0xfff1cfca> c011f55a <ptrace_writedata+2a/b0>
Code; c02025a0 <stext_lock+45f4/8d34>
10: 80 bb 74 06 00 00 00 cmpb $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU2, registers:

CPU:    2
EIP:    0010:[<c02025c0>]
EFLAGS: 00000082
eax: 00000000   ebx: c0285480   ecx: 00000040   edx: 00000002
esi: c4d7a000   edi: c010a810   ebp: c4d7bfac   esp: c4d7bf78
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d7b000)
Stack: c010a810 c4d7a000 c010a810 c4d7a000 c4d7a000 c010a810 00000000 00000000 
       c4d70018 c0100018 00000002 c4d7a000 c0285480 00000000 c010a8ae 00000003 
       00000000 00000000 c0116c84 c0289373 c012293f 00000003 00000602 00000000 
Call Trace: [<c010a810>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a8ae>] [<c0116c84>] [<c012293f>] 
       [<c0117204>] [<c020de7d>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 b9 d1 f1 ff 80 bb 74 06 

Trace; c010a810 <setup_sigcontext+50/128> Trace; c010a810 <setup_sigcontext+50/128> Trace; c010a810 <setup_sigcontext+50/128> Trace; c0100018 <startup_32+18/cc>
Trace; c010a8ae <setup_sigcontext+ee/128> Trace; c0116c84 <getrusage+168/1ec>
Trace; c012293f <do_mmap_pgoff+187/3f8> Trace; c0117204 <exec_usermodehelper+3d4/3e0> Trace; c020de7d <call_spurious_interrupt+67dd/77c8> Code; c02025c0 <stext_lock+4614/8d34> 00000000 <_EIP>:
Code; c02025c0 <stext_lock+4614/8d34> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c02025c7 <stext_lock+461b/8d34>
7: f3 90 repz nop
Code; c02025c9 <stext_lock+461d/8d34>
9: 7e f5 jle 0 <_EIP>
Code; c02025cb <stext_lock+461f/8d34>
b: e9 b9 d1 f1 ff jmp fff1d1c9 <_EIP+0xfff1d1c9> c011f789 <mod_timer+35/ec>
Code; c02025d0 <stext_lock+4624/8d34>
10: 80 bb 74 06 00 00 00 cmpb $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU3, registers:

CPU:    3
EIP:    0010:[<c0202724>]
EFLAGS: 00000082
eax: f71d4000   ebx: 00000002   ecx: f71d4000   edx: c4d66400
esi: 00000021   edi: 00000000   ebp: f6f15eb0   esp: f6f15eac
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2385, stackpage=f6f15000)
Stack: f71d4000 00000000 c011c3fa f71d4000 00000021 00000082 c011c4bd 00000021 
       f6f15f04 f71d4000 00040001 00000000 f6f14000 00000021 c011c941 00000021 
       f6f15f04 f71d4000 f6f14000 f71d4000 f6f14000 c0285420 00000021 00000000 
Call Trace: [<c011c3fa>] [<c011c4bd>] [<c011c941>] [<c01f2401>] [<c014d6c9>] [<c013b101>] [<c01237bb>] 
       [<c0123b28>] [<c0123b7e>] [<c010c66b>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 d5 e3 f1 ff 80 3b 00 f3 

Trace; c011c3fa <do_adjtimex+c2/390>
Trace; c011c4bd <do_adjtimex+185/390>
Trace; c011c941 <tasklet_hi_action+69/7c>
Trace; c01f2401 <packet_ioctl+16d/340>
Trace; c014d6c9 <mem_read+21/108>
Trace; c013b101 <pipe_read+205/210>
Trace; c01237bb <insert_vm_struct+33/174>
Trace; c0123b28 <__remove_inode_page+54/60>
Trace; c0123b7e <invalidate_inode_pages+6/bc>
Trace; c010c66b <__global_cli+b/124>
Code; c0202724 <stext_lock+4778/8d34>
00000000 <_EIP>:
Code; c0202724 <stext_lock+4778/8d34> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c020272b <stext_lock+477f/8d34>
7: f3 90 repz nop
Code; c020272d <stext_lock+4781/8d34>
9: 7e f5 jle 0 <_EIP>
Code; c020272f <stext_lock+4783/8d34>
b: e9 d5 e3 f1 ff jmp fff1e3e5 <_EIP+0xfff1e3e5> c0120b09 <clear_page_tables+71/84>
Code; c0202734 <stext_lock+4788/8d34>
10: 80 3b 00 cmpb $0x0,(%ebx)
Code; c0202737 <stext_lock+478b/8d34>
13: f3 00 00 repz add %al,(%eax)

1 warning and 1 error issued. Results may not be reliable.

processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 10
model name : Pentium III (Cascades)
stepping : 1
cpu MHz : 697.000732
cache size : 1024 KB
fdiv_bug : no
hlt_bug : no
sep_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 3
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips : 1392.64

processor : 1
vendor_id : GenuineIntel
cpu family : 6
model : 10
model name : Pentium III (Cascades)
stepping : 1
cpu MHz : 697.000732
cache size : 1024 KB
fdiv_bug : no
hlt_bug : no
sep_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 3
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips : 1392.64

processor : 2
vendor_id : GenuineIntel
cpu family : 6
model : 10
model name : Pentium III (Cascades)
stepping : 1
cpu MHz : 697.000732
cache size : 1024 KB
fdiv_bug : no
hlt_bug : no
sep_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 3
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips : 1392.64

processor : 3
vendor_id : GenuineIntel
cpu family : 6
model : 10
model name : Pentium III (Cascades)
stepping : 1
cpu MHz : 697.000732
cache size : 1024 KB
fdiv_bug : no
hlt_bug : no
sep_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 3
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips : 1392.64

00:00.0 Host bridge: Relience Computer CNB20HE (rev 21)
Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:00.1 Host bridge: Relience Computer CNB20HE (rev 01)
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-
Latency: 64 set, cache line size 08

00:00.2 Host bridge: Relience Computer: Unknown device 0006
Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-

00:00.3 Host bridge: Relience Computer: Unknown device 0006
Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-

00:04.0 Ethernet controller: Intel Corporation 82557 [Ethernet Pro 100] (rev 08)
Subsystem: Intel Corporation 82559 Fast Ethernet LAN on Motherboard
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 8 min, 56 max, 64 set, cache line size 08
Interrupt: pin A routed to IRQ 31
Region 0: Memory at e6100000 (32-bit, non-prefetchable) [size=4K]
Region 1: I/O ports at 1400 [size=64]
Region 2: Memory at e6000000 (32-bit, non-prefetchable) [size=1M]
Expansion ROM at <unassigned> [disabled] [size=1M]
Capabilities: [dc] Power Management version 2
Flags: PMEClk- AuxPwr- DSI+ D1+ D2+ PME+
Status: D0 PME-Enable- DSel=0 DScale=2 PME-

00:0c.0 VGA compatible controller: ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC] (rev 7a) (prog-if 00 [VGA])
Subsystem: Intel Corporation: Unknown device 4b52
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B-
Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 8 min, 66 set, cache line size 08
Region 0: Memory at e7000000 (32-bit, non-prefetchable) [size=16M]
Region 1: I/O ports at 1000 [size=256]
Region 2: Memory at e6101000 (32-bit, non-prefetchable) [size=4K]
Expansion ROM at <unassigned> [disabled] [size=128K]
Capabilities: [5c] Power Management version 1
Flags: PMEClk- AuxPwr- DSI- D1- D2- PME-
Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:0f.0 ISA bridge: Relience Computer: Unknown device 0200 (rev 50)
Subsystem: Relience Computer: Unknown device 0200
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 0 set

00:0f.1 IDE interface: Relience Computer: Unknown device 0211 (prog-if 8e [Master SecP SecO PriP])

	Control: I/O+ Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 64 set
	Region 2: I/O ports at 1458 [size=8]
	Region 3: I/O ports at 1450 [size=4]
	Region 4: I/O ports at 1440 [size=16]

00:0f.2 USB Controller: Relience Computer: Unknown device 0220 (rev 04) (prog-if 10 [OHCI])
Subsystem: Relience Computer: Unknown device 0220
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 80 max, 64 set, cache line size 08
Interrupt: pin A routed to IRQ 10
Region 0: Memory at e6102000 (32-bit, non-prefetchable) [size=4K]

01:05.0 PCI Hot-plug controller: Compaq Computer Corporation: Unknown device a0f7 (rev 11)
Subsystem: Intel Corporation: Unknown device 002c
Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap+ 66Mhz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Interrupt: pin A routed to IRQ 30
Region 0: Memory at ea800000 (32-bit, non-prefetchable) [size=256]
Capabilities: [58] Message Signalled Interrupts: 64bit+ Queue=0/0 Enable-
Address: 0000000000000000 Data: 0000
Capabilities: [68] #07 [0000]

01:0a.0 PCI bridge: Intel Corporation: Unknown device 0964 (rev 02) (prog-if 00 [Normal decode])
Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
Status: Cap- 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 64 set, cache line size 08
Bus: primary=01, secondary=04, subordinate=04, sec-latency=64
I/O behind bridge: 00007000-00007fff
Memory behind bridge: ea900000-ea9fffff
Prefetchable memory behind bridge: f6000000-f60fffff
BridgeCtl: Parity- SERR+ NoISA+ VGA- MAbort- >Reset- FastB2B-

01:0a.1 RAID bus controller: Mylex Corporation: Unknown device 0050 (rev 02)
Subsystem: Mylex Corporation: Unknown device 0050
Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR- FastB2B-
Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Latency: 64 set, cache line size 08
Interrupt: pin A routed to IRQ 25
Region 0: Memory at f0000000 (32-bit, prefetchable) [size=8K]
Expansion ROM at <unassigned> [disabled] [size=32K]
Capabilities: [80] Power Management version 2
Flags: PMEClk- AuxPwr- DSI- D1- D2- PME-
Status: D0 PME-Enable- DSel=0 DScale=0 PME-

07:07.0 PCI Hot-plug controller: Compaq Computer Corporation: Unknown device a0f7 (rev 11)
Subsystem: Intel Corporation: Unknown device 002d
Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
Status: Cap+ 66Mhz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
Interrupt: pin A routed to IRQ 30
Region 0: Memory at f6100000 (32-bit, non-prefetchable) [size=256]
Capabilities: [58] Message Signalled Interrupts: 64bit+ Queue=0/0 Enable-
Address: 0000000000000000 Data: 0000
Capabilities: [68] #07 [0000]

00000000-0009ebff : System RAM
0009ec00-0009ffff : reserved
000a0000-000bffff : Video RAM area
000c0000-000c7fff : Video ROM
000c8000-000c8fff : Extension ROM
000c9000-000cbfff : Extension ROM
000f0000-000fffff : System ROM
00100000-e5feffff : System RAM
00100000-0024bdbf : Kernel code
0024bdc0-0026167f : Kernel data
e5ff0000-e5fffbff : ACPI Tables
e5fffc00-e5ffffff : ACPI Non-volatile Storage
e6000000-e60fffff : Intel Corporation 82557 [Ethernet Pro 100] e6100000-e6100fff : Intel Corporation 82557 [Ethernet Pro 100]
e6100000-e6100fff : eepro100
e6101000-e6101fff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC] e6102000-e6102fff : PCI device 1166:0220 (ServerWorks) e7000000-e7ffffff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC] ea800000-ea8000ff : Compaq Computer Corporation PCI Hotplug Controller ea900000-ea9fffff : PCI Bus #04
f0000000-f0001fff : PCI device 1069:0050 (Mylex Corporation) f6000000-f60fffff : PCI Bus #04
f6100000-f61000ff : Compaq Computer Corporation PCI Hotplug Controller (#2) fec00000-fec0ffff : reserved
fee00000-fee00fff : reserved
ffc00000-ffffffff : reserved

0000-001f : dma1
0020-003f : pic1
0040-005f : timer
0060-006f : keyboard
0080-008f : dma page reg
00a0-00bf : pic2
00c0-00df : dma2
00f0-00ff : fpu
01f0-01f7 : ide0
02f8-02ff : serial(auto)
03c0-03df : vga+
03f6-03f6 : ide0
03f8-03ff : serial(auto)
0cf8-0cff : PCI conf1
1000-10ff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC] 1400-143f : Intel Corporation 82557 [Ethernet Pro 100]
1400-143f : eepro100
1440-144f : PCI device 1166:0211 (ServerWorks)
1450-1453 : PCI device 1166:0211 (ServerWorks)
1458-145f : PCI device 1166:0211 (ServerWorks)
7000-7fff : PCI Bus #04

this is not how the NMI oopser works. It does *not* shut down the console
- you can still see everything in 'dmesg'. If you want to see it on the console again, you can do 'dmesg -n 8'. Adding a 'once per reboot' restriction is unreasonable - there are user-space applications that can be terminated via the NMI-oopser safely. In 99% of the cases it's the first oops that counts, and most people do not have serial console set up, so writing the first oops down from screen is important.

Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

The patch addresses that - it still calls do_exit() for every NMI timeout.

Good point. Problem is, we're getting some reported lockups in which we need to know what the other CPUs are doing (the other 1%). If a critical spinlock is stuck on, we don't get to type `dmesg'.

So... In this updated patch the console_silent() call remains as you designed it, but the nmi_watchdog kernel boot parameter has been overloaded so that

nmi_watchdog=2

will now cause _all_ NMI oops messages to be displayed on the console.

Is this OK?

@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \

@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 

@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c

@@ -260,6 +261,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */

@@ -308,7 +311,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 

@@ -77,17 +77,40 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
+ */
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give syslogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+/*
+ * Called from the BUG() macro.
  */
-void bust_spinlocks(void)
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);

@@ -264,8 +287,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else

@@ -283,6 +305,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*

@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {

@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }

@@ -414,7 +415,7 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
 
 	/*

@@ -437,12 +438,15 @@
 			 * We are in trouble anyway, lets at least try
 			 * to get a message out.
 			 */
-			bust_spinlocks();
+			bust_spinlocks(1);
 			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
 			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
+			if (nmi_watchdog <= 1) {
+				printk("Console shuts up.  Use `dmesg' or boot with `nmi_watchdog=2' to see more logs.\n");
+				console_silent();
+			}
 			spin_unlock(&nmi_print_lock);
+			bust_spinlocks(0);
 			do_exit(SIGSEGV);
 		}
 	} else {

@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);

@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);

@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 

@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
 

@@ -24,8 +24,21 @@
 cannot even accept NMI interrupts, or the crash has made the kernel
 unable to print messages.
 
-NOTE: currently the NMI-oopser is enabled unconditionally on x86 SMP
-boxes.
+The NMI-oopser defaults to `1' (enabled) on x86 SMP boxes.
+
+The NMI-oopser only displays the first oops message on the console. This
+is to prevent vital information from scrolling off the scren if more
+errors occur.  You can find additional information (such as NMI oops
+information from other CPUs) by looking in your system logs or with
+the `dmesg' command.  
+
+If your system is crashing hard and preventing you from running `dmesg' 
+you can tell the NMI oopser to display all NMI oops messages (not just
+the first one) by setting
+
+	nmi_watchdog=2
+
+on the kernel boot command line or in lilo.conf, as shown above.
 
 [ feel free to send bug reports, suggestions and patches to
   Ingo Molnar <[email protected]> or the Linux SMP mailing

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

i know - but we cannot please everyone, so i went for the 99% :-)

this collides with the UP-IOAPIC path's use of nmi_watchdog == 2 ...

i'd rather suggest a cleaner, "nmi_watchdog=2,verbose" (default: silent) type of boot parameter, it looks like the NMI watchdog needs more parameters.

Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

OK, so the options are now:

nmi_watchdog=0
nmi_watchdog=1
nmi_watchdog=1,verbose

Third time lucky?

@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \

@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 

@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c

@@ -251,6 +252,9 @@
 	return do_syslog(type, buf, len);
 }
 
+/*
+ * printk("") may be used to wake up klogd
+ */
 asmlinkage int printk(const char *fmt, ...)
 {
 	va_list args;

@@ -260,6 +264,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */

@@ -308,7 +314,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 

@@ -77,17 +77,37 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
  */
-void bust_spinlocks(void)
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give klogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);

@@ -264,8 +284,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else

@@ -283,6 +302,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*

@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {

@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }

@@ -386,10 +387,14 @@
 #if CONFIG_X86_IO_APIC
 
 int nmi_watchdog = 1;
+static int nmi_watchdog_verbose;
 
 static int __init setup_nmi_watchdog(char *str)
 {
-        get_option(&str, &nmi_watchdog);
+ 	if (get_option(&str, &nmi_watchdog) == 2) {
+		if (strcmp(str, "verbose") == 0)
+			nmi_watchdog_verbose = 1;
+	}
         return 1;
 }
 

@@ -414,7 +419,7 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
 
 	/*

@@ -437,12 +442,15 @@
 			 * We are in trouble anyway, lets at least try
 			 * to get a message out.
 			 */
-			bust_spinlocks();
+			bust_spinlocks(1);
 			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
 			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
+			if (nmi_watchdog_verbose == 0) {
+				printk("Console shuts up. Run dmesg or boot with `nmi_watchdog=1,verbose' for more logs\n");
+				console_silent();
+			}
 			spin_unlock(&nmi_print_lock);
+			bust_spinlocks(0);
 			do_exit(SIGSEGV);
 		}
 	} else {

@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);

@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);

@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 

@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
 

@@ -24,8 +24,21 @@
 cannot even accept NMI interrupts, or the crash has made the kernel
 unable to print messages.
 
-NOTE: currently the NMI-oopser is enabled unconditionally on x86 SMP
-boxes.
+The NMI-oopser defaults to `1' (enabled) on x86 SMP boxes.
+
+The NMI-oopser only displays the first oops message on the console. This
+is to prevent vital information from scrolling off the scren if more
+errors occur.  You can find additional information (such as NMI oops
+information from other CPUs) by looking in your system logs or with
+the `dmesg' command.  
+
+If your system is crashing hard and preventing you from running `dmesg' 
+you can tell the NMI oopser to display all NMI oops messages (not just
+the first one) by setting
+
+	nmi_watchdog=1,verbose
+
+on the kernel boot command line or in lilo.conf, as shown above.
 
 [ feel free to send bug reports, suggestions and patches to
   Ingo Molnar <[email protected]> or the Linux SMP mailing

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

looks good for me!

Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Thank you. Quad 700 meg Xeons? Nice machine, that.

The NMI oops changes are working well. We can see three CPUs are stuck in different places spinning on the same lock with interrupts disabled, but unfortunately something went wrong with your `ksymoops' run.

Could you please rerun ksymoops, and make sure that you use the correct symbol file? Take the `System.map' from the directory where you built the kernel and run

ksymoops -m System.map < log_file

-
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Yes. 54 seconds for compiling kernel is quite good result :-)

In attachment. But don't beat me. I think I found the oops reason. /dev/shm was not mounted. After mounting it I couldn't get an oops yet. Could this be a reason?

regards and thanks for help
pkot

mailto:[email protected]
http://urtica.linuxnews.pl/~pkot/
http://newsreader.linuxnews.pl/

ksymoops 2.3.4 on i686 2.4.0-test11. Options used
-V (default)
-k /proc/ksyms (default)
-l /proc/modules (default)
-o /lib/modules/2.4.0-test11/ (default) -m /usr/src/linux/System.map (specified)

Error (regular_file): read_ksyms stat /proc/ksyms failed No modules in ksyms, skipping objects
No ksyms, skipping lsmod

CPU:    0
EIP:    0010:[<c010b5ee>]

Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00000097
eax: c0285420 ebx: f6f08000 ecx: 00000000 edx: bf1fff2c
esi: bf1fff2c edi: f6f09fa8 ebp: f6f09fa8 esp: f6f09f90
ds: 0018 es: 0018 ss: 0018
Process mysqld (pid: 2386, stackpage=f6f09000)

Stack: c02026f7 f6f08000 00000000 bf1fff24 fffffff2 00000000 f6f09fbc c0120404 
       00000952 00000000 bf1fff28 bf1ffe34 c010c66b 00000952 00000000 bf1fff28 
       00000000 bf1fff24 bf1ffe34 0000009c 0000002b 0000002b 0000009c 081313e4 
Call Trace: [<c02026f7>] [<c0120404>] [<c010c66b>] 
Code: 78 fb f0 ff 08 78 f3 c3 90 90 90 90 90 90 90 90 90 90 55 57 

Trace; c02026f7 <stext_lock+86b/8c14>
Trace; c0120404 <sys_sched_setscheduler+14/18> Trace; c010c66b <system_call+33/38>
Code; c010b5ee <__read_lock_failed+6/18> 00000000 <_EIP>:
Code; c010b5ee <__read_lock_failed+6/18> <=====
0: 78 fb js fffffffd <_EIP+0xfffffffd> c010b5eb <__read_lock_failed+3/18> <=====
Code; c010b5f0 <__read_lock_failed+8/18>
2: f0 ff 08 lock decl (%eax)
Code; c010b5f3 <__read_lock_failed+b/18>
5: 78 f3 js fffffffa <_EIP+0xfffffffa> c010b5e8 <__read_lock_failed+0/18>
Code; c010b5f5 <__read_lock_failed+d/18>
7: c3 ret
Code; c010b5f6 <__read_lock_failed+e/18>
8: 90 nop
Code; c010b5f7 <__read_lock_failed+f/18>
9: 90 nop
Code; c010b5f8 <__read_lock_failed+10/18>
a: 90 nop
Code; c010b5f9 <__read_lock_failed+11/18>
b: 90 nop
Code; c010b5fa <__read_lock_failed+12/18>
c: 90 nop
Code; c010b5fb <__read_lock_failed+13/18>
d: 90 nop
Code; c010b5fc <__read_lock_failed+14/18>
e: 90 nop
Code; c010b5fd <__read_lock_failed+15/18>
f: 90 nop
Code; c010b5fe <__read_lock_failed+16/18>
10: 90 nop
Code; c010b5ff <__read_lock_failed+17/18>
11: 90 nop
Code; c010b600 <copy_siginfo_to_user+0/a8>
12: 55 push %ebp
Code; c010b601 <copy_siginfo_to_user+1/a8>
13: 57 push %edi

NMI Watchdog detected LOCKUP on CPU1, registers:

CPU:    1
EIP:    0010:[<c0202590>]
EFLAGS: 00000082
eax: c028e7ec   ebx: 00000282   ecx: c4df0000   edx: c028e7ec
esi: c011f550   edi: 00000000   ebp: c4d59ef4   esp: c4d59ef0
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d59000)
Stack: c4df0000 c0285c40 c0128592 c4df0000 00000000 00000020 00000000 c0285c40 
       f70082a0 f7c06800 00000086 c0124de5 c02a1220 00000020 c0124cc7 00000000 
       00000001 c0286040 00000020 0000000e c0124b6c c0286040 c02a1580 c0283800 
Call Trace: [<c0128592>] [<c0124de5>] [<c0124cc7>] [<c0124b6c>] [<c010e225>] [<c010a810>] [<c010a810>] 
       [<c010c72c>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a83d>] [<c010a8a2>] [<c0116c84>] [<c0124b6c>] 
       [<c010e225>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 ba cf f1 ff 80 bb 74 06 

Trace; c0128592 <timer_bh+256/2b0>
Trace; c0124de5 <bh_action+4d/b0>
Trace; c0124cc7 <tasklet_hi_action+4f/7c>
Trace; c0124b6c <do_softirq+5c/8c>
Trace; c010e225 <do_IRQ+e5/f4>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c010c72c <ret_from_intr+0/20>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a83d <default_idle+2d/34>
Trace; c010a8a2 <cpu_idle+3e/54>
Trace; c0116c84 <start_secondary+20/24>
Trace; c0124b6c <do_softirq+5c/8c>
Trace; c010e225 <do_IRQ+e5/f4>
Code; c0202590 <stext_lock+704/8c14>
00000000 <_EIP>:
Code; c0202590 <stext_lock+704/8c14> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c0202597 <stext_lock+70b/8c14>
7: f3 90 repz nop
Code; c0202599 <stext_lock+70d/8c14>
9: 7e f5 jle 0 <_EIP>
Code; c020259b <stext_lock+70f/8c14>
b: e9 ba cf f1 ff jmp fff1cfca <_EIP+0xfff1cfca> c011f55a <process_timeout+a/60>
Code; c02025a0 <stext_lock+714/8c14>
10: 80 bb 74 06 00 00 00 cmpb $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU2, registers:

CPU:    2
EIP:    0010:[<c02025c0>]
EFLAGS: 00000082
eax: 00000000   ebx: c0285480   ecx: 00000040   edx: 00000002
esi: c4d7a000   edi: c010a810   ebp: c4d7bfac   esp: c4d7bf78
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d7b000)
Stack: c010a810 c4d7a000 c010a810 c4d7a000 c4d7a000 c010a810 00000000 00000000 
       c4d70018 c0100018 00000002 c4d7a000 c0285480 00000000 c010a8ae 00000003 
       00000000 00000000 c0116c84 c0289373 c012293f 00000003 00000602 00000000 
Call Trace: [<c010a810>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a8ae>] [<c0116c84>] [<c012293f>] 
       [<c0117204>] [<c020de7d>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 b9 d1 f1 ff 80 bb 74 06 

Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a8ae <cpu_idle+4a/54>
Trace; c0116c84 <start_secondary+20/24> Trace; c012293f <printk+18f/19c>
Trace; c0117204 <do_boot_cpu+544/6b0> Trace; c020de7d <call_spurious_interrupt+2a1d/77c8> Code; c02025c0 <stext_lock+734/8c14> 00000000 <_EIP>:
Code; c02025c0 <stext_lock+734/8c14> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c02025c7 <stext_lock+73b/8c14>
7: f3 90 repz nop
Code; c02025c9 <stext_lock+73d/8c14>
9: 7e f5 jle 0 <_EIP>
Code; c02025cb <stext_lock+73f/8c14>
b: e9 b9 d1 f1 ff jmp fff1d1c9 <_EIP+0xfff1d1c9> c011f789 <schedule+b1/648>
Code; c02025d0 <stext_lock+744/8c14>
10: 80 bb 74 06 00 00 00 cmpb $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU3, registers:

CPU:    3
EIP:    0010:[<c0202724>]
EFLAGS: 00000082
eax: f71d4000   ebx: 00000002   ecx: f71d4000   edx: c4d66400
esi: 00000021   edi: 00000000   ebp: f6f15eb0   esp: f6f15eac
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2385, stackpage=f6f15000)
Stack: f71d4000 00000000 c011c3fa f71d4000 00000021 00000082 c011c4bd 00000021 
       f6f15f04 f71d4000 00040001 00000000 f6f14000 00000021 c011c941 00000021 
       f6f15f04 f71d4000 f6f14000 f71d4000 f6f14000 c0285420 00000021 00000000 
Call Trace: [<c011c3fa>] [<c011c4bd>] [<c011c941>] [<c01f2401>] [<c014d6c9>] [<c013b101>] [<c01237bb>] 
       [<c0123b28>] [<c0123b7e>] [<c010c66b>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 d5 e3 f1 ff 80 3b 00 f3 

Trace; c011c3fa <deliver_signal+4a/88>
Trace; c011c4bd <send_sig_info+85/b0>
Trace; c011c941 <do_notify_parent+c9/e0>
Trace; c01f2401 <unix_release_sock+279/294>
Trace; c014d6c9 <dput+19/154>
Trace; c013b101 <fput+79/dc>
Trace; c01237bb <exit_notify+16b/2ac>
Trace; c0123b28 <do_exit+22c/274>
Trace; c0123b7e <sys_exit+e/10>
Trace; c010c66b <system_call+33/38>
Code; c0202724 <stext_lock+898/8c14>
00000000 <_EIP>:
Code; c0202724 <stext_lock+898/8c14> <=====
0: 80 3d 00 54 28 c0 00 cmpb $0x0,0xc0285400 <=====
Code; c020272b <stext_lock+89f/8c14>
7: f3 90 repz nop
Code; c020272d <stext_lock+8a1/8c14>
9: 7e f5 jle 0 <_EIP>
Code; c020272f <stext_lock+8a3/8c14>
b: e9 d5 e3 f1 ff jmp fff1e3e5 <_EIP+0xfff1e3e5> c0120b09 <wake_up_process+9/60>
Code; c0202734 <stext_lock+8a8/8c14>
10: 80 3b 00 cmpb $0x0,(%ebx)
Code; c0202737 <stext_lock+8ab/8c14>
13: f3 00 00 repz add %al,(%eax)

1 error issued. Results may not be reliable.

No.

Trace; c02026f7 <stext_lock+86b/8c14>
Trace; c0120404 <sys_sched_setscheduler+14/18>

                 ^^^^^^^^^^^^^^^^^^^^^^

This was fixed in test11-pre5. Please retest with that kernel.

Thanks again.

-
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/