Skip to contentSkip to conversation

2.6.0 MCA TLB error recovery

PATCH Keith Owens 3 participants 8 messages Dec 19, 2003 – Dec 20, 2003

lore
Share & tools
lore
#0 lore raw
Quoted 3 lines · Tony

It looks like salinfo_log_wakeup() is called right before ia64_log_print() ... so I'm not sure why the salinfo_decode daemon kept on snoozing. Keith: am I missing something obvious?

From the top of salinfo_log_wakeup()

* ... MCA and INIT events are
* not irq safe, do not call any routines that use spinlocks, they may deadlock.

MCA and INIT records are noted but it is not safe to call up() from those interrupts, so the daemon cannot be woken. This has not been a problem in the past because MCA and INIT were not recoverable, the records are picked up on the next boot. Once my patches are in David's tree, I will update salinfo to periodically check for any MCA or INIT records and kick the daemon. There was no point before, I had no way of testing this case.

#1 lore raw
Quoted 9 lines
Quoted 3 lines

It looks like salinfo_log_wakeup() is called right before ia64_log_print() ... so I'm not sure why the salinfo_decode daemon kept on snoozing. Keith: am I missing something obvious?

From the top of salinfo_log_wakeup()

* ... MCA and INIT events are
* not irq safe, do not call any routines that use spinlocks,
they may deadlock.

Okay ... that was pretty danged obvious! Thanks for pointing it out so gently :-)

Quoted 8 lines

MCA and INIT records are noted but it is not safe to call up() from those interrupts, so the daemon cannot be woken. This has not been a problem in the past because MCA and INIT were not recoverable, the records are picked up on the next boot. Once my patches are in David's
tree, I will update salinfo to periodically check for any MCA or INIT records and kick the daemon. There was no point before, I had no way of testing this case.

Sounds good. Salinfo-0.4 is beautiful by the way. Getting all the bits decoded from processor state parameter and ipsr/xpsr etc. is great.

-Tony

#3 lore raw
In reply to Keith Owens
Quoted 3 lines

I didn't keep the old patches around. Could you and Keith send me a consistent set of patches? That would help me.

Here's Keith's first patch, 2nd (much smaller) is in the next e-mail:

From: [email protected] on behalf of Keith Owens [[email protected]]
Sent: Tuesday, November 25, 2003 12:37 AM
To: [email protected]
Subject: [patch] 2.6.0-test9 pal/sal/salinfo/mca

Forward port the recent changes to pal.h, sal.h, mca.h, salinfo.c and mca.c from 2.4.23-rc2 to 2.6.0-test9.

This converts 2.6 to use salinfo instead of printing CMC/CPE/MCA/INIT records in the kernel. It makes the two kernel versions as close together as possible.

Index: 9.2/include/asm-ia64/sal.h

include/asm-ia64/sal.h Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/u/c/36_sal.h 1.4 644) renamed from include/asm-ia64/sal.h Sun, 24 Aug 2003 10:59:40 +1000 kaos (linux-2.6-test/u/c/36_sal.h 1.3 644) +7 -3
@@ -725,14 +725,16 @@ ia64_sal_mc_rendez (void)
  * Allow the OS to specify the interrupt number to be used by SAL to interrupt OS during
  * the machine check rendezvous sequence as well as the mechanism to wake up the
  * non-monarch processor at the end of machine check processing.
+ * Returns the complete ia64_sal_retval because some calls return more than just a status
+ * value.
  */
-static inline s64
+static inline struct ia64_sal_retval
 ia64_sal_mc_set_params (u64 param_type, u64 i_or_m, u64 i_or_m_val, u64 timeout, u64 rz_always)
 {
 	struct ia64_sal_retval isrv;
 	SAL_CALL(isrv, SAL_MC_SET_PARAMS, param_type, i_or_m, i_or_m_val,
 		 timeout, rz_always, 0, 0);
-	return isrv.status;
+	return isrv;
 }
 
 /* Read from PCI configuration space */
@@ -804,10 +806,12 @@ ia64_sal_update_pal (u64 param_buf, u64 
 
 extern unsigned long sal_platform_features;
 
+extern int (*salinfo_platform_oemdata)(const u8 *, u8 **, u64 *);
+
 struct sal_ret_values {
 	long r8; long r9; long r10; long r11;
 };
 
 #endif /* __ASSEMBLY__ */
 
-#endif /* _ASM_IA64_PAL_H */
+#endif /* _ASM_IA64_SAL_H */

Index: 9.2/include/asm-ia64/pal.h

include/asm-ia64/pal.h Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/v/c/12_pal.h 1.4 644) renamed from include/asm-ia64/pal.h Mon, 20 Oct 2003 11:16:44 +1000 kaos (linux-2.6-test/v/c/12_pal.h 1.3 644) +135 -25

209 lines not on this page · raw message

Index: 9.2/include/asm-ia64/mca.h

include/asm-ia64/mca.h Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/w/c/12_mca.h 1.4 644) renamed from include/asm-ia64/mca.h Mon, 20 Oct 2003 11:16:44 +1000 kaos (linux-2.6-test/w/c/12_mca.h 1.3 644) +0 -1
@@ -141,7 +141,6 @@ extern irqreturn_t ia64_mca_cpe_int_call
 extern int  ia64_log_print(int,prfunc_t);
 extern void ia64_mca_cmc_vector_setup(void);
 extern int  ia64_mca_check_errors(void);
-extern u64  ia64_log_get(int, prfunc_t);
 
 #define PLATFORM_CALL(fn, args)	printk("Platform call TBD\n")
 

Index: 9.2/arch/ia64/Kconfig

arch/ia64/Kconfig Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/F/e/0_Kconfig 1.7 644) renamed from arch/ia64/Kconfig Tue, 21 Oct 2003 18:45:55 +1000 kaos (linux-2.6-test/F/e/0_Kconfig 1.6 644) +0 -10
@@ -394,16 +394,6 @@ config IA64_PALINFO
 	  To use this option, you have to ensure that the "/proc file system
 	  support" (CONFIG_PROC_FS) is enabled, too.
 
-config IA64_SALINFO
-	tristate "/proc/sal support"
-	help
-	  The /proc/sal directory exports the SAL (system abstraction layer)
-	  feature bits, like whether the platform is subject to ITC drift.  It
-	  is intended to be used by user programs that care about such things.
-
-	  To use this option, you have to ensure that the "/proc file system
-	  support" (CONFIG_PROC_FS) is enabled, too.
-
 config EFI_VARS
 	tristate "/proc/efi/vars support"
 	help

Index: 9.2/arch/ia64/kernel/Makefile

arch/ia64/kernel/Makefile Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/I/e/3_Makefile 1.4 644) renamed from arch/ia64/kernel/Makefile Mon, 29 Sep 2003 13:13:55 +1000 kaos (linux-2.6-test/I/e/3_Makefile 1.3 644) +1 -2
@@ -6,7 +6,7 @@ extra-y	:= head.o init_task.o vmlinux.ld
 
 obj-y := acpi.o entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o	\
 	 irq_lsapic.o ivt.o machvec.o pal.o patch.o process.o perfmon.o ptrace.o sal.o		\
-	 semaphore.o setup.o signal.o sys_ia64.o time.o traps.o unaligned.o unwind.o
+	 salinfo.o semaphore.o setup.o signal.o sys_ia64.o time.o traps.o unaligned.o unwind.o
 
 obj-$(CONFIG_EFI_VARS)		+= efivars.o
 obj-$(CONFIG_IA64_BRL_EMU)	+= brl_emu.o
@@ -14,7 +14,6 @@ obj-$(CONFIG_IA64_GENERIC)	+= acpi-ext.o
 obj-$(CONFIG_IA64_HP_ZX1)	+= acpi-ext.o
 obj-$(CONFIG_IA64_MCA)		+= mca.o mca_asm.o
 obj-$(CONFIG_IA64_PALINFO)	+= palinfo.o
-obj-$(CONFIG_IA64_SALINFO)	+= salinfo.o
 obj-$(CONFIG_IOSAPIC)		+= iosapic.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-$(CONFIG_SMP)		+= smp.o smpboot.o

Index: 9.2/arch/ia64/kernel/salinfo.c

arch/ia64/kernel/salinfo.c Tue, 25 Nov 2003 19:23:21 +1100 kaos (linux-2.6-test/I/e/22_salinfo.c 1.3 644) renamed from arch/ia64/kernel/salinfo.c Mon, 29 Sep 2003 13:13:55 +1000 kaos (linux-2.6-test/I/e/22_salinfo.c 1.2 644) +503 -22

567 lines not on this page · raw message

Index: 9.2/arch/ia64/kernel/mca.c

arch/ia64/kernel/mca.c Tue, 25 Nov 2003 19:30:04 +1100 kaos (linux-2.6-test/I/e/30_mca.c 1.2.1.6 644) renamed from arch/ia64/kernel/mca.c Mon, 20 Oct 2003 11:16:44 +1000 kaos (linux-2.6-test/I/e/30_mca.c 1.2.1.4 644) +60 -40

215 lines not on this page · raw message

-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#4 lore raw
In reply to Keith Owens
Quoted 4 lines

I didn't keep the old patches around. Could you and Keith send me a consistent set of patches? That would help me.

Keith's second fix:

From: [email protected] on behalf of Keith Owens [[email protected]]
Sent: Sunday, December 07, 2003 10:25 PM
To: Alex Williamson
Cc: [email protected]
Subject: Re: [patch] 2.4.23 fix deadlock in ia64_mca_cmc_int_caller

Quoted 2 lines · Alex Williamson

Looks good to me, only comment I have would be to tack the below
chunk into ia64_mca_late_init().

Duh, how did I miss ia64_mca_init? Take 2.

smp_call_function() must not be called from interrupt context (can deadlock on tasklist_lock). Use keventd to call smp_call_function().

Index: 23.5/arch/ia64/kernel/mca.c

arch/ia64/kernel/mca.c Mon, 08 Dec 2003 17:23:55 +1100 kaos (linux-2.4/s/c/5_mca.c 1.1.3.2.3.1.1.1.1.2.1.1.1.1.1.5 644) renamed from arch/ia64/kernel/mca.c Tue, 18 Nov 2003 16:26:06 +1100 kaos (linux-2.4/s/c/5_mca.c 1.1.3.2.3.1.1.1.1.2.1.1.1.1.1.5 644) +42 -21
@@ -36,6 +36,10 @@
  *                      SAL 3.0 spec.
  * 00/03/29 C. Fleckenstein  Fixed PAL/SAL update issues, began MCA bug fixes, logging issues,
  *                           added min save state dump, added INIT handler.
+ *
+ * 2003-12-08 Keith Owens <[email protected]>
+ *            smp_call_function() must not be called from interrupt context (can
+ *            deadlock on tasklist_lock).  Use keventd to call smp_call_function().
  */
 #include <linux/config.h>
 #include <linux/types.h>
@@ -50,6 +54,7 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
+#include <linux/tqueue.h>
 
 #include <asm/delay.h>
 #include <asm/machvec.h>
@@ -154,6 +159,8 @@ static int cpe_poll_enabled = 1;
 
 extern void salinfo_log_wakeup(int type, u8 *buffer, u64 size);
 
+static struct tq_struct	cmc_disable_tq, cmc_enable_tq;
+
 /*
  *  ia64_mca_log_sal_error_record
  *
@@ -626,6 +633,36 @@ verify_guid (efi_guid_t *test, efi_guid_
 }
 
 /*
+ * ia64_mca_cmc_vector_disable_keventd
+ *
+ * Called via keventd (smp_call_function() is not safe in interrupt context) to
+ * disable the cmc interrupt vector.
+ *
+ * Note: needs preempt_disable() if you apply the preempt patch to 2.4.
+ */
+static void
+ia64_mca_cmc_vector_disable_keventd(void *unused)
+{
+	ia64_mca_cmc_vector_disable(NULL);
+	smp_call_function(ia64_mca_cmc_vector_disable, NULL, 1, 0);
+}
+
+/*
+ * ia64_mca_cmc_vector_enable_keventd
+ *
+ * Called via keventd (smp_call_function() is not safe in interrupt context) to
+ * enable the cmc interrupt vector.
+ *
+ * Note: needs preempt_disable() if you apply the preempt patch to 2.4.
+ */
+static void
+ia64_mca_cmc_vector_enable_keventd(void *unused)
+{
+	smp_call_function(ia64_mca_cmc_vector_enable, NULL, 1, 0);
+	ia64_mca_cmc_vector_enable(NULL);
+}
+
+/*
  * ia64_mca_init
  *
  *  Do all the system level mca specific initialization.
@@ -658,6 +695,9 @@ ia64_mca_init(void)
 
 	IA64_MCA_DEBUG("ia64_mca_init: begin\n");
 
+	INIT_TQUEUE(&cmc_disable_tq, ia64_mca_cmc_vector_disable_keventd, NULL);
+	INIT_TQUEUE(&cmc_enable_tq, ia64_mca_cmc_vector_enable_keventd, NULL);
+
 	/* initialize recovery success indicator */
 	ia64_os_mca_recovery_successful = 0;
 
@@ -1062,14 +1102,7 @@ ia64_mca_cmc_int_handler(int cmc_irq, vo
 
 			cmc_polling_enabled = 1;
 			spin_unlock(&cmc_history_lock);
-
-			/*
-			 * We rely on the local_irq_enable() above so
-			 * that this can't deadlock.
-			 */
-			ia64_mca_cmc_vector_disable(NULL);
-
-			smp_call_function(ia64_mca_cmc_vector_disable, NULL, 1, 0);
+			schedule_task(&cmc_disable_tq);
 
 			/*
 			 * Corrected errors will still be corrected, but
@@ -1163,19 +1196,7 @@ ia64_mca_cmc_int_caller(int cpe_irq, voi
 		if (start_count = IA64_LOG_COUNT(SAL_INFO_TYPE_CMC)) {
 
 			printk(KERN_WARNING "%s: Returning to interrupt driven CMC handler\n", __FUNCTION__);
-
-			/*
-			 * The cmc interrupt handler enabled irqs, so
-			 * this can't deadlock.
-			 */
-			smp_call_function(ia64_mca_cmc_vector_enable, NULL, 1, 0);
-
-			/*
-			 * Turn off interrupts before re-enabling the
-			 * cmc vector locally.  Make sure we get out.
-			 */
-			local_irq_disable();
-			ia64_mca_cmc_vector_enable(NULL);
+			schedule_task(&cmc_enable_tq);
 			cmc_polling_enabled = 0;
 
 		} else {

-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#5 lore raw
In reply to Keith Owens
Quoted 9 lines · Tony

DavidM wroe

Quoted 3 lines

I didn't keep the old patches around. Could you and Keith send me a consistent set of patches? That would help me.

Keith's second fix:
smp_call_function() must not be called from interrupt context (can deadlock on tasklist_lock). Use keventd to call smp_call_function().
Index: 23.5/arch/ia64/kernel/mca.c

Do not apply this patch to 2.6, it is for 2.4 only. 2.6 uses different structure and function names, plus the patch needs tweaking for preempt.

David, please apply the first mca/salinfo patch to your tree (Forward port the recent changes to pal.h, sal.h, mca.h, salinfo.c and mca.c from 2.4.23-rc2 to 2.6.0-test9), then I will do a 2.6 version of the second patch over the top.

#6 lore raw
In reply to Keith Owens
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> David, please apply the first mca/salinfo patch to your tree Keith> (Forward port the recent changes to pal.h, sal.h, mca.h, Keith> salinfo.c and mca.c from 2.4.23-rc2 to 2.6.0-test9), then I Keith> will do a 2.6 version of the second patch over the top.

Whoops, too late. I did see the preempt comments and figured you'll fix that once it's in the tree. Can you go from what's in the bk trees now (give me a minute or two, I'm just pushing things to to-linus-2.5 and linux-ia64-2.5 will be next).

	--david
#7 lore raw
In reply to Keith Owens
Quoted 11 lines · David Mosberger
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Keith Owens

Keith> David, please apply the first mca/salinfo patch to your tree Keith> (Forward port the recent changes to pal.h, sal.h, mca.h, Keith> salinfo.c and mca.c from 2.4.23-rc2 to 2.6.0-test9), then I Keith> will do a 2.6 version of the second patch over the top.

Whoops, too late. I did see the preempt comments and figured you'll fix that once it's in the tree. Can you go from what's in the bk trees now (give me a minute or two, I'm just pushing things to to-linus-2.5 and linux-ia64-2.5 will be next).

Convert cmc deadlock avoidance patch from 2.4 to 2.6.

arch/ia64/kernel/mca.c renamed from arch/ia64/kernel/mca.c.orig +8 -16
@@ -55,7 +55,7 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
-#include <linux/tqueue.h>
+#include <linux/workqueue.h>
 
 #include <asm/delay.h>
 #include <asm/machvec.h>
@@ -158,8 +158,6 @@
 
 extern void salinfo_log_wakeup(int type, u8 *buffer, u64 size);
 
-static struct tq_struct	cmc_disable_tq, cmc_enable_tq;
-
 /*
  *  ia64_mca_log_sal_error_record
  *
@@ -622,14 +620,11 @@
  *
  * Called via keventd (smp_call_function() is not safe in interrupt context) to
  * disable the cmc interrupt vector.
- *
- * Note: needs preempt_disable() if you apply the preempt patch to 2.4.
  */
 static void
 ia64_mca_cmc_vector_disable_keventd(void *unused)
 {
-	ia64_mca_cmc_vector_disable(NULL);
-	smp_call_function(ia64_mca_cmc_vector_disable, NULL, 1, 0);
+	on_each_cpu(ia64_mca_cmc_vector_disable, NULL, 1, 0);
 }
 
 /*
@@ -637,14 +632,11 @@
  *
  * Called via keventd (smp_call_function() is not safe in interrupt context) to
  * enable the cmc interrupt vector.
- *
- * Note: needs preempt_disable() if you apply the preempt patch to 2.4.
  */
 static void
 ia64_mca_cmc_vector_enable_keventd(void *unused)
 {
-	smp_call_function(ia64_mca_cmc_vector_enable, NULL, 1, 0);
-	ia64_mca_cmc_vector_enable(NULL);
+	on_each_cpu(ia64_mca_cmc_vector_enable, NULL, 1, 0);
 }
 
 /*
@@ -680,9 +672,6 @@
 
 	IA64_MCA_DEBUG("ia64_mca_init: begin\n");
 
-	INIT_TQUEUE(&cmc_disable_tq, ia64_mca_cmc_vector_disable_keventd, NULL);
-	INIT_TQUEUE(&cmc_enable_tq, ia64_mca_cmc_vector_enable_keventd, NULL);
-
 	/* initialize recovery success indicator */
 	ia64_os_mca_recovery_successful = 0;
 
@@ -1055,6 +1044,9 @@
 	ia64_return_to_sal_check();
 }
 
+static DECLARE_WORK(cmc_disable_work, ia64_mca_cmc_vector_disable_keventd, NULL);
+static DECLARE_WORK(cmc_enable_work, ia64_mca_cmc_vector_enable_keventd, NULL);
+
 /*
  * ia64_mca_cmc_int_handler
  *
@@ -1101,7 +1093,7 @@
 
 			cmc_polling_enabled = 1;
 			spin_unlock(&cmc_history_lock);
-			schedule_task(&cmc_disable_tq);
+			schedule_work(&cmc_disable_work);
 
 			/*
 			 * Corrected errors will still be corrected, but
@@ -1196,7 +1188,7 @@
 		if (start_count = IA64_LOG_COUNT(SAL_INFO_TYPE_CMC)) {
 
 			printk(KERN_WARNING "%s: Returning to interrupt driven CMC handler\n", __FUNCTION__);
-			schedule_task(&cmc_enable_tq);
+			schedule_work(&cmc_enable_work);
 			cmc_polling_enabled = 0;
 
 		} else {