stack overflow safe kdump (2.6.15-i386) - private nmi stack
PATCH 5/5 stack overflow safe kdump (2.6.15-i386) - private nmi stack
Use a private NMI stack.
---
diff -urNp linux-2.6.15/arch/i386/kernel/irq.c linux-2.6.15-sov/arch/i386/kernel/irq.c
--- linux-2.6.15/arch/i386/kernel/irq.c 2006-01-03 12:21:10.000000000
+0900
+++ linux-2.6.15-sov/arch/i386/kernel/irq.c 2006-01-16
22:04:07.000000000 +0900
@@ -37,11 +37,6 @@ void ack_bad_irq(unsigned int irq)
/*
* per-CPU IRQ handling contexts (thread information and stack) */
-union irq_ctx {
- struct thread_info tinfo;
- u32 stack[THREAD_SIZE/sizeof(u32)];
-};
-
static union irq_ctx *hardirq_ctx[NR_CPUS];
static union irq_ctx *softirq_ctx[NR_CPUS];
#endif
@@ -154,6 +149,8 @@ void irq_ctx_init(int cpu)
printk("CPU %u irqstacks, hard=%p soft=%p\n",
cpu,hardirq_ctx[cpu],softirq_ctx[cpu]);
+
+ nmi_ctx_init(cpu);
}
void irq_ctx_exit(int cpu)
diff -urNp linux-2.6.15/arch/i386/kernel/traps.c linux-2.6.15-sov/arch/i386/kernel/traps.c
--- linux-2.6.15/arch/i386/kernel/traps.c 2006-01-16 22:00:54.000000000
+0900
+++ linux-2.6.15-sov/arch/i386/kernel/traps.c 2006-01-16
22:05:51.000000000 +0900
@@ -643,6 +643,13 @@ static int dummy_nmi_callback(struct pt_
static nmi_callback_t nmi_callback = dummy_nmi_callback;
+#ifdef CONFIG_4KSTACKS
+/*
+ * per-CPU NMI handling contexts (thread information and stack)
+ */
+static union irq_ctx *nmi_ctx[NR_CPUS];
+#endif /* CONFIG_4KSTACKS */
+
static fastcall unsigned int __do_nmi(struct pt_regs * regs, longerror_code)
{
int cpu;
@@ -657,7 +664,43 @@ static fastcall unsigned int __do_nmi(st
return 0;
}
+#ifdef CONFIG_4KSTACKS
+#define _do_nmi(regs, error_code, nmih) \
+{ \
+ union irq_ctx *curctx, *nmictx; \
+ u32 *isp; \
+ \
+ curctx = (union irq_ctx *) current_thread_info(); \
+ nmictx = nmi_ctx[safe_smp_processor_id()]; \
+ \
+ /* \
+ * This is where we switch to the NMI stack. \
+ */ \
+ if (curctx != nmictx) { \
+ int arg1, arg2, ebx; \
+ \
+ /* build the stack frame on the IRQ stack */ \
+ isp = (u32*) ((char *)nmictx + sizeof(*nmictx)); \
+ if (!virt_addr_valid(curctx)) \
+ nmictx->tinfo.task = NULL; \
+ else \
+ nmictx->tinfo.task = curctx->tinfo.task; \
+ nmictx->tinfo.previous_esp = current_stack_pointer; \
+ \
+ asm volatile ( \
+ " xchgl %%ebx,%%esp \n" \
+ " call "#nmih" \n" \
+ " movl %%ebx,%%esp \n" \
+ : "=a" (arg1), "=d" (arg2), "=b" (ebx) \
+ : "0" (regs), "1" (error_code), "2" (isp) \
+ : "memory", "cc", "ecx" \
+ ); \
+ } else \
+ nmih(regs, error_code); \
+}
+#else /* CONFIG_4KSTACKS */
#define _do_nmi(regs, error_code, nmih) nmih(regs, error_code);
+#endif /* CONFIG_4KSTACKS */
fastcall void do_nmi(struct pt_regs * regs, long error_code) {
@@ -696,6 +739,46 @@ void set_crash_nmi_callback(nmi_callback
}
EXPORT_SYMBOL_GPL(set_crash_nmi_callback);
+#ifdef CONFIG_4KSTACKS
+
+/*
+ * These should really be __section__(".bss.page_aligned") as well, but
+ * gcc's 3.0 and earlier don't handle that correctly.
+ * NOTE: support for gcc 3.0 and earlier will eventually be dropped
+ */
+
+static char nmi_stack[NR_CPUS * THREAD_SIZE]
+ __attribute__((__aligned__(THREAD_SIZE)));
+
+/*
+ * Allocate per-cpu stacks for NMI processing
+ */
+void nmi_ctx_init(int cpu)
+{
+ union irq_ctx *irqctx;
+
+ if (nmi_ctx[cpu])
+ return;
+
+ irqctx = (union irq_ctx*) &nmi_stack[cpu*THREAD_SIZE];
+ irqctx->tinfo.task = NULL;
+ irqctx->tinfo.exec_domain = NULL;
+ irqctx->tinfo.cpu = cpu;
+ irqctx->tinfo.preempt_count = HARDIRQ_OFFSET;
+ irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
+
+ nmi_ctx[cpu] = irqctx;
+
+ printk("CPU %u nmistack=%p\n", cpu, nmi_ctx[cpu]);
+}
+
+void nmi_ctx_exit(int cpu)
+{
+ nmi_ctx[cpu] = NULL;
+}
+
+#endif /* CONFIG_4KSTACKS */
+
#ifdef CONFIG_KPROBES
fastcall void __kprobes do_int3(struct pt_regs *regs, long error_code) {
diff -urNp linux-2.6.15/include/asm-i386/irq.h linux-2.6.15-sov/include/asm-i386/irq.h
--- linux-2.6.15/include/asm-i386/irq.h 2006-01-03 12:21:10.000000000
+0900
+++ linux-2.6.15-sov/include/asm-i386/irq.h 2006-01-16
22:04:07.000000000 +0900
@@ -28,12 +28,21 @@ extern void release_vm86_irqs(struct tas
#endif
#ifdef CONFIG_4KSTACKS
- extern void irq_ctx_init(int cpu);
- extern void irq_ctx_exit(int cpu);
+union irq_ctx {
+ struct thread_info tinfo;
+ u32 stack[THREAD_SIZE/sizeof(u32)];
+};
+
+extern void irq_ctx_init(int cpu);
+extern void irq_ctx_exit(int cpu);
+extern void nmi_ctx_init(int cpu);
+extern void nmi_ctx_exit(int cpu);
# define __ARCH_HAS_DO_SOFTIRQ
#else
# define irq_ctx_init(cpu) do { } while (0)
# define irq_ctx_exit(cpu) do { } while (0)
+# define nmi_ctx_init(cpu) do { } while (0)
+# define nmi_ctx_exit(cpu) do { } while (0)
#endif #ifdef CONFIG_IRQBALANCE
Quoted 37 lines · Fernando Luis Vazquez Cao
Use a private NMI stack.
---
diff -urNplinux-2.6.15/arch/i386/kernel/irq.clinux-2.6.15-sov/arch/i386/kernel/irq.c
---linux-2.6.15/arch/i386/kernel/irq.c2006-01-03 12:21:10.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/irq.c2006-01-16
22:04:07.000000000 +0900(quoted diff) +2
-5@@ -37,11 +37,6 @@ void ack_bad_irq(unsigned int irq) /* * per-CPU IRQ handling contexts (thread information and stack) */ -union irq_ctx { - struct thread_info tinfo; - u32 stack[THREAD_SIZE/sizeof(u32)]; -}; - static union irq_ctx *hardirq_ctx[NR_CPUS]; static union irq_ctx *softirq_ctx[NR_CPUS]; #endif@@ -154,6 +149,8 @@ void irq_ctx_init(int cpu) printk("CPU %u irqstacks, hard=%p soft=%p\n", cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); + + nmi_ctx_init(cpu); } void irq_ctx_exit(int cpu)diff -urNp
linux-2.6.15/arch/i386/kernel/traps.clinux-2.6.15-sov/arch/i386/kernel/traps.c
---linux-2.6.15/arch/i386/kernel/traps.c2006-01-16 22:00:54.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/traps.c2006-01-16
22:05:51.000000000 +0900
Review static int dummy_nmi_callback(struct pt_ · line 647
Quoted 5 lines
(quoted diff) +2
-0@@ -643,6 +643,13 @@ static int dummy_nmi_callback(struct pt_ static nmi_callback_t nmi_callback = dummy_nmi_callback; +#ifdef CONFIG_4KSTACKS +/*
Does not work for 8K stacks. Also we are switching the stack all the time for NMI. I am not sure if that is really required (performance?).
Can't it be made to work both for 4K and 8K stack. And switch to reserved stack on NMI, only if crash has happened.
Quoted 132 lines
(quoted diff) +5
-0+ * per-CPU NMI handling contexts (thread information and stack) + */ +static union irq_ctx *nmi_ctx[NR_CPUS]; +#endif /* CONFIG_4KSTACKS */ + static fastcall unsigned int __do_nmi(struct pt_regs * regs, long error_code) {(quoted diff) +76
-0int cpu;@@ -657,7 +664,43 @@ static fastcall unsigned int __do_nmi(st return 0; } +#ifdef CONFIG_4KSTACKS +#define _do_nmi(regs, error_code, nmih) \ +{ \ + union irq_ctx *curctx, *nmictx; \ + u32 *isp; \ + \ + curctx = (union irq_ctx *) current_thread_info(); \ + nmictx = nmi_ctx[safe_smp_processor_id()]; \ + \ + /* \ + * This is where we switch to the NMI stack. \ + */ \ + if (curctx != nmictx) { \ + int arg1, arg2, ebx; \ + \ + /* build the stack frame on the IRQ stack */ \ + isp = (u32*) ((char *)nmictx + sizeof(*nmictx)); \ + if (!virt_addr_valid(curctx)) \ + nmictx->tinfo.task = NULL; \ + else \ + nmictx->tinfo.task = curctx->tinfo.task; \ + nmictx->tinfo.previous_esp = current_stack_pointer; \ + \ + asm volatile ( \ + " xchgl %%ebx,%%esp \n" \ + " call "#nmih" \n" \ + " movl %%ebx,%%esp \n" \ + : "=a" (arg1), "=d" (arg2), "=b" (ebx) \ + : "0" (regs), "1" (error_code), "2" (isp) \ + : "memory", "cc", "ecx" \ + ); \ + } else \ + nmih(regs, error_code); \ +} +#else /* CONFIG_4KSTACKS */ #define _do_nmi(regs, error_code, nmih) nmih(regs, error_code); +#endif /* CONFIG_4KSTACKS */ fastcall void do_nmi(struct pt_regs * regs, long error_code) {@@ -696,6 +739,46 @@ void set_crash_nmi_callback(nmi_callback } EXPORT_SYMBOL_GPL(set_crash_nmi_callback); +#ifdef CONFIG_4KSTACKS + +/* + * These should really be __section__(".bss.page_aligned") as well, but + * gcc's 3.0 and earlier don't handle that correctly. + * NOTE: support for gcc 3.0 and earlier will eventually be dropped + */ + +static char nmi_stack[NR_CPUS * THREAD_SIZE] + __attribute__((__aligned__(THREAD_SIZE))); + +/* + * Allocate per-cpu stacks for NMI processing + */ +void nmi_ctx_init(int cpu) +{ + union irq_ctx *irqctx; + + if (nmi_ctx[cpu]) + return; + + irqctx = (union irq_ctx*) &nmi_stack[cpu*THREAD_SIZE]; + irqctx->tinfo.task = NULL; + irqctx->tinfo.exec_domain = NULL; + irqctx->tinfo.cpu = cpu; + irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; + irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); + + nmi_ctx[cpu] = irqctx; + + printk("CPU %u nmistack=%p\n", cpu, nmi_ctx[cpu]); +} + +void nmi_ctx_exit(int cpu) +{ + nmi_ctx[cpu] = NULL; +} + +#endif /* CONFIG_4KSTACKS */ + #ifdef CONFIG_KPROBES fastcall void __kprobes do_int3(struct pt_regs *regs, long error_code) {diff -urNp
linux-2.6.15/include/asm-i386/irq.hlinux-2.6.15-sov/include/asm-i386/irq.h
---linux-2.6.15/include/asm-i386/irq.h2006-01-03 12:21:10.000000000
+0900
+++linux-2.6.15-sov/include/asm-i386/irq.h2006-01-16
22:04:07.000000000 +0900(quoted diff) +11
-2@@ -28,12 +28,21 @@ extern void release_vm86_irqs(struct tas #endif #ifdef CONFIG_4KSTACKS - extern void irq_ctx_init(int cpu); - extern void irq_ctx_exit(int cpu); +union irq_ctx { + struct thread_info tinfo; + u32 stack[THREAD_SIZE/sizeof(u32)]; +}; + +extern void irq_ctx_init(int cpu); +extern void irq_ctx_exit(int cpu); +extern void nmi_ctx_init(int cpu); +extern void nmi_ctx_exit(int cpu); # define __ARCH_HAS_DO_SOFTIRQ #else # define irq_ctx_init(cpu) do { } while (0) # define irq_ctx_exit(cpu) do { } while (0) +# define nmi_ctx_init(cpu) do { } while (0) +# define nmi_ctx_exit(cpu) do { } while (0) #endif #ifdef CONFIG_IRQBALANCE
(Resending without Japanese characters. I apologize for the noise.)
Quoted 39 lines · Vivek Goyal
Quoted 38 lines · Fernando Luis Vazquez Cao
Use a private NMI stack.
---
diff -urNplinux-2.6.15/arch/i386/kernel/irq.clinux-2.6.15-sov/arch/i386/kernel/irq.c
---linux-2.6.15/arch/i386/kernel/irq.c2006-01-03 12:21:10.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/irq.c2006-01-16
22:04:07.000000000 +0900(quoted diff) +2
-5@@ -37,11 +37,6 @@ void ack_bad_irq(unsigned int irq) /* * per-CPU IRQ handling contexts (thread information and stack) */ -union irq_ctx { - struct thread_info tinfo; - u32 stack[THREAD_SIZE/sizeof(u32)]; -}; - static union irq_ctx *hardirq_ctx[NR_CPUS]; static union irq_ctx *softirq_ctx[NR_CPUS]; #endif@@ -154,6 +149,8 @@ void irq_ctx_init(int cpu) printk("CPU %u irqstacks, hard=%p soft=%p\n", cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); + + nmi_ctx_init(cpu); } void irq_ctx_exit(int cpu)diff -urNp
linux-2.6.15/arch/i386/kernel/traps.clinux-2.6.15-sov/arch/i386/kernel/traps.c
---linux-2.6.15/arch/i386/kernel/traps.c2006-01-16 22:00:54.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/traps.c2006-01-16
22:05:51.000000000 +0900(quoted diff) +0
-0@@ -643,6 +643,13 @@ static int dummy_nmi_callback(struct pt_
Quoted 7 lines
Review static int dummy_nmi_callback(struct pt_ · line 647
Quoted 4 lines
(quoted diff) +2
-0static nmi_callback_t nmi_callback = dummy_nmi_callback; +#ifdef CONFIG_4KSTACKS +/*Does not work for 8K stacks. Also we are switching the stack all the time for NMI. I am not sure if that is really required (performance?).
Yes, it does not work for 8K stacks, but this is something premeditated. Since private stacks for interrupts are only used when 4KSTACKS is enabled I felt that to be consistent it should be the same in the NMI's case. Anyway if it is deemed correct (I agree it is desirable) I could implement it.
Regarding the impact in performance, note that when we use 4K stacks we are switching stacks _every_ time an interrupt occurs. I do not see why we should not do the same for NMIs. Specially since the cost of switching stacks is relatively small when compared to the cost of executing the NMI watchdog's handler.
Quoted 2 lines
Can't it be made to work both for 4K and 8K stack. And switch to reserved stack on NMI, only if crash has happened.
Yes, it could be done, but I think it is safer to use a private stack all the time, so that the NMI handler does not contribute to an eventual stack overflow. I would like to avoid the case of the the stack overflowing inside the NMI handler.
Thanks for your comments.
Regards,
Fernando
Quoted 39 lines · Vivek Goyal
Quoted 38 lines · Fernando Luis Vazquez Cao
Use a private NMI stack.
---
diff -urNplinux-2.6.15/arch/i386/kernel/irq.clinux-2.6.15-sov/arch/i386/kernel/irq.c
---linux-2.6.15/arch/i386/kernel/irq.c2006-01-03 12:21:10.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/irq.c2006-01-16
22:04:07.000000000 +0900(quoted diff) +2
-5@@ -37,11 +37,6 @@ void ack_bad_irq(unsigned int irq) /* * per-CPU IRQ handling contexts (thread information and stack) */ -union irq_ctx { - struct thread_info tinfo; - u32 stack[THREAD_SIZE/sizeof(u32)]; -}; - static union irq_ctx *hardirq_ctx[NR_CPUS]; static union irq_ctx *softirq_ctx[NR_CPUS]; #endif@@ -154,6 +149,8 @@ void irq_ctx_init(int cpu) printk("CPU %u irqstacks, hard=%p soft=%p\n", cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); + + nmi_ctx_init(cpu); } void irq_ctx_exit(int cpu)diff -urNp
linux-2.6.15/arch/i386/kernel/traps.clinux-2.6.15-sov/arch/i386/kernel/traps.c
---linux-2.6.15/arch/i386/kernel/traps.c2006-01-16 22:00:54.000000000
+0900
+++linux-2.6.15-sov/arch/i386/kernel/traps.c2006-01-16
22:05:51.000000000 +0900(quoted diff) +0
-0@@ -643,6 +643,13 @@ static int dummy_nmi_callback(struct pt_
Quoted 7 lines
Review static int dummy_nmi_callback(struct pt_ · line 647
Quoted 4 lines
(quoted diff) +2
-0static nmi_callback_t nmi_callback = dummy_nmi_callback; +#ifdef CONFIG_4KSTACKS +/*Does not work for 8K stacks. Also we are switching the stack all the time for NMI. I am not sure if that is really required (performance?).
Yes, it does not work for 8K stacks, but this is something premeditated. Since private stacks for interrupts are only used when 4KSTACKS is enabled I felt that to be consistent it should be the same in the NMI's case. Anyway if it is deemed correct (I agree it is desirable) I could implement it.
Regarding the impact in performance, note that when we use 4K stacks we are switching stacks _every_ time an interrupt occurs. I do not see why we should not do the same for NMIs. Specially since the cost of switching stacks is relatively small when compared to the cost of executing the NMI watchdog's handler.
Quoted 2 lines
Can't it be made to work both for 4K and 8K stack. And switch to reserved stack on NMI, only if crash has happened.
Yes, it could be done, but I think it is safer to use a private stack all the time, so that the NMI handler does not contribute to an eventual stack overflow. I would like to avoid the case of the the stack overflowing inside the NMI handler.
Thanks for your comments.
Regards,
Fernando
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 19 lines · Vivek Goyal
Quoted 18 lines · Fernando Luis Vazquez Cao
Quoted 3 lines · Vivek Goyal
Does not work for 8K stacks. Also we are switching the stack all the time for NMI. I am not sure if that is really required (performance?).
Yes, it does not work for 8K stacks, but this is something premeditated. Since private stacks for interrupts are only used when 4KSTACKS is enabled I felt that to be consistent it should be the same in the NMI's case. Anyway if it is deemed correct (I agree it is desirable) I could implement it.
Regarding the impact in performance, note that when we use 4K stacks we are switching stacks _every_ time an interrupt occurs. I do not see why we should not do the same for NMIs. Specially since the cost of switching stacks is relatively small when compared to the cost of executing the NMI watchdog's handler.
Quoted 2 lines
Can't it be made to work both for 4K and 8K stack. And switch to reserved stack on NMI, only if crash has happened.
Quoted 8 lines
Quoted 4 lines
Yes, it could be done, but I think it is safer to use a private stack all the time, so that the NMI handler does not contribute to an eventual stack overflow. I would like to avoid the case of the the stack overflowing inside the NMI handler.
Ok. Then we are looking at an entirely different problem and that is avoid stack overflows for NMI for 4K stacks in general and not necessarily a crash dump specific code hardening.
Exactly.
Fernando