Skip to contentSkip to conversation

i386: improve double fault handling

PATCH In review Jan Beulich 5 participants 15 messages Jul 18, 2008 – Jul 31, 2008 x86

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 i386: improve double fault handling

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.
Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

---
 arch/x86/kernel/cpu/common.c     |   17 +++++--
 arch/x86/kernel/doublefault_32.c |   86 ++++++++++++++++++++++++---------------
 arch/x86/kernel/smpboot.c        |   44 +++++++++++++++++++
 arch/x86/kernel/traps_32.c       |   51 ++++++++++++++++++++++-
 drivers/lguest/segments.c        |    3 -
 include/asm-x86/kdebug.h         |    1 
 include/asm-x86/processor.h      |    7 ++-
 include/asm-x86/segment.h        |   15 ++++--
 include/asm-x86/thread_info_32.h |    9 +++-
 9 files changed, 187 insertions(+), 46 deletions(-)
arch/x86/kernel/cpu/common.c +13 -4
@@ -650,6 +650,13 @@ void switch_to_new_gdt(void)
 	asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
 }
 
+static void *__init_refok alloc_boot_stack(void)
+{
+	BUILD_BUG_ON(EXCEPTION_STACK_ORDER > THREAD_ORDER);
+	return __alloc_bootmem(EXCEPTION_STACK_SIZE, THREAD_SIZE,
+			       __pa(MAX_DMA_ADDRESS));
+}
+
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
  * initialized (naturally) in the bootstrap process, such as the GDT
@@ -690,10 +697,12 @@ void __cpuinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
-#endif
+	if (cpu == 0) {
+		unsigned i;
+
+		for (i = 0; i < N_EXCEPTION_TSS; ++i)
+			setup_exception_tss(cpu, i, alloc_boot_stack);
+	}
 
 	/* Clear %gs. */
 	asm volatile ("mov %0, %%gs" : : "r" (0));
arch/x86/kernel/doublefault_32.c +53 -33

122 lines not on this page · raw message

arch/x86/kernel/smpboot.c +44 -0

56 lines not on this page · raw message

arch/x86/kernel/traps_32.c +50 -1

69 lines not on this page · raw message

drivers/lguest/segments.c +2 -1
@@ -50,7 +50,8 @@ static int ignored_gdt(unsigned int num)
 	return (num == GDT_ENTRY_TSS
 		|| num == GDT_ENTRY_LGUEST_CS
 		|| num == GDT_ENTRY_LGUEST_DS
-		|| num == GDT_ENTRY_DOUBLEFAULT_TSS);
+		|| (num >= GDT_ENTRY_EXCEPTION_TSS
+		    && num < GDT_ENTRY_EXCEPTION_TSS + N_EXCEPTION_TSS));
 }
 
 /*H:630 Once the Guest gave us new GDT entries, we fix them up a little.  We
include/asm-x86/kdebug.h +1 -0
@@ -20,6 +20,7 @@ enum die_val {
 	DIE_CALL,
 	DIE_NMI_IPI,
 	DIE_PAGE_FAULT,
+	DIE_DOUBLE_FAULT,
 	DIE_NMIUNKNOWN,
 };
 
include/asm-x86/processor.h +6 -1
@@ -128,7 +128,6 @@ struct cpuinfo_x86 {
 extern struct cpuinfo_x86	boot_cpu_data;
 extern struct cpuinfo_x86	new_cpu_data;
 
-extern struct tss_struct	doublefault_tss;
 extern __u32			cleared_cpu_caps[NCAPINTS];
 
 #ifdef CONFIG_SMP
@@ -841,6 +840,12 @@ static inline void spin_lock_prefetch(co
 	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },	  \
 }
 
+#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STACK_SIZE (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+void __cpuinit setup_exception_tss(unsigned int cpu, unsigned int idx,
+				   void *(*alloc_stack)(void));
+
 extern unsigned long thread_saved_pc(struct task_struct *tsk);
 
 #define THREAD_SIZE_LONGS      (THREAD_SIZE/sizeof(unsigned long))
include/asm-x86/segment.h +11 -4
@@ -55,7 +55,7 @@
  *  28 - unused
  *  29 - unused
  *  30 - unused
- *  31 - TSS for double fault handler
+ *  31+  TSSes for exception handlers
  */
 #define GDT_ENTRY_TLS_MIN	6
 #define GDT_ENTRY_TLS_MAX 	(GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1)
@@ -90,12 +90,19 @@
 #define __KERNEL_PERCPU 0
 #endif
 
-#define GDT_ENTRY_DOUBLEFAULT_TSS	31
+#define GDT_ENTRY_EXCEPTION_TSS	31
+#ifdef CONFIG_DOUBLEFAULT
+#define DOUBLEFAULT_TSS 0
+#define N_EXCEPTION_TSS 1
+#else
+#undef GDT_ENTRY_EXCEPTION_TSS
+#define N_EXCEPTION_TSS 0
+#endif
 
 /*
- * The GDT has 32 entries
+ * The GDT has 31+ entries
  */
-#define GDT_ENTRIES 32
+#define GDT_ENTRIES (31 + N_EXCEPTION_TSS)
 
 /* The PnP BIOS entries in the GDT */
 #define GDT_ENTRY_PNPBIOS_CS32		(GDT_ENTRY_PNPBIOS_BASE + 0)
include/asm-x86/thread_info_32.h +7 -2
@@ -53,9 +53,14 @@ struct thread_info {
 
 #define PREEMPT_ACTIVE		0x10000000
 #ifdef CONFIG_4KSTACKS
-#define THREAD_SIZE            (4096)
+#define THREAD_ORDER 0
 #else
-#define THREAD_SIZE		(8192)
+#define THREAD_ORDER 1
+#endif
+#ifndef __ASSEMBLY__
+#define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER)
+#else
+#define THREAD_SIZE (PAGE_SIZE_asm << THREAD_ORDER)
 #endif
 
 #define STACK_WARN             (THREAD_SIZE/8)
#1 lore raw
Quoted 10 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.
Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

This patch doesn't apply for me to the extent that I'm hesitant to fix it up manually. Could you please refresh it against current -linus?

-hpa

P.S. All your patches came through QP-damaged, which made them more difficult to deal with manually.

#2 lore raw
Quoted 10 lines
Quoted 1 line
Quoted 1 line

"H. Peter Anvin" <[email protected]> 19.07.08 01:24 >>>

Quoted 8 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.
Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

Quoted 5 lines
Quoted 2 lines

This patch doesn't apply for me to the extent that I'm hesitant to fix it up manually. Could you please refresh it against current -linus?

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.
Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

---
 arch/x86/kernel/cpu/common.c     |   17 +++++--
 arch/x86/kernel/doublefault_32.c |   86 ++++++++++++++++++++++++---------------
 arch/x86/kernel/smpboot.c        |   44 +++++++++++++++++++
 arch/x86/kernel/traps_32.c       |   51 ++++++++++++++++++++++-
 drivers/lguest/segments.c        |    3 -
 include/asm-x86/kdebug.h         |    1 
 include/asm-x86/processor.h      |    7 ++-
 include/asm-x86/segment.h        |   15 ++++--
 8 files changed, 180 insertions(+), 44 deletions(-)
arch/x86/kernel/cpu/common.c +13 -4
@@ -650,6 +650,13 @@ void switch_to_new_gdt(void)
 	asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
 }
 
+static void *__init_refok alloc_boot_stack(void)
+{
+	BUILD_BUG_ON(EXCEPTION_STACK_ORDER > THREAD_ORDER);
+	return __alloc_bootmem(EXCEPTION_STACK_SIZE, THREAD_SIZE,
+			       __pa(MAX_DMA_ADDRESS));
+}
+
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
  * initialized (naturally) in the bootstrap process, such as the GDT
@@ -690,10 +697,12 @@ void __cpuinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
-#endif
+	if (cpu == 0) {
+		unsigned i;
+
+		for (i = 0; i < N_EXCEPTION_TSS; ++i)
+			setup_exception_tss(cpu, i, alloc_boot_stack);
+	}
 
 	/* Clear %gs. */
 	asm volatile ("mov %0, %%gs" : : "r" (0));
arch/x86/kernel/doublefault_32.c +53 -33

122 lines not on this page · raw message

arch/x86/kernel/smpboot.c +44 -0

56 lines not on this page · raw message

arch/x86/kernel/traps_32.c +50 -1

69 lines not on this page · raw message

drivers/lguest/segments.c +2 -1
@@ -50,7 +50,8 @@ static int ignored_gdt(unsigned int num)
 	return (num == GDT_ENTRY_TSS
 		|| num == GDT_ENTRY_LGUEST_CS
 		|| num == GDT_ENTRY_LGUEST_DS
-		|| num == GDT_ENTRY_DOUBLEFAULT_TSS);
+		|| (num >= GDT_ENTRY_EXCEPTION_TSS
+		    && num < GDT_ENTRY_EXCEPTION_TSS + N_EXCEPTION_TSS));
 }
 
 /*H:630 Once the Guest gave us new GDT entries, we fix them up a little.  We
include/asm-x86/kdebug.h +1 -0
@@ -20,6 +20,7 @@ enum die_val {
 	DIE_CALL,
 	DIE_NMI_IPI,
 	DIE_PAGE_FAULT,
+	DIE_DOUBLE_FAULT,
 	DIE_NMIUNKNOWN,
 };
 
include/asm-x86/processor.h +6 -1
@@ -128,7 +128,6 @@ struct cpuinfo_x86 {
 extern struct cpuinfo_x86	boot_cpu_data;
 extern struct cpuinfo_x86	new_cpu_data;
 
-extern struct tss_struct	doublefault_tss;
 extern __u32			cleared_cpu_caps[NCAPINTS];
 
 #ifdef CONFIG_SMP
@@ -841,6 +840,12 @@ static inline void spin_lock_prefetch(co
 	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },	  \
 }
 
+#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STACK_SIZE (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+void __cpuinit setup_exception_tss(unsigned int cpu, unsigned int idx,
+				   void *(*alloc_stack)(void));
+
 extern unsigned long thread_saved_pc(struct task_struct *tsk);
 
 #define THREAD_SIZE_LONGS      (THREAD_SIZE/sizeof(unsigned long))
include/asm-x86/segment.h +11 -4
@@ -55,7 +55,7 @@
  *  28 - unused
  *  29 - unused
  *  30 - unused
- *  31 - TSS for double fault handler
+ *  31+  TSSes for exception handlers
  */
 #define GDT_ENTRY_TLS_MIN	6
 #define GDT_ENTRY_TLS_MAX 	(GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1)
@@ -90,12 +90,19 @@
 #define __KERNEL_PERCPU 0
 #endif
 
-#define GDT_ENTRY_DOUBLEFAULT_TSS	31
+#define GDT_ENTRY_EXCEPTION_TSS	31
+#ifdef CONFIG_DOUBLEFAULT
+#define DOUBLEFAULT_TSS 0
+#define N_EXCEPTION_TSS 1
+#else
+#undef GDT_ENTRY_EXCEPTION_TSS
+#define N_EXCEPTION_TSS 0
+#endif
 
 /*
- * The GDT has 32 entries
+ * The GDT has 31+ entries
  */
-#define GDT_ENTRIES 32
+#define GDT_ENTRIES (31 + N_EXCEPTION_TSS)
 
 /* The PnP BIOS entries in the GDT */
 #define GDT_ENTRY_PNPBIOS_CS32		(GDT_ENTRY_PNPBIOS_BASE + 0)
include/asm-x86/thread_info_32.h +7 -2
@@ -53,9 +53,14 @@ struct thread_info {
 
 #define PREEMPT_ACTIVE		0x10000000
 #ifdef CONFIG_4KSTACKS
-#define THREAD_SIZE            (4096)
+#define THREAD_ORDER 0
 #else
-#define THREAD_SIZE		(8192)
+#define THREAD_ORDER 1
+#endif
+#ifndef __ASSEMBLY__
+#define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER)
+#else
+#define THREAD_SIZE (PAGE_SIZE_asm << THREAD_ORDER)
 #endif
 
 #define STACK_WARN             (THREAD_SIZE/8)
#3 lore raw
Quoted 11 lines · Jan Beulich
Quoted 3 lines

This patch doesn't apply for me to the extent that I'm hesitant to fix it up manually. Could you please refresh it against current -linus?

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates. Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

this still doesnt apply to latest -git. (or tip/master)

Ingo

#4 lore raw
Quoted 2 lines
Quoted 1 line
Quoted 1 line

Ingo Molnar <[email protected]> 21.07.08 13:05 >>>

this still doesnt apply to latest -git. (or tip/master)

Indeed, tip/master had a __pa -> __phys_addr_const conversion that I now sync-ed the patch with (without another round of testing):

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.
Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

---
 arch/x86/kernel/cpu/common.c     |   17 +++++--
 arch/x86/kernel/doublefault_32.c |   86 ++++++++++++++++++++++++---------------
 arch/x86/kernel/smpboot.c        |   44 +++++++++++++++++++
 arch/x86/kernel/traps_32.c       |   51 ++++++++++++++++++++++-
 drivers/lguest/segments.c        |    3 -
 include/asm-x86/kdebug.h         |    1 
 include/asm-x86/processor.h      |    7 ++-
 include/asm-x86/segment.h        |   15 ++++--
 include/asm-x86/thread_info_32.h |    9 +++-
 9 files changed, 187 insertions(+), 46 deletions(-)
arch/x86/kernel/cpu/common.c +13 -4
@@ -650,6 +650,13 @@ void switch_to_new_gdt(void)
 	asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
 }
 
+static void *__init_refok alloc_boot_stack(void)
+{
+	BUILD_BUG_ON(EXCEPTION_STACK_ORDER > THREAD_ORDER);
+	return __alloc_bootmem(EXCEPTION_STACK_SIZE, THREAD_SIZE,
+			       __phys_addr_const(MAX_DMA_ADDRESS));
+}
+
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
  * initialized (naturally) in the bootstrap process, such as the GDT
@@ -690,10 +697,12 @@ void __cpuinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
-#endif
+	if (cpu == 0) {
+		unsigned i;
+
+		for (i = 0; i < N_EXCEPTION_TSS; ++i)
+			setup_exception_tss(cpu, i, alloc_boot_stack);
+	}
 
 	/* Clear %gs. */
 	asm volatile ("mov %0, %%gs" : : "r" (0));
arch/x86/kernel/doublefault_32.c +53 -33

122 lines not on this page · raw message

arch/x86/kernel/smpboot.c +44 -0

56 lines not on this page · raw message

arch/x86/kernel/traps_32.c +50 -1

69 lines not on this page · raw message

drivers/lguest/segments.c +2 -1
@@ -50,7 +50,8 @@ static int ignored_gdt(unsigned int num)
 	return (num == GDT_ENTRY_TSS
 		|| num == GDT_ENTRY_LGUEST_CS
 		|| num == GDT_ENTRY_LGUEST_DS
-		|| num == GDT_ENTRY_DOUBLEFAULT_TSS);
+		|| (num >= GDT_ENTRY_EXCEPTION_TSS
+		    && num < GDT_ENTRY_EXCEPTION_TSS + N_EXCEPTION_TSS));
 }
 
 /*H:630 Once the Guest gave us new GDT entries, we fix them up a little.  We
include/asm-x86/kdebug.h +1 -0
@@ -20,6 +20,7 @@ enum die_val {
 	DIE_CALL,
 	DIE_NMI_IPI,
 	DIE_PAGE_FAULT,
+	DIE_DOUBLE_FAULT,
 	DIE_NMIUNKNOWN,
 };
 
include/asm-x86/processor.h +6 -1
@@ -128,7 +128,6 @@ struct cpuinfo_x86 {
 extern struct cpuinfo_x86	boot_cpu_data;
 extern struct cpuinfo_x86	new_cpu_data;
 
-extern struct tss_struct	doublefault_tss;
 extern __u32			cleared_cpu_caps[NCAPINTS];
 
 #ifdef CONFIG_SMP
@@ -838,6 +837,12 @@ static inline void spin_lock_prefetch(co
 	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },	  \
 }
 
+#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STACK_SIZE (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+void __cpuinit setup_exception_tss(unsigned int cpu, unsigned int idx,
+				   void *(*alloc_stack)(void));
+
 extern unsigned long thread_saved_pc(struct task_struct *tsk);
 
 #define THREAD_SIZE_LONGS      (THREAD_SIZE/sizeof(unsigned long))
include/asm-x86/segment.h +11 -4
@@ -55,7 +55,7 @@
  *  28 - unused
  *  29 - unused
  *  30 - unused
- *  31 - TSS for double fault handler
+ *  31+  TSSes for exception handlers
  */
 #define GDT_ENTRY_TLS_MIN	6
 #define GDT_ENTRY_TLS_MAX 	(GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1)
@@ -86,12 +86,19 @@
 #define __KERNEL_PERCPU 0
 #endif
 
-#define GDT_ENTRY_DOUBLEFAULT_TSS	31
+#define GDT_ENTRY_EXCEPTION_TSS	31
+#ifdef CONFIG_DOUBLEFAULT
+#define DOUBLEFAULT_TSS 0
+#define N_EXCEPTION_TSS 1
+#else
+#undef GDT_ENTRY_EXCEPTION_TSS
+#define N_EXCEPTION_TSS 0
+#endif
 
 /*
- * The GDT has 32 entries
+ * The GDT has 31+ entries
  */
-#define GDT_ENTRIES 32
+#define GDT_ENTRIES (31 + N_EXCEPTION_TSS)
 
 /* The PnP BIOS entries in the GDT */
 #define GDT_ENTRY_PNPBIOS_CS32		(GDT_ENTRY_PNPBIOS_BASE + 0)
#5 lore raw
Quoted 17 lines · Jan Beulich
Quoted 2 lines
Quoted 1 line
Quoted 1 line

Ingo Molnar <[email protected]> 21.07.08 13:05 >>>

this still doesnt apply to latest -git. (or tip/master)

Indeed, tip/master had a __pa -> __phys_addr_const conversion that I now sync-ed the patch with (without another round of testing):

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates. Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

Quoted 11 lines
---
 arch/x86/kernel/cpu/common.c     |   17 +++++--
 arch/x86/kernel/doublefault_32.c |   86 ++++++++++++++++++++++++---------------
 arch/x86/kernel/smpboot.c        |   44 +++++++++++++++++++
 arch/x86/kernel/traps_32.c       |   51 ++++++++++++++++++++++-
 drivers/lguest/segments.c        |    3 -
 include/asm-x86/kdebug.h         |    1 
 include/asm-x86/processor.h      |    7 ++-
 include/asm-x86/segment.h        |   15 ++++--
 include/asm-x86/thread_info_32.h |    9 +++-
 9 files changed, 187 insertions(+), 46 deletions(-)

I dont know.

All CPUs hitting a double fault simultaneously and corrupting each others' kernel stack is a theoretical possibility - but is handling it worth the complexity? It appears to me that a lock plus a short stub function that takes the lock (with no stack usage) would handle that much better.

Also, you seem to be setting things up to turn NMIs and MCEs into task gates too, right?

So i'm really uneasy about all this. Breakage in such rarely used code gets found very late, and has thus a high risk of losing debug information when we need it the most. (i.e. it works in the exact _opposite_ way of the intented goal of making things more robust - it makes things less robust)

Firstly, 64-bit does not use a task gate for double faults anymore. (but uses a separate IST stack for double faults)

Secondly, task gates are really a relic that should not be proliferated. Besides the complications in virtualized environments (if more common things like Big Real Mode are not supported well in virtual mode what do we expect of more esoteric features as task gates?) it does not get nearly as much testing on real silicon as other, more mainstream CPU features.

Thirdly, NMI based profiling is quite common, so by turning NMIs into task gates we'd slow that down quite a lot.

Also, the change to doublefault_fn is quite ugly - that inner block should be split out into a separate function.

Plus the notifier - why do we care about that? It's not like we can sanely kexec into a safe kernel from double faulting kernels in most cases. In real cases where i've seen double faults it was due to us corrupting kernel pagetables - kexec has no chance there. To recover from that we'd have to set up the TSS with a safe(r) cr3 as well - but your patch leaves _that_ untouched. (nor do we want to waste extra unswappable memory on such remote possibilities i think)

Ingo

#6 lore raw
Quoted 3 lines · Ingo Molnar

Secondly, task gates are really a relic that should not be proliferated. Besides the complications in virtualized environments (if more common things like Big Real Mode are not supported well in virtual mode what do we expect of more esoteric features as task gates?) it does not get nearly as much testing on real silicon as other, more mainstream CPU features.

I think that using it as a bailout mechanism is going to remain supported and tested. It's just never going to be fast.

-hpa

#7 lore raw
In reply to Ingo Molnar
Quoted 2 lines

Also, you seem to be setting things up to turn NMIs and MCEs into task gates too, right?

Yes, at the very minimum I'd like to have the possibility to do so. Perhaps under a default-off config option.

Quoted 5 lines

So i'm really uneasy about all this. Breakage in such rarely used code gets found very late, and has thus a high risk of losing debug information when we need it the most. (i.e. it works in the exact _opposite_ way of the intented goal of making things more robust - it makes things less robust)

I realize this aspect, but think that either way has its advantages and disadvantages.

Quoted 2 lines

Firstly, 64-bit does not use a task gate for double faults anymore. (but uses a separate IST stack for double faults)

Sure - because there are no task gates on 64-bit.

Quoted 9 lines

Secondly, task gates are really a relic that should not be proliferated. Besides the complications in virtualized environments (if more common things like Big Real Mode are not supported well in virtual mode what do we expect of more esoteric features as task gates?) it does not get nearly as much testing on real silicon as other, more mainstream CPU features.

Thirdly, NMI based profiling is quite common, so by turning NMIs into task gates we'd slow that down quite a lot.

As said above, I'd like to allow the option of doing so. Profiling via NMI certainly will not want this. I'm really uncertain whether modern machines can report any hardware issue through NMI (no chipset spec I read 'recently' [covering quite a number of years] was really explicit about this) - if it can't, MCE would be the only candidate unless running on really old hardware.

Quoted 2 lines

Also, the change to doublefault_fn is quite ugly - that inner block should be split out into a separate function.

That's certainly doable - if the whole thing is acceptable apart from that issue, which it doesn't seem it is...

Quoted 1 line

Plus the notifier - why do we care about that? It's not like we can

In order to let a kernel debugger take control.

Quoted 6 lines

sanely kexec into a safe kernel from double faulting kernels in most cases. In real cases where i've seen double faults it was due to us corrupting kernel pagetables - kexec has no chance there. To recover from that we'd have to set up the TSS with a safe(r) cr3 as well - but your patch leaves _that_ untouched. (nor do we want to waste extra unswappable memory on such remote possibilities i think)

I've seen double faults due to other than page table corruption, but I do understand if it is the page tables that caused it handling the condition is almost impossible without a second complete set of (kernel) page tables.

Jan

#8 lore raw
Quoted 5 lines · Jan Beulich
Quoted 2 lines

Firstly, 64-bit does not use a task gate for double faults anymore. (but uses a separate IST stack for double faults)

Sure - because there are no task gates on 64-bit.

What we're doing here is really using task gates to emulate IST anyway.

-hpa

#9 lore raw
Quoted 10 lines · H. Peter Anvin
Quoted 6 lines · Jan Beulich
Quoted 3 lines

Firstly, 64-bit does not use a task gate for double faults anymore. (but uses a separate IST stack for double faults)

Sure - because there are no task gates on 64-bit.

What we're doing here is really using task gates to emulate IST anyway.

yes, because we dont use the main feature that differentiates task gates from ISTs: a different cr3 entry. (the rest of the differences is really just fluff)

Ingo

#10 lore raw
In reply to Ingo Molnar
Quoted 2 lines · Ingo Molnar

All CPUs hitting a double fault simultaneously and corrupting each others' kernel stack is a theoretical possibility - but is handling it worth the complexity? It appears to me that a lock plus a short stub function that takes the lock (with no stack usage) would handle that much better.

That can't happen now because the TSS gets marked busy so we will get a triple fault instead. One thing we might want to do in the current code is unset the busy flag after handling the fault and before we start looping at the end of the handler so we can handle another fault later.

Quoted 3 lines

So i'm really uneasy about all this. Breakage in such rarely used code gets found very late, and has thus a high risk of losing debug information when we need it the most. (i.e. it works in the exact _opposite_ way of the intented goal of making things more robust - it makes things less robust)

Also how much bloat does this cause, having a per-CPU TSS and stack for every fault handler that uses this method?

#11 lore raw
Quoted 8 lines · Chuck Ebbert
Quoted 6 lines · Ingo Molnar

All CPUs hitting a double fault simultaneously and corrupting each others' kernel stack is a theoretical possibility - but is handling it worth the complexity? It appears to me that a lock plus a short stub function that takes the lock (with no stack usage) would handle that much better.

Quoted 5 lines

That can't happen now because the TSS gets marked busy so we will get a triple fault instead. One thing we might want to do in the current code is unset the busy flag after handling the fault and before we start looping at the end of the handler so we can handle another fault later.

that would be a nice improvement.

Ingo

#12 lore raw
In reply to Jan Beulich
Quoted 3 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.

What is the benefit of exception handlers going through task gates? Hardware task switches are not very well supported in virtualization (e.g. its has issues in KVM and is also not in Xen for a long time).

Joerg

#13 lore raw
Quoted 9 lines
Quoted 1 line
Quoted 1 line

Joerg Roedel <[email protected]> 23.07.08 23:43 >>>

Quoted 3 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.

What is the benefit of exception handlers going through task gates? Hardware task switches are not very well supported in virtualization (e.g. its has issues in KVM and is also not in Xen for a long time).

The main goal is to get to a different stack. While at present this is done only for the double fault, I think generally NMI and MCE should also do so, as they may be caused by a stack access (see x86-64, which runs them on IST stacks), and hence continuing to run on that same stack may not allow the exception to be handled.

Jan

#14 lore raw
Quoted 9 lines · Jan Beulich
Quoted 8 lines
Quoted 1 line
Quoted 1 line

Joerg Roedel <[email protected]> 23.07.08 23:43 >>>

Quoted 3 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future change of other exception handlers to go through task gates.

What is the benefit of exception handlers going through task gates? Hardware task switches are not very well supported in virtualization (e.g. its has issues in KVM and is also not in Xen for a long time).

Quoted 5 lines

The main goal is to get to a different stack. While at present this is done only for the double fault, I think generally NMI and MCE should also do so, as they may be caused by a stack access (see x86-64, which runs them on IST stacks), and hence continuing to run on that same stack may not allow the exception to be handled.

NMI, MCE and #DF are the obvious candidates.

Now, keep in mind TSSes have to be prepared per-CPU, since they get marked "busy" when in use, so it's a bit of a nontrivial undertaking.

-hpa