Skip to contentSkip to conversation

i386: per-CPU double fault TSS and stack

PATCH In review Jan Beulich 2 participants 4 messages Aug 28, 2007 – Sep 3, 2007

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 i386: per-CPU double fault TSS and stack

Make the double fault handler use CPU-specific stacks. Add some abstraction to simplify future changes of other exception handlers to also go through task gates. Add a new notification of the event through the die notifier chain, also providing some environmental adjustments so that various infrastructural things work independent of the fact that the fault and the callbacks are running on other then the normal kernel stack.

 arch/i386/kernel/cpu/common.c  |   63 ++++++++++++++++++++++++++++--
 arch/i386/kernel/doublefault.c |   85 +++++++++++++++++++++++++----------------
 arch/i386/kernel/traps.c       |   30 +++++++++++++-
 include/asm-i386/kdebug.h      |    1 
 include/asm-i386/processor.h   |    8 +++
 include/asm-i386/segment.h     |   16 +++++--
 include/asm-i386/thread_info.h |    9 +++-
 7 files changed, 168 insertions(+), 44 deletions(-)
arch/i386/kernel/cpu/common.c +60 -3
@@ -650,6 +650,16 @@ void switch_to_new_gdt(void)
 	asm("mov %0, %%fs" : : "r" (__KERNEL_PERCPU) : "memory");
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+static void *noinline __init_refok
+#else
+static inline void *__init
+#endif
+do_alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal)
+{
+	return __alloc_bootmem(size, align, goal);
+}
+
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
  * initialized (naturally) in the bootstrap process, such as the GDT
@@ -659,6 +669,9 @@ void switch_to_new_gdt(void)
 void __cpuinit cpu_init(void)
 {
 	int cpu = smp_processor_id();
+#if N_EXCEPTION_TSS
+	unsigned i;
+#endif
 	struct task_struct *curr = current;
 	struct tss_struct * t = &per_cpu(init_tss, cpu);
 	struct thread_struct *thread = &curr->thread;
@@ -696,9 +709,53 @@ void __cpuinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
+#if N_EXCEPTION_TSS
+#if EXCEPTION_STACK_ORDER > THREAD_ORDER
+#error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER
+#endif
+	for (i = 0; i < N_EXCEPTION_TSS; ++i) {
+		unsigned long stack;
+
+		/* Set up exception handling TSS */
+		exception_tss[cpu][i].ebx = (unsigned long)&exception_tss[cpu][i];
+
+		/* Set up exception handling stacks */
+#ifdef CONFIG_SMP
+		if (cpu) {
+			stack = __get_free_pages(GFP_ATOMIC, THREAD_ORDER);
+			if (!stack)
+				panic("Cannot allocate exception stack %u %d\n",
+				      i, cpu);
+		}
+		else
+#endif
+			stack = (unsigned long)do_alloc_bootmem(EXCEPTION_STACK_SIZE,
+								THREAD_SIZE,
+								__pa(MAX_DMA_ADDRESS));
+		stack += EXCEPTION_STACK_SIZE;
+		exception_tss[cpu][i].esp = exception_tss[cpu][i].esp0 = stack;
+#ifdef CONFIG_SMP
+		if (cpu) {
+			unsigned j;
+
+			for (j = EXCEPTION_STACK_ORDER; j < THREAD_ORDER; ++j) {
+				/* set_page_refs sets the page count only for the first
+				   page, but since we split the larger-order page here,
+				   we need to adjust the page count before freeing the
+				   pieces. */
+				struct page * page = virt_to_page((void *)stack);
+
+				BUG_ON(page_count(page));
+				init_page_count(page);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);
+			}
+		}
+#endif
+
+		/* Set up exception handling TSS pointer in the GDT */
+		__set_tss_desc(cpu, GDT_ENTRY_EXCEPTION_TSS + i, &exception_tss[cpu][i]);
+	}
 #endif
 
 	/* Clear %gs. */
arch/i386/kernel/doublefault.c +52 -33

120 lines not on this page · raw message

arch/i386/kernel/traps.c +29 -1
@@ -67,6 +67,30 @@ int panic_on_unrecovered_nmi;
 
 asmlinkage int system_call(void);
 
+#if N_EXCEPTION_TSS
+void doublefault_fn(void);
+
+struct i386_hw_tss exception_tss[NR_CPUS][N_EXCEPTION_TSS] __cacheline_aligned = {
+	[0 ... NR_CPUS-1] = {
+		[0 ... N_EXCEPTION_TSS-1] = {
+			.cs       = __KERNEL_CS,
+			.ss       = __KERNEL_DS,
+			.ss0      = __KERNEL_DS,
+			.__cr3    = __pa(swapper_pg_dir),
+			.io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
+			.ds       = __USER_DS,
+			.es       = __USER_DS,
+			.fs       = __KERNEL_PERCPU,
+			/* 0x2 bit is always set */
+			.eflags	  = X86_EFLAGS_SF | 0x2,
+		},
+#ifdef CONFIG_DOUBLEFAULT
+		[DOUBLEFAULT_TSS].eip = (unsigned long)doublefault_fn
+#endif
+	}
+};
+#endif
+
 /* Do we ignore FPU interrupts ? */
 char ignore_fpu_irq = 0;
 
@@ -1148,10 +1172,12 @@ static void __init set_system_gate(unsig
 	_set_gate(n, DESCTYPE_TRAP | DESCTYPE_DPL3, addr, __KERNEL_CS);
 }
 
+#if N_EXCEPTION_TSS
 static void __init set_task_gate(unsigned int n, unsigned int gdt_entry)
 {
 	_set_gate(n, DESCTYPE_TASK, (void *)0, (gdt_entry<<3));
 }
+#endif
 
 
 void __init trap_init(void)
@@ -1176,7 +1202,9 @@ void __init trap_init(void)
 	set_trap_gate(5,&bounds);
 	set_trap_gate(6,&invalid_op);
 	set_trap_gate(7,&device_not_available);
-	set_task_gate(8,GDT_ENTRY_DOUBLEFAULT_TSS);
+#ifdef DOUBLEFAULT_TSS
+	set_task_gate(8,GDT_ENTRY_EXCEPTION_TSS + DOUBLEFAULT_TSS);
+#endif
 	set_trap_gate(9,&coprocessor_segment_overrun);
 	set_trap_gate(10,&invalid_TSS);
 	set_trap_gate(11,&segment_not_present);
include/asm-i386/kdebug.h +1 -0
@@ -28,6 +28,7 @@ enum die_val {
 	DIE_CALL,
 	DIE_NMI_IPI,
 	DIE_PAGE_FAULT,
+	DIE_DOUBLE_FAULT
 };
 
 #endif
include/asm-i386/processor.h +7 -1
@@ -99,7 +99,6 @@ struct cpuinfo_x86 {
 
 extern struct cpuinfo_x86 boot_cpu_data;
 extern struct cpuinfo_x86 new_cpu_data;
-extern struct tss_struct doublefault_tss;
 DECLARE_PER_CPU(struct tss_struct, init_tss);
 
 #ifdef CONFIG_SMP
@@ -389,6 +388,13 @@ struct thread_struct {
 	.io_bitmap	= { [ 0 ... IO_BITMAP_LONGS] = ~0 },		\
 }
 
+#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STACK_SIZE (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+#if N_EXCEPTION_TSS
+extern struct i386_hw_tss exception_tss[NR_CPUS][N_EXCEPTION_TSS];
+#endif
+
 #define start_thread(regs, new_eip, new_esp) do {		\
 	__asm__("movl %0,%%gs": :"r" (0));			\
 	regs->xfs = 0;						\
include/asm-i386/segment.h +12 -4
@@ -43,7 +43,8 @@
  *  28 - unused
  *  29 - unused
  *  30 - unused
- *  31 - TSS for double fault handler
+ *  31 - TSS for first exception handler (double fault)
+ *  32+  TSSes for further exception handlers
  */
 #define GDT_ENTRY_TLS_ENTRIES	3
 #define GDT_ENTRY_TLS_MIN	6
@@ -81,12 +82,19 @@
 #define __KERNEL_PERCPU 0
 #endif
 
-#define GDT_ENTRY_DOUBLEFAULT_TSS	31
+#define GDT_ENTRY_EXCEPTION_TSS	31
+#ifdef CONFIG_DOUBLEFAULT
+#define DOUBLEFAULT_TSS 0
+#define N_EXCEPTION_TSS 1
+#else
+#undef GDT_ENTRY_EXCEPTION_TSS
+#define N_EXCEPTION_TSS 0
+#endif
 
 /*
- * The GDT has 32 entries
+ * The GDT has 31+ entries
  */
-#define GDT_ENTRIES 32
+#define GDT_ENTRIES (31 + N_EXCEPTION_TSS)
 #define GDT_SIZE (GDT_ENTRIES * 8)
 
 /* Simple and small GDT entries for booting only */
include/asm-i386/thread_info.h +7 -2
@@ -54,9 +54,14 @@ struct thread_info {
 
 #define PREEMPT_ACTIVE		0x10000000
 #ifdef CONFIG_4KSTACKS
-#define THREAD_SIZE            (4096)
+#define THREAD_ORDER 0
 #else
-#define THREAD_SIZE		(8192)
+#define THREAD_ORDER 1
+#endif
+#ifndef __ASSEMBLY__
+#define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER)
+#else
+#define THREAD_SIZE (PAGE_SIZE_asm << THREAD_ORDER)
 #endif
 
 #define STACK_WARN             (THREAD_SIZE/8)
#1 lore raw

Can you cc the next version to Linus please? He's probably best qualified to review the i386 double fault handler because he wrote it originally. I must admit the code always scared me a bit.

arch/i386/kernel/cpu/common.c void switch_to_new_gdt(void) · line 657

Quoted 5 lines
arch/i386/kernel/cpu/common.c +5 -0
+#ifdef CONFIG_HOTPLUG_CPU
+static void *noinline __init_refok
+#else
+static inline void *__init
+#endif

I really wonder if there isn't a cleaner way to do that :-( These init reference checks are starting to become a major annoyance.

Quoted 9 lines
arch/i386/kernel/cpu/common.c +5 -0
+do_alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal)
+{
+	return __alloc_bootmem(size, align, goal);
+}
+
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
  * initialized (naturally) in the bootstrap process, such as the GDT

arch/i386/kernel/cpu/common.c void switch_to_new_gdt(void) · line 674

Quoted 6 lines
arch/i386/kernel/cpu/common.c +3 -0
@@ -659,6 +669,9 @@ void switch_to_new_gdt(void)
 void __cpuinit cpu_init(void)
 {
 	int cpu = smp_processor_id();
+#if N_EXCEPTION_TSS
+	unsigned i;
+#endif

Would it be that bad to have the TSS even around without CONFIG_DOUBLEFAULT?

In fact I would prefer to just eliminate CONFIG_DOUBLEFAULT (imho
it always a bad idea because the amount of code it saves is miniscule) instead of adding such a ifdef maze.

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 715

Quoted 7 lines
arch/i386/kernel/cpu/common.c +4 -3
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
+#if N_EXCEPTION_TSS
+#if EXCEPTION_STACK_ORDER > THREAD_ORDER
+#error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER
+#endif

BUILD_BUG_ON would look nicer

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 724

Quoted 4 lines
arch/i386/kernel/cpu/common.c +4 -0
+
+		/* Set up exception handling stacks */
+#ifdef CONFIG_SMP
+		if (cpu) {

If you move the code after the gs pda setup you could use smp_processor_id() and avoid the ifdefs (on UP it expands to 0 so the optimizer would do it cleanly)

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 751

Quoted 4 lines
arch/i386/kernel/cpu/common.c +4 -0
+				BUG_ON(page_count(page));
+				init_page_count(page);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);

In 2.4-aa I added a alloc_pages_exact() for this. I don't think such games should be played outside page_alloc.c. I would recommend to readd alloc_pages_exact() and then use it.

arch/i386/kernel/doublefault.c line 13

Quoted 4 lines
arch/i386/kernel/doublefault.c +1 -3
-#define DOUBLEFAULT_STACKSIZE (1024)
-static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
-#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
+extern unsigned long max_low_pfn;

No externs in .c

arch/i386/kernel/doublefault.c line 19

Quoted 8 lines
arch/i386/kernel/doublefault.c +4 -2
+#define ptr_ok(x, l) ((x) >= PAGE_OFFSET \
+                      && (x) + (l) <= PAGE_OFFSET + max_low_pfn * PAGE_SIZE - 1)
 
-#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
+#define THREAD_INFO_FROM(x) ((struct thread_info *)((x) & ~(THREAD_SIZE - 1)))
 
-static void doublefault_fn(void)
+register const struct i386_hw_tss *self __asm__("ebx");

Can't you just move that to a proper argument register in assembler code?

-Andi

#2 lore raw
Quoted 5 lines
Quoted 1 line
Quoted 1 line

Andi Kleen <[email protected]> 01.09.07 12:33 >>>

Can you cc the next version to Linus please? He's probably best qualified to review the i386 double fault handler because he wrote it originally. I must admit the code always scared me a bit.

Will do.

Quoted 8 lines

arch/i386/kernel/cpu/common.c void switch_to_new_gdt(void) · line 657

Quoted 5 lines
arch/i386/kernel/cpu/common.c +5 -0
+#ifdef CONFIG_HOTPLUG_CPU
+static void *noinline __init_refok
+#else
+static inline void *__init
+#endif

I really wonder if there isn't a cleaner way to do that :-( These init reference checks are starting to become a major annoyance.

They just aren't flexible enough. And no, I can't think of a better way here.

Quoted 5 lines

arch/i386/kernel/cpu/common.c void switch_to_new_gdt(void) · line 674

Quoted 3 lines
arch/i386/kernel/cpu/common.c +3 -0
+#if N_EXCEPTION_TSS
+	unsigned i;
+#endif

Would it be that bad to have the TSS even around without CONFIG_DOUBLEFAULT?

It costs 4.xx k space per CPU - perhaps a constraint for embedded?

Quoted 3 lines

In fact I would prefer to just eliminate CONFIG_DOUBLEFAULT (imho
it always a bad idea because the amount of code it saves is miniscule) instead of adding such a ifdef maze.

It's configurable for embedded only anyway, and I think there's some value in allowing it to be configured off for that environment.

Quoted 9 lines

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 715

Quoted 7 lines
arch/i386/kernel/cpu/common.c +4 -3
-#ifdef CONFIG_DOUBLEFAULT
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
+#if N_EXCEPTION_TSS
+#if EXCEPTION_STACK_ORDER > THREAD_ORDER
+#error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER
+#endif

BUILD_BUG_ON would look nicer

Agreed, will change (this code dates back to pre-BUILD_BUG_ON days).

Quoted 7 lines

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 724

Quoted 4 lines
arch/i386/kernel/cpu/common.c +4 -0
+
+		/* Set up exception handling stacks */
+#ifdef CONFIG_SMP
+		if (cpu) {

If you move the code after the gs pda setup you could use smp_processor_id() and avoid the ifdefs (on UP it expands to 0 so the optimizer would do it cleanly)

I'll check if that's feasible.

Quoted 8 lines

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 751

Quoted 4 lines
arch/i386/kernel/cpu/common.c +4 -0
+				BUG_ON(page_count(page));
+				init_page_count(page);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);

In 2.4-aa I added a alloc_pages_exact() for this. I don't think such games should be played outside page_alloc.c. I would recommend to readd alloc_pages_exact() and then use it.

Will need to track that patch down.

Quoted 6 lines

arch/i386/kernel/doublefault.c line 13

Quoted 4 lines
arch/i386/kernel/doublefault.c +1 -3
-#define DOUBLEFAULT_STACKSIZE (1024)
-static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
-#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
+extern unsigned long max_low_pfn;

No externs in .c

The question is - is it acceptable to declare max_low_pfn in any header?

Quoted 10 lines

arch/i386/kernel/doublefault.c line 19

Quoted 8 lines
arch/i386/kernel/doublefault.c +4 -2
+#define ptr_ok(x, l) ((x) >= PAGE_OFFSET \
+                      && (x) + (l) <= PAGE_OFFSET + max_low_pfn * PAGE_SIZE - 1)
 
-#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
+#define THREAD_INFO_FROM(x) ((struct thread_info *)((x) & ~(THREAD_SIZE - 1)))
 
-static void doublefault_fn(void)
+register const struct i386_hw_tss *self __asm__("ebx");

Can't you just move that to a proper argument register in assembler code?

That should be possible, though I can't see anything wrong with the approach I used.

Jan

#3 lore raw
Quoted 2 lines

It costs 4.xx k space per CPU - perhaps a constraint for embedded?

Not a major one.

Quoted 7 lines
Quoted 3 lines

In fact I would prefer to just eliminate CONFIG_DOUBLEFAULT (imho it always a bad idea because the amount of code it saves is miniscule) instead of adding such a ifdef maze.

It's configurable for embedded only anyway, and I think there's some value in allowing it to be configured off for that environment.

With less ifdefs then please.

Quoted 11 lines
Quoted 9 lines

arch/i386/kernel/cpu/common.c void __cpuinit cpu_init(void) · line 751

Quoted 5 lines
arch/i386/kernel/cpu/common.c +4 -0
+				BUG_ON(page_count(page));
+				init_page_count(page);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);

In 2.4-aa I added a alloc_pages_exact() for this. I don't think such games should be played outside page_alloc.c. I would recommend to readd alloc_pages_exact() and then use it.

Will need to track that patch down.

http://www.kernel.org/pub/linux/kernel/people/andrea/kernels/v2.4/2.4.23aa3/00_module-gfp-7
But it'll need quite some changes for 2.6 anyways.

Quoted 9 lines
Quoted 7 lines
Quoted 5 lines
arch/i386/kernel/doublefault.c +0 -3
-#define DOUBLEFAULT_STACKSIZE (1024)
-static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
-#define STACK_START (unsigned

long)(doublefault_stack+DOUBLEFAULT_STACKSIZE) +extern unsigned long max_low_pfn;

No externs in .c

The question is - is it acceptable to declare max_low_pfn in any header?

Sure, why not?

-Andi