Skip to contentSkip to conversation

fix i386 double fault handler

PATCH Chuck Ebbert 2 participants 2 messages Sep 9, 2005 – Sep 12, 2005

lore
Share & tools
lore
#0 lore raw

In-Reply-To: <[email protected]>

Quoted 5 lines · Jan Beulich

Make the double fault handler use CPU-specific stacks, add some abstraction to simplify future change of other exception handlers to go through task gates. Change the pointer validity checks in the double fault handler to account for the fact that both GDT and TSS aren't in static kernel space anymore.

This is really nice. Some comments, though:

Quoted 32 lines
arch/i386/kernel/cpu/common.c +10 -2
@@ -4,6 +4,7 @@
 #include <linux/smp.h>
 #include <linux/module.h>
 #include <linux/percpu.h>
+#include <linux/bootmem.h>
 #include <asm/semaphore.h>
 #include <asm/processor.h>
 #include <asm/i387.h>
@@ -571,6 +572,7 @@ void __init early_cpu_init(void)
 void __devinit cpu_init(void)
 {
 	int cpu = smp_processor_id();
+	unsigned i;
 	struct tss_struct * t = &per_cpu(init_tss, cpu);
 	struct thread_struct *thread = &current->thread;
 	__u32 stk16_off = (__u32)&per_cpu(cpu_16bit_stack, cpu);
@@ -635,8 +637,57 @@ void __devinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
+#if EXCEPTION_STACK_ORDER > THREAD_ORDER
+# error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER
+#endif
+	for (i = 0; i < N_EXCEPTION_TSS; ++i) {
+		unsigned long stack;
+
+		/* Set up exception handling TSS */
+		exception_tss[cpu][i].esp2 = cpu;

arch/i386/kernel/cpu/common.c void __devinit cpu_init(void) · line 657

Quoted 10 lines
arch/i386/kernel/cpu/common.c +10 -0
+		exception_tss[cpu][i].esi = (unsigned long)&exception_tss[cpu][i];
+
+		/* Set up exception handling stacks */
+#ifdef CONFIG_SMP
+		if (cpu) {
+			stack = __get_free_pages(GFP_ATOMIC, THREAD_ORDER);
+			if (!stack)
+				panic("Cannot allocate exception stack %u %d\n",
+				      i,
+				      cpu);

Indent gone wild?

arch/i386/kernel/cpu/common.c void __devinit cpu_init(void) · line 665

Quoted 8 lines
arch/i386/kernel/cpu/common.c +8 -0
+		}
+		else
+#endif
+			stack = (unsigned long)__alloc_bootmem(EXCEPTION_STKSZ,
+			                                       THREAD_SIZE,
+			                                       __pa(MAX_DMA_ADDRESS));
+		stack += EXCEPTION_STKSZ;
+		exception_tss[cpu][i].esp = exception_tss[cpu][i].esp0 = stack;

Please don't cascade assignments like that. It's much more obvious
when it's like this:

		exception_tss[cpu][i].esp = stack;
		exception_tss[cpu][i].esp0 = stack;
Quoted 11 lines
arch/i386/kernel/cpu/common.c +11 -0
+#ifdef CONFIG_SMP
+		if (cpu) {
+			unsigned j;
+
+			for (j = EXCEPTION_STACK_ORDER; j < THREAD_ORDER; ++j) {
+				/* set_page_refs sets the page count only for the first
+				   page, but since we split the larger-order page here,
+				   we need to adjust the page count before freeing the
+				   pieces. */
+				struct page * page = virt_to_page((void *)stack);
+

arch/i386/kernel/cpu/common.c void __devinit cpu_init(void) · line 686

Quoted 10 lines
arch/i386/kernel/cpu/common.c +10 -0
+				BUG_ON(page_count(page));
+				set_page_count(page, 1);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);
+			}
+		}
+		else
+#endif
+		if (EXCEPTION_STACK_ORDER > THREAD_ORDER)
+			free_bootmem(stack, THREAD_SIZE - EXCEPTION_STKSZ);

This comparison should be "if (EXCEPTION_STACK_ORDER < THREAD_ORDER)"

Quoted 24 lines
arch/i386/kernel/cpu/common.c +4 -0
+
+		/* Set up exception handling TSS pointer in the GDT */
+		__set_tss_desc(cpu, GDT_ENTRY_EXCEPTION_TSS + i, &exception_tss[cpu][i]);
+	}
 
 	/* Clear %fs and %gs. */
 	asm volatile ("xorl %eax, %eax; movl %eax, %fs; movl %eax, %gs");
arch/i386/kernel/head.S +2 -2
@@ -475,9 +475,9 @@ ENTRY(boot_gdt_table)
 	.quad 0x00cf92000000ffff	/* kernel 4GB data at 0x00000000 */
 
 /*
- * The Global Descriptor Table contains 28 quadwords, per-CPU.
+ * The Global Descriptor Table contains at least 31 quadwords, per-CPU.
  */
-	.align PAGE_SIZE_asm
+	.align L1_CACHE_BYTES
 ENTRY(cpu_gdt_table)
 	.quad 0x0000000000000000	/* NULL descriptor */
 	.quad 0x0000000000000000	/* 0x0b reserved */

arch/i386/kernel/head.S ENTRY(cpu_gdt_table) · line 521

Quoted 7 lines
arch/i386/kernel/head.S +0 -4
@@ -515,9 +515,5 @@ ENTRY(cpu_gdt_table)
 	.quad 0x0040920000000000	/* 0xc8 APM DS    data */
 
 	.quad 0x0000920000000000	/* 0xd0 - ESPFIX 16-bit SS */
-	.quad 0x0000000000000000	/* 0xd8 - unused */
-	.quad 0x0000000000000000	/* 0xe0 - unused */
-	.quad 0x0000000000000000	/* 0xe8 - unused */
-	.quad 0x0000000000000000	/* 0xf0 - unused */

Could you leave these in? They make it easier to see they're available.

arch/i386/kernel/head.S ENTRY(cpu_gdt_table) · line 518

Quoted 3 lines
arch/i386/kernel/head.S +1 -2
-	.quad 0x0000000000000000	/* 0xf8 - GDT entry 31: double-fault TSS */
-
+	/* remaining entries run-time initialized */

A better comment would help here.

arch/i386/kernel/head.S ENTRY(cpu_gdt_table) · line 519

Quoted 1 line
arch/i386/kernel/head.S +1 -0
+	.fill GDT_ENTRIES - (. - cpu_gdt_table) / 8, 8, 0

__
Chuck
Subliminal URL: www.sluggy.com/daily.php?date=050905

#1 lore raw
Quoted 9 lines · Jan Beulich
Quoted 6 lines

Make the double fault handler use CPU-specific stacks, add some abstraction to simplify future change of other exception handlers to go through task gates. Change the pointer validity checks in the double fault handler to account for the fact that both GDT and TSS aren't in static kernel space anymore.

This is really nice. Some comments, though:

Thanks!

Quoted 1 line
Quoted 1 line

--- 2.6.13/arch/i386/kernel/cpu/common.c 2005-08-29

01:41:01.000000000 +0200

Quoted 1 line
Quoted 1 line

+++

2.6.13-i386-double-fault/arch/i386/kernel/cpu/common.c 2005-09-08
15:00:54.000000000 +0200

Quoted 2 lines

Review

Quoted 2 lines

...

(quoted diff) +1 -0
+				panic("Cannot allocate exception stack

%u %d\n",

Quoted 4 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+				      i,
+				      cpu);

Indent gone wild?

Not really. There doesn't seem to be a clear indentation rule for function arguments not all fitting on one line - I re-checked a few instances (namely calls to do_fork), and the most consistent thing I can conclude from this would be "indent using spaces to match the first arguments position, and replace groups of 8 spaces with tabs". The code here seems to match this, except that I dislike putting two arguments on the same line if I have to split the line anyway.

Quoted 2 lines

Review

Quoted 2 lines

...

(quoted diff) +1 -0
+		exception_tss[cpu][i].esp = exception_tss[cpu][i].esp0 =

stack;

Quoted 5 lines

Please don't cascade assignments like that. It's much more obvious
when it's like this:

		exception_tss[cpu][i].esp = stack;
		exception_tss[cpu][i].esp0 = stack;

Changed.

Quoted 3 lines

Review

Quoted 3 lines

...

(quoted diff) +2 -0
+		if (EXCEPTION_STACK_ORDER > THREAD_ORDER)
+			free_bootmem(stack, THREAD_SIZE -

EXCEPTION_STKSZ);

Quoted 2 lines

This comparison should be "if (EXCEPTION_STACK_ORDER <

THREAD_ORDER)"

Indeed. However, the whole piece is unnecessary as it turns out (I added it shortly before submitting the patch, thinking I had overlooked the freeing part for the bootmem case, but in reality there's nothing to be freed in that case at all).

Quoted 2 lines
Quoted 2 lines

...
--- 2.6.13/arch/i386/kernel/head.S 2005-08-29 01:41:01.000000000

+0200

Quoted 1 line
Quoted 1 line

+++

2.6.13-i386-double-fault/arch/i386/kernel/head.S 2005-09-01
13:06:01.000000000 +0200

Quoted 7 lines

Review

Quoted 5 lines

...

(quoted diff) +0 -4
-	.quad 0x0000000000000000	/* 0xd8 - unused */
-	.quad 0x0000000000000000	/* 0xe0 - unused */
-	.quad 0x0000000000000000	/* 0xe8 - unused */
-	.quad 0x0000000000000000	/* 0xf0 - unused */

Could you leave these in? They make it easier to see they're

available.

No problem.

Quoted 1 line

Review

Quoted 1 line
(quoted diff) +0 -1
-	.quad 0x0000000000000000	/* 0xf8 - GDT entry 31:

double-fault TSS */

Quoted 4 lines

Review

Quoted 2 lines
(quoted diff) +1 -1
-
+	/* remaining entries run-time initialized */

A better comment would help here.

Hopefully the new version will be liked better.

New patch attached.

Thanks again, Jan

Make the double fault handler use CPU-specific stacks, add some abstraction to simplify future change of other exception handlers to go through task gates. Change the pointer validity checks in the double fault handler to account for the fact that both GDT and TSS aren't in static kernel space anymore.

This patch depends upon the presence of THREAD_ORDER, as added in a previously submitted patch.

arch/i386/kernel/cpu/common.c +51 -2
diff -Npru 2.6.13/arch/i386/kernel/cpu/common.c 2.6.13-i386-double-fault/arch/i386/kernel/cpu/common.c
@@ -4,6 +4,7 @@
 #include <linux/smp.h>
 #include <linux/module.h>
 #include <linux/percpu.h>
+#include <linux/bootmem.h>
 #include <asm/semaphore.h>
 #include <asm/processor.h>
 #include <asm/i387.h>
@@ -571,6 +572,7 @@ void __init early_cpu_init(void)
 void __devinit cpu_init(void)
 {
 	int cpu = smp_processor_id();
+	unsigned i;
 	struct tss_struct * t = &per_cpu(init_tss, cpu);
 	struct thread_struct *thread = &current->thread;
 	__u32 stk16_off = (__u32)&per_cpu(cpu_16bit_stack, cpu);
@@ -635,8 +637,55 @@ void __devinit cpu_init(void)
 	load_TR_desc();
 	load_LDT(&init_mm.context);
 
-	/* Set up doublefault TSS pointer in the GDT */
-	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
+#if EXCEPTION_STACK_ORDER > THREAD_ORDER
+# error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER
+#endif
+	for (i = 0; i < N_EXCEPTION_TSS; ++i) {
+		unsigned long stack;
+
+		/* Set up exception handling TSS */
+		exception_tss[cpu][i].esp2 = cpu;
+		exception_tss[cpu][i].esi = (unsigned long)&exception_tss[cpu][i];
+
+		/* Set up exception handling stacks */
+#ifdef CONFIG_SMP
+		if (cpu) {
+			stack = __get_free_pages(GFP_ATOMIC, THREAD_ORDER);
+			if (!stack)
+				panic("Cannot allocate exception stack %u %d\n",
+				      i,
+				      cpu);
+		}
+		else
+#endif
+			stack = (unsigned long)__alloc_bootmem(EXCEPTION_STKSZ,
+			                                       THREAD_SIZE,
+			                                       __pa(MAX_DMA_ADDRESS));
+		stack += EXCEPTION_STKSZ;
+		exception_tss[cpu][i].esp = stack;
+		exception_tss[cpu][i].esp0 = stack;
+#ifdef CONFIG_SMP
+		if (cpu) {
+			unsigned j;
+
+			for (j = EXCEPTION_STACK_ORDER; j < THREAD_ORDER; ++j) {
+				/* set_page_refs sets the page count only for the first
+				   page, but since we split the larger-order page here,
+				   we need to adjust the page count before freeing the
+				   pieces. */
+				struct page * page = virt_to_page((void *)stack);
+
+				BUG_ON(page_count(page));
+				set_page_count(page, 1);
+				free_pages(stack, j);
+				stack += (PAGE_SIZE << j);
+			}
+		}
+#endif
+
+		/* Set up exception handling TSS pointer in the GDT */
+		__set_tss_desc(cpu, GDT_ENTRY_EXCEPTION_TSS + i, &exception_tss[cpu][i]);
+	}
 
 	/* Clear %fs and %gs. */
 	asm volatile ("xorl %eax, %eax; movl %eax, %fs; movl %eax, %gs");
arch/i386/kernel/doublefault.c +9 -28
diff -Npru 2.6.13/arch/i386/kernel/doublefault.c 2.6.13-i386-double-fault/arch/i386/kernel/doublefault.c
@@ -9,13 +9,11 @@
 #include <asm/processor.h>
 #include <asm/desc.h>
 
-#define DOUBLEFAULT_STACKSIZE (1024)
-static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
-#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
+extern unsigned long max_low_pfn;
+#define ptr_ok(x, l) ((x) >= PAGE_OFFSET \
+                      && (x) + (l) <= PAGE_OFFSET + max_low_pfn * PAGE_SIZE - 1)
 
-#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + 0x1000000)
-
-static void doublefault_fn(void)
+void doublefault_fn(void)
 {
 	struct Xgt_desc_struct gdt_desc = {0, 0};
 	unsigned long gdt, tss;
@@ -23,43 +21,26 @@ static void doublefault_fn(void)
 	__asm__ __volatile__("sgdt %0": "=m" (gdt_desc): :"memory");
 	gdt = gdt_desc.address;
 
-	printk("double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size);
+	printk("double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size + 1);
 
-	if (ptr_ok(gdt)) {
+	if (ptr_ok(gdt, gdt_desc.size)) {
 		gdt += GDT_ENTRY_TSS << 3;
 		tss = *(u16 *)(gdt+2);
 		tss += *(u8 *)(gdt+4) << 16;
 		tss += *(u8 *)(gdt+7) << 24;
 		printk("double fault, tss at %08lx\n", tss);
 
-		if (ptr_ok(tss)) {
+		if (ptr_ok(tss, *(u16 *)gdt)) {
 			struct tss_struct *t = (struct tss_struct *)tss;
 
 			printk("eip = %08lx, esp = %08lx\n", t->eip, t->esp);
 
 			printk("eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n",
 				t->eax, t->ebx, t->ecx, t->edx);
-			printk("esi = %08lx, edi = %08lx\n",
-				t->esi, t->edi);
+			printk("esi = %08lx, edi = %08lx, ebp = %08lx\n",
+				t->esi, t->edi, t->ebp);
 		}
 	}
 
 	for (;;) /* nothing */;
 }
-
-struct tss_struct doublefault_tss __cacheline_aligned = {
-	.esp0		= STACK_START,
-	.ss0		= __KERNEL_DS,
-	.ldt		= 0,
-	.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
-
-	.eip		= (unsigned long) doublefault_fn,
-	.eflags		= X86_EFLAGS_SF | 0x2,	/* 0x2 bit is always set */
-	.esp		= STACK_START,
-	.es		= __USER_DS,
-	.cs		= __KERNEL_CS,
-	.ss		= __KERNEL_DS,
-	.ds		= __USER_DS,
-
-	.__cr3		= __pa(swapper_pg_dir)
-};
arch/i386/kernel/head.S +5 -3
diff -Npru 2.6.13/arch/i386/kernel/head.S 2.6.13-i386-double-fault/arch/i386/kernel/head.S
@@ -475,9 +475,9 @@ ENTRY(boot_gdt_table)
 	.quad 0x00cf92000000ffff	/* kernel 4GB data at 0x00000000 */
 
 /*
- * The Global Descriptor Table contains 28 quadwords, per-CPU.
+ * The Global Descriptor Table contains at least 31 quadwords, per-CPU.
  */
-	.align PAGE_SIZE_asm
+	.align L1_CACHE_BYTES
 ENTRY(cpu_gdt_table)
 	.quad 0x0000000000000000	/* NULL descriptor */
 	.quad 0x0000000000000000	/* 0x0b reserved */
@@ -519,5 +519,7 @@ ENTRY(cpu_gdt_table)
 	.quad 0x0000000000000000	/* 0xe0 - unused */
 	.quad 0x0000000000000000	/* 0xe8 - unused */
 	.quad 0x0000000000000000	/* 0xf0 - unused */
-	.quad 0x0000000000000000	/* 0xf8 - GDT entry 31: double-fault TSS */
 
+	/* Remaining entries represent TSSes for handling exceptions and
+	   are run-time initialized. */
+	.fill GDT_ENTRIES - (. - cpu_gdt_table) / 8, 8, 0
arch/i386/kernel/traps.c +19 -1
diff -Npru 2.6.13/arch/i386/kernel/traps.c 2.6.13-i386-double-fault/arch/i386/kernel/traps.c
@@ -62,6 +62,24 @@ asmlinkage int system_call(void);
 struct desc_struct default_ldt[] = { { 0, 0 }, { 0, 0 }, { 0, 0 },
 		{ 0, 0 }, { 0, 0 } };
 
+void doublefault_fn(void);
+
+struct tss_struct exception_tss[NR_CPUS][N_EXCEPTION_TSS] __cacheline_aligned = {
+	[0 ... NR_CPUS-1] = {
+		[0 ... N_EXCEPTION_TSS-1] = {
+			.cs       = __KERNEL_CS,
+			.ss       = __KERNEL_DS,
+			.ss0      = __KERNEL_DS,
+			.__cr3    = __pa(swapper_pg_dir),
+			.io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
+			.ds       = __USER_DS,
+			.es       = __USER_DS,
+			.eflags	  = X86_EFLAGS_SF | 0x2, /* 0x2 bit is always set */
+		},
+		[DOUBLEFAULT_TSS].eip = (unsigned long)doublefault_fn
+	}
+};
+
 /* Do we ignore FPU interrupts ? */
 char ignore_fpu_irq = 0;
 
@@ -1083,7 +1101,7 @@ void __init trap_init(void)
 	set_system_gate(5,&bounds);
 	set_trap_gate(6,&invalid_op);
 	set_trap_gate(7,&device_not_available);
-	set_task_gate(8,GDT_ENTRY_DOUBLEFAULT_TSS);
+	set_task_gate(8,GDT_ENTRY_EXCEPTION_TSS + DOUBLEFAULT_TSS);
 	set_trap_gate(9,&coprocessor_segment_overrun);
 	set_trap_gate(10,&invalid_TSS);
 	set_trap_gate(11,&segment_not_present);
include/asm-i386/processor.h +6 -1
diff -Npru 2.6.13/include/asm-i386/processor.h 2.6.13-i386-double-fault/include/asm-i386/processor.h
@@ -86,7 +86,6 @@ struct cpuinfo_x86 {
 
 extern struct cpuinfo_x86 boot_cpu_data;
 extern struct cpuinfo_x86 new_cpu_data;
-extern struct tss_struct doublefault_tss;
 DECLARE_PER_CPU(struct tss_struct, init_tss);
 
 #ifdef CONFIG_SMP
@@ -479,6 +478,12 @@ struct thread_struct {
 	.io_bitmap	= { [ 0 ... IO_BITMAP_LONGS] = ~0 },		\
 }
 
+#define DOUBLEFAULT_TSS 0
+#define EXCEPTION_STACK_ORDER 0
+#define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
+
+extern struct tss_struct exception_tss[NR_CPUS][N_EXCEPTION_TSS];
+
 static inline void load_esp0(struct tss_struct *tss, struct thread_struct *thread)
 {
 	tss->esp0 = thread->esp0;
include/asm-i386/segment.h +16 -7
diff -Npru 2.6.13/include/asm-i386/segment.h 2.6.13-i386-double-fault/include/asm-i386/segment.h
@@ -1,6 +1,8 @@
 #ifndef _ASM_SEGMENT_H
 #define _ASM_SEGMENT_H
 
+#include <asm/cache.h>
+
 /*
  * The layout of the per-CPU GDT under Linux:
  *
@@ -43,7 +45,8 @@
  *  28 - unused
  *  29 - unused
  *  30 - unused
- *  31 - TSS for double fault handler
+ *  31 - TSS for first exception handler (double fault)
+ *  32+  TSSes for further exception handlers
  */
 #define GDT_ENTRY_TLS_ENTRIES	3
 #define GDT_ENTRY_TLS_MIN	6
@@ -74,12 +77,20 @@
 #define GDT_ENTRY_ESPFIX_SS		(GDT_ENTRY_KERNEL_BASE + 14)
 #define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8)
 
-#define GDT_ENTRY_DOUBLEFAULT_TSS	31
+#define GDT_ENTRY_EXCEPTION_TSS	31
+#define DOUBLEFAULT_TSS 0
+#define N_EXCEPTION_TSS 1
 
 /*
- * The GDT has 32 entries
+ * The GDT has 32+ entries
  */
-#define GDT_ENTRIES 32
+#if L1_CACHE_BYTES < 8
+# error Assertion failed: L1_CACHE_BYTES >= 8
+#endif
+#if L1_CACHE_BYTES & (L1_CACHE_BYTES - 1)
+# error Assertion failed: L1_CACHE_BYTES is power of two
+#endif
+#define GDT_ENTRIES ((31 + N_EXCEPTION_TSS + L1_CACHE_BYTES / 8 - 1) & ~(L1_CACHE_BYTES / 8 - 1))
 
 #define GDT_SIZE (GDT_ENTRIES * 8)
 
@@ -92,9 +103,7 @@
 #define __BOOT_DS	(GDT_ENTRY_BOOT_DS * 8)
 
 /*
- * The interrupt descriptor table has room for 256 idt's,
- * the global descriptor table is dependent on the number
- * of tasks we can have..
+ * The interrupt descriptor table has room for 256 idt's.
  */
 #define IDT_ENTRIES 256