Skip to contentSkip to conversation

x86 page fault handler not interrupt safe

PATCH In review Brian Gerst 5 participants 13 messages May 6, 2001 – May 8, 2001 Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86 page fault handler not interrupt safe

Currently the page fault handler on the x86 can get a clobbered value for %cr2 if an interrupt occurs and causes another page fault (interrupt handler touches a vmalloced area for example) before %cr2 is read. This patch changes the page fault and alignment check (currently unused) handlers to interrupt gates so that %cr2 can be read before an interrupt can occur. I'm not certain how much of a problem this really is, but I suspect it could cause random seg faults to user space under heavy interrupt load. Comments are welcome.

arch/i386/kernel/entry.S +13 -5
diff -urN linux-2.4.5-pre1/arch/i386/kernel/entry.S linux/arch/i386/kernel/entry.S
@@ -296,20 +296,21 @@
 	pushl %ds
 	pushl %eax
 	xorl %eax,%eax
+error_code_cr2:
 	pushl %ebp
 	pushl %edi
 	pushl %esi
 	pushl %edx
-	decl %eax			# eax = -1
 	pushl %ecx
 	pushl %ebx
 	cld
 	movl %es,%ecx
 	movl ORIG_EAX(%esp), %esi	# get the error code
 	movl ES(%esp), %edi		# get the function address
-	movl %eax, ORIG_EAX(%esp)
+	movl $-1, ORIG_EAX(%esp)
 	movl %ecx, ES(%esp)
 	movl %esp,%edx
+	pushl %eax			# push address (cr2)
 	pushl %esi			# push the error code
 	pushl %edx			# push the pt_regs pointer
 	movl $(__KERNEL_DS),%edx
@@ -317,7 +318,7 @@
 	movl %edx,%es
 	GET_CURRENT(%ebx)
 	call *%edi
-	addl $8,%esp
+	addl $12,%esp
 	jmp ret_from_exception
 
 ENTRY(coprocessor_error)
@@ -405,11 +406,18 @@
 
 ENTRY(alignment_check)
 	pushl $ SYMBOL_NAME(do_alignment_check)
-	jmp error_code
+	jmp get_cr2
 
 ENTRY(page_fault)
 	pushl $ SYMBOL_NAME(do_page_fault)
-	jmp error_code
+get_cr2:
+	pushl %ds
+	pushl %eax
+	movl %cr2,%eax
+	testl $IF_MASK,EFLAGS-EAX(%esp)
+	jz error_code_cr2
+	sti
+	jmp error_code_cr2
 
 ENTRY(machine_check)
 	pushl $0
arch/i386/kernel/traps.c +7 -16
diff -urN linux-2.4.5-pre1/arch/i386/kernel/traps.c linux/arch/i386/kernel/traps.c
@@ -225,15 +225,6 @@
 		die(str, regs, err);
 }
 
-static inline unsigned long get_cr2(void)
-{
-	unsigned long address;
-
-	/* get the address */
-	__asm__("movl %%cr2,%0":"=r" (address));
-	return address;
-}
-
 static void inline do_trap(int trapnr, int signr, char *str, int vm86,
 			   struct pt_regs * regs, long error_code, siginfo_t *info)
 {
@@ -270,13 +261,13 @@
 }
 
 #define DO_ERROR(trapnr, signr, str, name) \
-asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
+asmlinkage void do_##name(struct pt_regs * regs, long error_code, unsigned long address) \
 { \
 	do_trap(trapnr, signr, str, 0, regs, error_code, NULL); \
 }
 
 #define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
-asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
+asmlinkage void do_##name(struct pt_regs * regs, long error_code, unsigned long address) \
 { \
 	siginfo_t info; \
 	info.si_signo = signr; \
@@ -287,13 +278,13 @@
 }
 
 #define DO_VM86_ERROR(trapnr, signr, str, name) \
-asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
+asmlinkage void do_##name(struct pt_regs * regs, long error_code, unsigned long address) \
 { \
 	do_trap(trapnr, signr, str, 1, regs, error_code, NULL); \
 }
 
 #define DO_VM86_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
-asmlinkage void do_##name(struct pt_regs * regs, long error_code) \
+asmlinkage void do_##name(struct pt_regs * regs, long error_code, unsigned long address) \
 { \
 	siginfo_t info; \
 	info.si_signo = signr; \
@@ -314,7 +305,7 @@
 DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS)
 DO_ERROR(11, SIGBUS,  "segment not present", segment_not_present)
 DO_ERROR(12, SIGBUS,  "stack segment", stack_segment)
-DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, get_cr2())
+DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, address)
 
 asmlinkage void do_general_protection(struct pt_regs * regs, long error_code)
 {
@@ -973,10 +964,10 @@
 	set_trap_gate(11,&segment_not_present);
 	set_trap_gate(12,&stack_segment);
 	set_trap_gate(13,&general_protection);
-	set_trap_gate(14,&page_fault);
+	set_intr_gate(14,&page_fault);
 	set_trap_gate(15,&spurious_interrupt_bug);
 	set_trap_gate(16,&coprocessor_error);
-	set_trap_gate(17,&alignment_check);
+	set_intr_gate(17,&alignment_check);
 	set_trap_gate(18,&machine_check);
 	set_trap_gate(19,&simd_coprocessor_error);
 
arch/i386/mm/fault.c +1 -5
diff -urN linux-2.4.5-pre1/arch/i386/mm/fault.c linux/arch/i386/mm/fault.c
@@ -103,19 +103,15 @@
  *	bit 1 == 0 means read, 1 means write
  *	bit 2 == 0 means kernel, 1 means user-mode
  */
-asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
+asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 {
 	struct task_struct *tsk;
 	struct mm_struct *mm;
 	struct vm_area_struct * vma;
-	unsigned long address;
 	unsigned long page;
 	unsigned long fixup;
 	int write;
 	siginfo_t info;
-
-	/* get the address */
-	__asm__("movl %%cr2,%0":"=r" (address));
 
 	tsk = current;
 
#1 lore raw
Quoted 4 lines · Brian Gerst

Currently the page fault handler on the x86 can get a clobbered value for %cr2 if an interrupt occurs and causes another page fault (interrupt handler touches a vmalloced area for example) before %cr2 is read.

That should be ok.

Yes, we'll get a clobbered value, but we'll get a _valid_ clobbered value, and we'll just end up doing the fixups twice (and returning to the user process that didn't get the page it wanted, which will end up re-doing the page fault).

[ Looks closer.. ]

Actually, the second time we'd do the fixup we'd be unhappy, because it has already been done. That test should probably be removed. Hmm.

Hmm.. The threading people wanted this same thing. Maybe we should just make it so.

Linus

#2 lore raw
Quoted 16 lines · Linus Torvalds
Quoted 4 lines · Brian Gerst

Currently the page fault handler on the x86 can get a clobbered value for %cr2 if an interrupt occurs and causes another page fault (interrupt handler touches a vmalloced area for example) before %cr2 is read.

That should be ok.

Yes, we'll get a clobbered value, but we'll get a _valid_ clobbered value, and we'll just end up doing the fixups twice (and returning to the user process that didn't get the page it wanted, which will end up re-doing the page fault).

[ Looks closer.. ]

Quoted 7 lines

Actually, the second time we'd do the fixup we'd be unhappy, because it has already been done. That test should probably be removed. Hmm.

Hmm.. The threading people wanted this same thing. Maybe we should just make it so.

Linus

I think it's better to be on the side of correctness. I designed the patch to have interrupts disabled for the minimum time possible, so there should be nearly no impact.

--
Brian Gerst

#3 lore raw
In reply to Linus Torvalds
Quoted 4 lines

Yes, we'll get a clobbered value, but we'll get a _valid_ clobbered value, and we'll just end up doing the fixups twice (and returning to the user process that didn't get the page it wanted, which will end up re-doing the page fault).

I dont see that we will get a valid value in both cases.

get_user
fault - set %cr2
IRQ vmalloc fault
set %cr2 fixup runs
end IRQ
cr2 is corrupt

At this point the Linus tree suffers from one problem because it gets parallel fixups wrong, and the -ac tree suffers from a different problem because it gets parallel fixups right and to handle that case wont allow a vmalloc fixup on a fault from userspace (or you get infinite loops)

Quoted 4 lines

[ Looks closer.. ]

Actually, the second time we'd do the fixup we'd be unhappy, because it has already been done. That test should probably be removed. Hmm.

There are a whole set of races with the vmalloc fixups. Most are I think fixed in the -ac bits if you want a look (I've not submitted them as they are not too pretty). What I don't currently see is how you handle this without looping forever or getting the SMP race fixup wrong.

(The current -ac fix for the double vmalloc races is below. WP test makes it more complex than is nice)

usr/src/LINUS/LINUX2.4/linux.ac/arch/i386/mm/fault.c renamed from usr/src/LINUS/LINUX2.4/linux.245p1/arch/i386/mm/fault.c +31 -2
@@ -127,8 +183,11 @@
 	 * be in an interrupt or a critical region, and should
 	 * only copy the information from the master page table,
 	 * nothing more.
+	 *
+	 * Handle kernel vmalloc fill in faults. User space doesn't take
+	 * this path. It isnt permitted to go poking up there.
 	 */
-	if (address >= TASK_SIZE)
+	if (address >= TASK_SIZE && !(error_code & 4))
 		goto vmalloc_fault;
 
 	mm = tsk->mm;
@@ -325,7 +378,11 @@
 		 *
 		 * Do _not_ use "tsk" here. We might be inside
 		 * an interrupt in the middle of a task switch..
+		 *
+		 * Note. There is 1 gotcha here. We may be doing the WP
+		 * test. If so then fixing the pgd/pmd won't help.
 		 */
+		 
 		int offset = __pgd_offset(address);
 		pgd_t *pgd, *pgd_k;
 		pmd_t *pmd, *pmd_k;
@@ -344,7 +401,29 @@
 		pmd = pmd_offset(pgd, address);
 		pmd_k = pmd_offset(pgd_k, address);
 
-		if (pmd_present(*pmd) || !pmd_present(*pmd_k))
+		/* If the pmd is present then we either have two cpus trying
+		 * to fill in the vmalloc entries at once, or we have an
+		 * exception. We can treat the collision as a slow path without
+		 * worry. Its incredibly incredibly rare
+		 *
+		 * If the pte is read only then we know its a fault and we must
+		 * exception or Oops as it would loop forever otherwise
+		 */
+		 
+		if (pmd_present(*pmd))
+		{
+			pte_t *ptep = pte_offset(pmd, address);
+			if ((error_code & 2) && !pte_write(*ptep))
+			{
+				if ((fixup = search_exception_table(regs->eip)) != 0) {
+					regs->eip = fixup;
+					return;
+				}
+				goto bad_area_nosemaphore;
+			}
+			/* SMP race.. continue */
+		}
+		if (!pmd_present(*pmd_k))
 			goto bad_area_nosemaphore;
 		set_pmd(pmd, *pmd_k);
 		return;
#4 lore raw
Quoted 2 lines · Alan Cox

(The current -ac fix for the double vmalloc races is below. WP test makes it more complex than is nice)

WP test is easy to handle. Just filter out protection violations and only take the vmalloc path if the page was not found.

- if (address >= TASK_SIZE && !(error_code & 4))
+ if (address >= TASK_SIZE && !(error_code & 5))

--

Brian Gerst

#5 lore raw

Review

Quoted 9 lines
Quoted 2 lines · Alan Cox

(The current -ac fix for the double vmalloc races is below. WP test makes it more complex than is nice)

WP test is easy to handle. Just filter out protection violations and only take the vmalloc path if the page was not found.

(quoted diff) +1 -1
-       if (address >= TASK_SIZE && !(error_code & 4))
+       if (address >= TASK_SIZE && !(error_code & 5))

That is nice. I hadn't thought about doing it that way. It still has the problem if %cr2 is corrupted by a vmalloc fault but it cleans up my other code paths nicely.

Thanks

#6 lore raw
Quoted 4 lines · Alan Cox

That is nice. I hadn't thought about doing it that way. It still has the problem if %cr2 is corrupted by a vmalloc fault but it cleans up my other code paths nicely.

See about "corruption" in previous email. It doesn't exist.

For better debugging, we should probably walk the whole init_mm page table tree when we take the fault, so this patch does that too: it unconditionally copies the init_mm page table entries into the current page table, while at the same time checking that they exist (including the very last level that we didn't use to check at all).

This means that if you access one page past a vmalloc'ed area, you will get a nice oops instead of endless page faults that will fix up the page tables with mappings that already exist.

Untested.

In particular, does anybody have a buggy Pentium to test with the F0 0F lock-up bug? It _should_ be caught with the error-code test (it's a protection fault, not a non-present fault and thus the F0 0F case never enters the vmalloc path), but it's been several years since the thing..

If anybody has such a beast, please try this kernel patch _and_ running the F0 0F bug-producing program (search for it on the 'net - it must be out there somewhere) to verify that the code still correctly handles that case.

Linus

-----

arch/i386/mm/fault.c renamed from linux/arch/i386/mm/fault.c +21 -13
@@ -127,8 +127,12 @@
 	 * be in an interrupt or a critical region, and should
 	 * only copy the information from the master page table,
 	 * nothing more.
+	 *
+	 * This verifies that the fault happens in kernel space
+	 * (error_code & 4) == 0, and that the fault was not a
+	 * protection error (error_code & 1) == 0.
 	 */
-	if (address >= TASK_SIZE)
+	if (address >= TASK_SIZE && !(error_code & 5))
 		goto vmalloc_fault;
 
 	mm = tsk->mm;
@@ -224,7 +228,6 @@
 bad_area:
 	up_read(&mm->mmap_sem);
 
-bad_area_nosemaphore:
 	/* User mode accesses just cause a SIGSEGV */
 	if (error_code & 4) {
 		tsk->thread.cr2 = address;
@@ -322,27 +325,32 @@
 		/*
 		 * Synchronize this task's top level page-table
 		 * with the 'reference' page table.
+		 *
+		 * Do _not_ use "tsk" here. We might be inside
+		 * an interrupt in the middle of a task switch..
 		 */
 		int offset = __pgd_offset(address);
 		pgd_t *pgd, *pgd_k;
 		pmd_t *pmd, *pmd_k;
+		pte_t *pte_k;
 
-		pgd = tsk->active_mm->pgd + offset;
+		asm("movl %%cr3,%0":"=r" (pgd));
+		pgd = offset + (pgd_t *)__va(pgd);
 		pgd_k = init_mm.pgd + offset;
 
-		if (!pgd_present(*pgd)) {
-			if (!pgd_present(*pgd_k))
-				goto bad_area_nosemaphore;
-			set_pgd(pgd, *pgd_k);
-			return;
-		}
-
+		if (!pgd_present(*pgd_k))
+			goto no_context;
+		set_pgd(pgd, *pgd_k);
+		
 		pmd = pmd_offset(pgd, address);
 		pmd_k = pmd_offset(pgd_k, address);
-
-		if (pmd_present(*pmd) || !pmd_present(*pmd_k))
-			goto bad_area_nosemaphore;
+		if (!pmd_present(*pmd_k))
+			goto no_context;
 		set_pmd(pmd, *pmd_k);
+
+		pte_k = pte_offset(pmd_k, address);
+		if (!pte_present(*pte_k))
+			goto no_context;
 		return;
 	}
 }
#7 lore raw
Quoted 4 lines · [email protected] said:

If anybody has such a beast, please try this kernel patch _and_
running the F0 0F bug-producing program (search for it on the 'net - it must be out there somewhere) to verify that the code still correctly handles that case.

Something along the lines of:

echo "unsigned long main=0xf00fc7c8;" > f00fbug.c ; make f00fbug

--
dwmw2

#8 lore raw
In reply to Linus Torvalds
Quoted 15 lines · Linus Torvalds
Quoted 4 lines · Alan Cox

That is nice. I hadn't thought about doing it that way. It still has the problem if %cr2 is corrupted by a vmalloc fault but it cleans up my other code paths nicely.

See about "corruption" in previous email. It doesn't exist.

For better debugging, we should probably walk the whole init_mm page table tree when we take the fault, so this patch does that too: it unconditionally copies the init_mm page table entries into the current page table, while at the same time checking that they exist (including the very last level that we didn't use to check at all).

Quoted 3 lines

This means that if you access one page past a vmalloc'ed area, you will get a nice oops instead of endless page faults that will fix up the page tables with mappings that already exist.

This patch will still cause the user process to seg fault: The error code on the stack will not match the address in %cr2.

user fault (cr2=useraddr, error_code=5 or 7) interrupt
vmalloc fault (cr2=vmallocaddr, error_code=0 or 2)
handle vmalloc fault
iret
iret
handle user fault (cr2=vmallocaddr, error_code=5 or 7)

We then fall down to find_vma() which will fail and then send SIGSEGV to the user process.

--

Brian Gerst

#9 lore raw
Quoted 3 lines · Brian Gerst

This patch will still cause the user process to seg fault: The error code on the stack will not match the address in %cr2.

You've convinced me. Good thinking. Let's do the irq thing.

Linus

#10 lore raw
In reply to Linus Torvalds
Quoted 11 lines · Linus Torvalds

In particular, does anybody have a buggy Pentium to test with the F0 0F lock-up bug? It _should_ be caught with the error-code test (it's a protection fault, not a non-present fault and thus the F0 0F case never enters the vmalloc path), but it's been several years since the thing..

If anybody has such a beast, please try this kernel patch _and_ running the F0 0F bug-producing program (search for it on the 'net - it must be out there somewhere) to verify that the code still correctly handles that case.

I have a Thinkpad with a buggy pentium (see cat /proc/cpuinfo below) and I tried running the F00F test program available from http://lwn.net/2001/0329/a/ltp-f00f.php3 first on a 2.2.17 kernel that I've been running for ages without problems I got this output:

Testing for proper f00f instruction handling.
SIGILL received from f00f instruction. Good.

Then I tried 2.4.4 with your patch applied and got the same output (and no lockup), so according to that test program your patch does not break the F00F handling code. :-)

If you want me to test other patches, just let me know and I'll be happy to do so!

$ cat /proc/cpuinfo
processor       : 0
vendor_id       : GenuineIntel
cpu family      : 5
model           : 8
model name      : Mobile Pentium MMX
stepping        : 1
cpu MHz         : 232.111
fdiv_bug        : no
hlt_bug         : no
sep_bug         : no
f00f_bug        : yes
coma_bug        : no
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr mce cx8 mmx
bogomips        : 463.67

Best regards,
Jesper Juhl - [email protected]

#11 lore raw
In reply to Linus Torvalds
Quoted 2 lines

If anybody has such a beast, please try this kernel patch _and_ running the F0 0F bug-producing program (search for it on the 'net - it must be

One apparent problem with this implementation

arch/i386/mm/fault.c line 136

Quoted 8 lines
arch/i386/mm/fault.c renamed from linux/arch/i386/mm/fault.c +5 -1
+	 *
+	 * This verifies that the fault happens in kernel space
+	 * (error_code & 4) == 0, and that the fault was not a
+	 * protection error (error_code & 1) == 0.
 	 */
-	if (address >= TASK_SIZE)
+	if (address >= TASK_SIZE && !(error_code & 5))
 		goto vmalloc_fault;

address might be from the following vmalloc fault. The error code would indicate user space, so we would do a bogus user space fix up for vmalloc space, fault and die.

#12 lore raw
In reply to Alan Cox
Quoted 12 lines · Alan Cox

I dont see that we will get a valid value in both cases.

get_user
fault - set %cr2
IRQ vmalloc fault
set %cr2 fixup runs
end IRQ
cr2 is corrupt

Wrong. "%cr2" is _not_ "corrupt". It has a well-defined value.

So what happens is

get_user (or user-mode access)
fault - set %cr2 to fault1
irq
vmalloc fault - set %cr2 to fault2
fixup runs, iret
irq runs, iret
%cr2 is still %fault2
vmalloc fault - nothing to do "false fixup" runs, iret
get_user (or user-mode access)
fault - set %cr2 to fault1
... get the right behaviour now ...

Quoted 1 line

There are a whole set of races with the vmalloc fixups.

As far as I can tell, there are no races anywhere. Just silly bugs that are hard to see.

Linus