Fix 32-bit CPU hotplug issue on AMD
Subsystems (5)
cover Fix 32-bit CPU hotplug issue on AMD
From: Borislav Petkov <[email protected]>
Hi,
the following two patches fix an issue which one of our tests triggers here on a 32-bit kernel while exercising the CPU hotplug path. More details are to be found in the respective commit messages.
Now, considering the seriousness of the issue, I'd very much like to backport the fixes to .32 and .34. And while the fixes are not oneliners, they're straightforward enough to not cause any problems IMHO. But I may be biased from staring at them for a while now so what do you guys, think, any objections to backporting those?
Thanks.
PATCH 1/2 x86-32: Fix crashes with CPU hotplug on AMD machines
From: Joerg Roedel <[email protected]>
This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:
1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.
2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.
3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.
4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.
5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.
6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.
7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.
There are two ways to fix this issue:
1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications
2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.
This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.
--- arch/x86/include/asm/pgtable_32.h | 1 + arch/x86/include/asm/trampoline.h | 3 +++ arch/x86/kernel/head_32.S | 8 +++++++- arch/x86/kernel/setup.c | 2 ++ arch/x86/kernel/smpboot.c | 18 +++--------------- arch/x86/kernel/trampoline.c | 18 ++++++++++++++++++ 6 files changed, 34 insertions(+), 16 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
@@ -26,6 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; +extern pgd_t trampoline_pg_dir[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/trampoline.h +3 -0
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; +extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); +extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else static inline void reserve_trampoline_memory (void ) {}; +extern void __init setup_trampoline_page_table (void ) {};#endif /* CONFIG_X86_TRAMPOLINE */ #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif ENTRY(empty_zero_page) .fill 4096,1,0
arch/x86/kernel/setup.c +2 -0
@@ -1008,6 +1008,8 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); +setup_trampoline_page_table (); +tboot_probe ();#ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +3 -15
@@ -73,7 +73,6 @@#ifdef CONFIG_X86_32 u8 apicid_2_node[MAX_APICID ]; -static int low_mappings;#endif /* State of each CPU */
@@ -300,8 +299,8 @@ notrace static void __cpuinit start_secondary(void *unused) }#ifdef CONFIG_X86_32 -while (low_mappings) -cpu_relax (); +/* switch away from the trampoline page-table */ +load_cr3 (swapper_pg_dir);__flush_tlb_all ();#endif
@@ -754,6 +753,7 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); + initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -894,20 +894,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; -#ifdef CONFIG_X86_32 -/* init low mem mapping */ -clone_pgd_range (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); -flush_tlb_all (); - low_mappings =1 ; - err =do_boot_cpu (apicid, cpu); -zap_low_mappings (false); - low_mappings =0 ; -#else - err =do_boot_cpu (apicid, cpu); -#endif if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +18 -0
@@ -1,6 +1,7 @@#include <linux/io.h> #include <asm/trampoline.h> +#include <asm/pgtable.h> #include <asm/e820.h> #if defined(CONFIG_X86_64 ) && defined(CONFIG_ACPI_SLEEP )
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } + +void __init setup_trampoline_page_table (void ) +{ +#ifdef CONFIG_X86_32 +/* Copy kernel address range */ +clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); + +/* Initialize low mappings */ +clone_pgd_range (trampoline_pg_dir, + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); +#endif +}
Signature · 1 line
1.6.4.4
H. Peter Anvin
reviewerRe: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 10 lines · Borislav Petkov
2. Do not use
swapper_pg_dirto boot secondary CPUs like 64-bit does.This patch implements solution 2. It introduces a
trampoline_pg_dirwhich has the same layout asswapper_pg_dirwithlow_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches toswapper_pg_dirand does a global TLB flush. This fixes the crashes in our test cases.
I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.
arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif
I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes trampoline_pg_dir; we can then set up and change to swapper_pg_dir almost immediately.
I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.
-hpa
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 04, 2010 at 07:05:47PM -0400
Quoted 46 lines · Borislav Petkov
Quoted 10 lines
2. Do not use
swapper_pg_dirto boot secondary CPUs like 64-bit does.This patch implements solution 2. It introduces a
trampoline_pg_dirwhich has the same layout asswapper_pg_dirwithlow_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches toswapper_pg_dirand does a global TLB flush. This fixes the crashes in our test cases.I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.
arch/x86/kernel/head_32.S +7
-1@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax@@ -608,6 +608,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif
Quoted 3 lines
I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes
trampoline_pg_dir; we can then set up and change toswapper_pg_diralmost immediately.
Yeah, now we use swapper_pg_dir at all times and zap the low mappings. However, this is not perfectly clean, as this case in point shows how unrelated CPUs might establish TLB entries speculatively. Now imagine if they don't mcheck about it but silently and merrily continue on. In this particular case, there were no improper accesses due to the user/kernel permissions mismatch but imagine if suddenly kernel code started accessing userspace and this not through copy_to_user() et al.
So it really does make sense to have an initial page table and copy swapper_pg_dir from it. Which would be a perfect exercise for someone who would like to play with the boot code a bit more, ^hint hint^, if Joerg doesn't beat me to it.
But I'd suggest we get those fixes in now if there are no objections and later adjustments should come ontop after excessive testing. And what about backporting those fixes to .32 and .34, would you be ok with that? Greg, what about you?
Thanks.
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
In reply to H. Peter AnvinQuoted 12 lines · H. Peter Anvin
Quoted 10 lines · Borislav Petkov
2. Do not use
swapper_pg_dirto boot secondary CPUs like 64-bit does.This patch implements solution 2. It introduces a
trampoline_pg_dirwhich has the same layout asswapper_pg_dirwithlow_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches toswapper_pg_dirand does a global TLB flush. This fixes the crashes in our test cases.
Quoted 4 lines
I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.
Yeah, the page table with the low mappings is trampoline_pg_dir introduced in this patch :-)
Quoted 29 lines
arch/x86/kernel/head_32.S +7
-1@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax@@ -608,6 +608,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif
Quoted 3 lines
I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes
trampoline_pg_dir; we can then set up and change toswapper_pg_diralmost immediately.
To make sure I understand correctly, you suggest to initialize tramponline_pg_dir in the boot sequence of the first cpu and fork swapper_pg_dir from it later on?
Quoted 2 lines
I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.
Yeah, may make sense. This patch already brings the 32 bit implementation closer to the 64 bit one. On 64 bit things are somewhat simpler because the tramponline page table can be defined at compile-time there (contains only 2 pgd_t entries) while on 32 bit we have to initialize it at runtime.
Joerg
Signature · 5 lines
AMD Operating System Research Center Advanced Micro Devices GmbH Einsteinring 24 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Landkr. Muenchen; Registerger. Muenchen, HRB Nr. 43632
H. Peter Anvin
reviewerRe: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 5 lines · Joerg
To make sure I understand correctly, you suggest to initialize
tramponline_pg_dirin the boot sequence of the first cpu and forkswapper_pg_dirfrom it later on?
Correct.
Quoted 8 lines
Quoted 2 lines
I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.
Yeah, may make sense. This patch already brings the 32 bit implementation closer to the 64 bit one. On 64 bit things are somewhat simpler because the tramponline page table can be defined at compile-time there (contains only 2
pgd_tentries) while on 32 bit we have to initialize it at runtime.
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 4 lines · H. Peter Anvin
On 08/05/2010 12:45 AM, Roedel, Joerg wrote:
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.
Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.
Joerg
H. Peter Anvin
reviewerRe: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 6 lines · Joerg Roedel
Quoted 4 lines · H. Peter Anvin
On 08/05/2010 12:45 AM, Roedel, Joerg wrote:
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.
Quoted 10 lines
Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.
Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 11:34:37AM -0400
Quoted 18 lines · Joerg Roedel
Quoted 6 lines · H. Peter Anvin
Quoted 4 lines · Joerg
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.
Quoted 10 lines
Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.
Quoted 3 lines
Agreed. I do have a concern about the kernel page tables not being synchronized into
trampoline_pg_dir(it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...
I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the stack_start on the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
H. Peter Anvin
reviewerRe: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 10 lines · Borislav Petkov
Quoted 4 lines
Agreed. I do have a concern about the kernel page tables not being synchronized into
trampoline_pg_dir(it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the
stack_starton the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...
Sounds good. This is obviously .37-or-higher material.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 11:47:55AM -0400
Quoted 6 lines · Borislav Petkov
Quoted 5 lines
Quoted 4 lines
Agreed. I do have a concern about the kernel page tables not being synchronized into
trampoline_pg_dir(it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...
Quoted 7 lines
Quoted 5 lines
I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the
stack_starton the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...Sounds good. This is obviously .37-or-higher material.
Definitely, this needs a lot of hammering before going upstream.
So here's what I got so far, I went and changed everything called swapper_* when used to prep the initial page table to initial_*. Then in setup_arch I switch to the swapper_pg_dir after copying the kernel address mappings into it so all other places touching swapper_pg_dir can remain unchanged. Then, initial_page_table is still used for booting the APs.
The nice effect is that we drop all that swsusp_pg_dir for ACPI_SLEEP and unifying the do_boot_cpu() part of start_secondary() even more.
So this boots fine on 32-bit PAE but reboots the machine on !PAE when the AP does
/* Set up the stack pointer */ lss stack_start,%esp pushl $0 popfl
in head_32.S. I've debugged this by adding endless loops in the manner of
@@@ -274,6 -274,6 +274,7 @@@ ENTRY(startup_32_smp
movl %eax,%es
movl %eax,%fs
movl %eax,%gs
+ movl $0xdeadbeef, %ebx
#endif /* CONFIG_SMP */
so that this happens only on the AP and then did
+667:
+ cmpl $0xdeadbeef, %ebx
+ je 667b
+
So when the loop is before the "pushl.. popfl" sequence above, the BSP would print "CPU stuck??" for the AP and continue booting by not bringing the AP online. If I put the endless loop after that sequence, the machine reboots. This is also nicely reproducible in kvm for even faster and safer experimenting with the code.
Thanks.
--
arch/x86/include/asm/pgtable_32.h +1 -0
@@ -26,6 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; +extern pgd_t initial_page_table[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/tlbflush.h +0 -2
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start, flush_tlb_all(); } -extern void zap_low_mappings (bool early); -#endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/kernel/acpi/sleep.c +6 -1
@@ -12,6 +12,11 @@#include <asm/segment.h> #include <asm/desc.h> +#ifdef CONFIG_X86_32 +#include <asm/pgtable.h> +#include <asm/pgtable_32.h> +#endif +#include "realmode/wakeup.h" #include "sleep.h"
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)#ifndef CONFIG_64BIT header->pmode_entry = (u32)&wakeup_pmode_return; - header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET); + header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET); saved_magic =0x12345678 ;#else /* CONFIG_64BIT */ header->trampoline_segment =setup_trampoline () >>4 ;
arch/x86/kernel/head_32.S +21 -19
@@ -177,7 +177,7 @@ default_entry: #ifdef CONFIG_X86_PAE /* - * In PAE mode swapper_pg_dir is statically defined to contain enough + * In PAE mode initial_page_table is statically defined to contain enough * entries to cover the VMSPLIT option (that is the top 1, 2 or 3 * entries). The identity mapping is handled by pointing two PGD * entries to the first kernel PMD.
@@ -191,7 +191,7 @@ default_entry: xorl %ebx,%ebx /* %ebx is kept at zero */ movl $pa(__brk_base), %edi - movl $pa(swapper_pg_pmd), %edx + movl $pa(initial_pg_pmd), %edx movl $PTE_IDENT_ATTR, %eax 10: leal PDE_IDENT_ATTR(%edi),%ecx /* Create PMD entry */
@@ -220,14 +220,14 @@ default_entry: movl %eax, pa(max_pfn_mapped) /* Do early initialization of the fixmap area */ - movl $pa(swapper_pg_fixmap)+PDE_IDENT_ATTR,%eax - movl %eax,pa(swapper_pg_pmd+0x1000*KPMDS-8) + movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax + movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8) #else /* Not PAE */ page_pde_offset = (__PAGE_OFFSET >> 20); movl $pa(__brk_base), %edi - movl $pa(swapper_pg_dir), %edx + movl $pa(initial_page_table), %edx movl $PTE_IDENT_ATTR, %eax 10: leal PDE_IDENT_ATTR(%edi),%ecx /* Create PDE entry */
@@ -251,8 +251,8 @@ page_pde_offset = (__PAGE_OFFSET >> 20); movl %eax, pa(max_pfn_mapped) /* Do early initialization of the fixmap area */ - movl $pa(swapper_pg_fixmap)+PDE_IDENT_ATTR,%eax - movl %eax,pa(swapper_pg_dir+0xffc) + movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax + movl %eax,pa(initial_page_table+0xffc) #endif jmp 3f /*
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl $pa(initial_page_table),%eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax
@@ -615,16 +615,18 @@ ENTRY(initial_code) __PAGE_ALIGNED_BSS .align PAGE_SIZE_asm #ifdef CONFIG_X86_PAE -swapper_pg_pmd: +initial_pg_pmd: .fill 1024*KPMDS,4,0 #else -ENTRY(swapper_pg_dir) +ENTRY(initial_page_table) .fill 1024,4,0 #endif -swapper_pg_fixmap: +initial_pg_fixmap: .fill 1024,4,0 ENTRY(empty_zero_page) .fill 4096,1,0 +ENTRY(swapper_pg_dir) + .fill 1024,4,0 /* * This starts the data section.
@@ -633,20 +635,20 @@ ENTRY(empty_zero_page) __PAGE_ALIGNED_DATA /* Page-aligned for the benefit of paravirt? */ .align PAGE_SIZE_asm -ENTRY(swapper_pg_dir) - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ +ENTRY(initial_page_table) + .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ # if KPMDS == 3 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR),0 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x1000),0 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x2000),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0 # elif KPMDS == 2 .long 0,0 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR),0 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x1000),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0 # elif KPMDS == 1 .long 0,0 .long 0,0 - .long pa(swapper_pg_pmd+PGD_IDENT_ATTR),0 + .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 # else # error "Kernel PMDs should be 1, 2 or 3" # endif
arch/x86/kernel/setup.c +9 -0
@@ -727,6 +727,15 @@ void __init setup_arch(char **cmdline_p)int k8 =0 ;#ifdef CONFIG_X86_32 +/* Copy kernel address range */ +clone_pgd_range (swapper_pg_dir +KERNEL_PGD_BOUNDARY , + initial_page_table +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); + +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +memcpy (&boot_cpu_data, &new_cpu_data,sizeof (new_cpu_data));visws_early_detect ();#else
arch/x86/kernel/smpboot.c +1 -16
@@ -73,7 +73,6 @@#ifdef CONFIG_X86_32 u8 apicid_2_node[MAX_APICID ]; -static int low_mappings;#endif /* State of each CPU */
@@ -300,8 +299,7 @@ notrace static void __cpuinit start_secondary(void *unused) }#ifdef CONFIG_X86_32 -while (low_mappings) -cpu_relax (); +load_cr3 (swapper_pg_dir);__flush_tlb_all ();#endif
@@ -894,20 +892,7 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; -#ifdef CONFIG_X86_32 -/* init low mem mapping */ -clone_pgd_range (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); -flush_tlb_all (); - low_mappings =1 ; - err =do_boot_cpu (apicid, cpu); - -zap_low_mappings (false); - low_mappings =0 ; -#else - err =do_boot_cpu (apicid, cpu); -#endif if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/mm/init_32.c +0 -45
57 lines not on this page · raw message
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
Borislav Petkov
maintainerInitial working version (Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines)
In reply to H. Peter AnvinFrom: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 08:47:55AM -0700
Quoted 7 lines
Quoted 5 lines
I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the
stack_starton the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...Sounds good. This is obviously .37-or-higher material.
Ok, here we go, the version below boots a !PAE and a PAE kernel fine in kvm. The approach taken is, IMHO, the least intrusive in trying to touch code assuming swapper_pg_dir as little as possible and use initial_page_table for boot/acpi sleep/reboot only.
For example, paths like init_memory_mapping -> kernel_physical_mapping_init -> ... which fill in the kernel mappings into swapper_pg_dir are left unchanged by switching to swapper_pg_dir at the beginning of setup_arch() BTW, I guess this could be done even earlier since I copy the mappings from initial_page_table so there's no difference between the two at the switch point.
Later, after paging_init(), synching back all swapper_pg_dir changes into initial_page_table is done for the APs to boot and have a stack mapping in the initial_page_table so that they don't pagefault into a triple fault (this was the problem with the previous version, btw.)
Please take a look and let me know if I've forgotten something before I start hammering on it on a real hardware.
Thanks.
---
arch/x86/include/asm/pgtable_32.h +1 -1
@@ -26,7 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; -extern pgd_t trampoline_pg_dir[1024 ]; +extern pgd_t initial_page_table[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/tlbflush.h +0 -2
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start, flush_tlb_all(); } -extern void zap_low_mappings (bool early); -#endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; -extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); -extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else -static inline void setup_trampoline_page_table (void ) {}static inline void reserve_trampoline_memory (void ) {}#endif /* CONFIG_X86_TRAMPOLINE */
arch/x86/kernel/acpi/sleep.c +6 -1
@@ -12,6 +12,11 @@#include <asm/segment.h> #include <asm/desc.h> +#ifdef CONFIG_X86_32 +#include <asm/pgtable.h> +#include <asm/pgtable_32.h> +#endif +#include "realmode/wakeup.h" #include "sleep.h"
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)#ifndef CONFIG_64BIT header->pmode_entry = (u32)&wakeup_pmode_return; - header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET); + header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET); saved_magic =0x12345678 ;#else /* CONFIG_64BIT */ header->trampoline_segment =setup_trampoline () >>4 ;
arch/x86/kernel/head32.c +1 -0
@@ -17,6 +17,7 @@#include <asm/apic.h> #include <asm/io_apic.h> #include <asm/bios_ebda.h> +#include <asm/tlbflush.h> static void __init i386_default_early_setup (void ) {
arch/x86/kernel/head_32.S +21 -25
112 lines not on this page · raw message
arch/x86/kernel/reboot.c +2 -8
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)CMOS_WRITE (0x00 ,0x8f );spin_unlock (&rtc_lock); -/* Remap the kernel at virtual address zero, as well as offset zero -from the kernel segment. This assumes the kernel segment starts at -virtual address PAGE_OFFSET. */ -memcpy (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , -sizeof (swapper_pg_dir [0 ]) *KERNEL_PGD_PTRS ); -/* -* Use `swapper_pg_dir' as our page directory. +* Switch back to the initial page table. */ -load_cr3 (swapper_pg_dir); +load_cr3 (initial_page_table);/* Write 0x1234 to absolute memory location 0x472. The BIOS reads this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)#ifdef CONFIG_X86_32 memcpy (&boot_cpu_data, &new_cpu_data,sizeof (new_cpu_data));visws_early_detect (); + +/* +* copy kernel address range established so far and switch +* to the proper swapper page table +*/ +clone_pgd_range (swapper_pg_dir +KERNEL_PGD_BOUNDARY , + initial_page_table +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); + +load_cr3 (swapper_pg_dir); +__flush_tlb_all ();#else printk (KERN_INFO "Command line: %s\n" , boot_command_line);#endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); -setup_trampoline_page_table (); +#ifdef CONFIG_X86_32 +/* sync back kernel address range */ +clone_pgd_range (initial_page_table +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); +#endif tboot_probe ();
arch/x86/kernel/smpboot.c +6 -13
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)* most necessary things. */ -#ifdef CONFIG_X86_32 -/* -* Switch away from the trampoline page-table -* -* Do this before cpu_init() because it needs to access per-cpu -* data which may not be mapped in the trampoline page-table. -*/ -load_cr3 (swapper_pg_dir); -__flush_tlb_all (); -#endif -vmi_bringup ();cpu_init ();preempt_disable ();smp_callin (); +#ifdef CONFIG_X86_32 +/* switch away from the initial page table */ +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +#endif +/* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier ();/*
@@ -774,7 +769,6 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); - initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; err =do_boot_cpu (apicid, cpu); -if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +0 -17
@@ -38,20 +38,3 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } - -void __init setup_trampoline_page_table (void ) -{ -#ifdef CONFIG_X86_32 -/* Copy kernel address range */ -clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , - swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, -KERNEL_PGD_BOUNDARY )); - -/* Initialize low mappings */ -clone_pgd_range (trampoline_pg_dir, - swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, -KERNEL_PGD_BOUNDARY )); -#endif -}
arch/x86/mm/init_32.c +0 -45
57 lines not on this page · raw message
Signature · 2 lines
Regards/Gruss,
Boris.Borislav Petkov
maintainerPATCH 1/1 x86-32, mm: Add an initial page table for core bootstrapping
This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to swapper_pg_dir and zap them later or create own swsusp PGD page solely for ACPI sleep needs.
---
Ok, here's the initial version. This patch has survived a whole weekend of randconfig builds and has been boot/reboot/suspend/resume tested on 32-bit kvm guests and bare metal in all possible configurations of PAE and !PAE kernels with all possible VMSPLIT_* config options.
Patch is against tip/x86/urgent ontop of Hugh's CONFIG_VMSPLIT_1G and 2G_OPT fix.
arch/x86/include/asm/pgtable_32.h | 2 +- arch/x86/include/asm/tlbflush.h | 2 - arch/x86/include/asm/trampoline.h | 3 -- arch/x86/kernel/acpi/sleep.c | 7 ++++- arch/x86/kernel/head32.c | 1 + arch/x86/kernel/head_32.S | 55 +++++++++++++++++-------------------- arch/x86/kernel/reboot.c | 10 +----- arch/x86/kernel/setup.c | 18 +++++++++++- arch/x86/kernel/smpboot.c | 19 ++++--------- arch/x86/kernel/trampoline.c | 16 ----------- arch/x86/mm/init_32.c | 45 ------------------------------ 11 files changed, 58 insertions(+), 120 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -1
@@ -26,7 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; -extern pgd_t trampoline_pg_dir[1024 ]; +extern pgd_t initial_page_table[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/tlbflush.h +0 -2
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start, flush_tlb_all(); } -extern void zap_low_mappings (bool early); -#endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; -extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); -extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else -static inline void setup_trampoline_page_table (void ) {}static inline void reserve_trampoline_memory (void ) {}#endif /* CONFIG_X86_TRAMPOLINE */
arch/x86/kernel/acpi/sleep.c +6 -1
@@ -12,6 +12,11 @@#include <asm/segment.h> #include <asm/desc.h> +#ifdef CONFIG_X86_32 +#include <asm/pgtable.h> +#include <asm/pgtable_32.h> +#endif +#include "realmode/wakeup.h" #include "sleep.h"
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)#ifndef CONFIG_64BIT header->pmode_entry = (u32)&wakeup_pmode_return; - header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET); + header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET); saved_magic =0x12345678 ;#else /* CONFIG_64BIT */ header->trampoline_segment =setup_trampoline () >>4 ;
arch/x86/kernel/head32.c +1 -0
@@ -17,6 +17,7 @@#include <asm/apic.h> #include <asm/io_apic.h> #include <asm/bios_ebda.h> +#include <asm/tlbflush.h> static void __init i386_default_early_setup (void ) {
arch/x86/kernel/head_32.S +25 -30
122 lines not on this page · raw message
arch/x86/kernel/reboot.c +2 -8
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)CMOS_WRITE (0x00 ,0x8f );spin_unlock (&rtc_lock); -/* Remap the kernel at virtual address zero, as well as offset zero -from the kernel segment. This assumes the kernel segment starts at -virtual address PAGE_OFFSET. */ -memcpy (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , -sizeof (swapper_pg_dir [0 ]) *KERNEL_PGD_PTRS ); -/* -* Use `swapper_pg_dir' as our page directory. +* Switch back to the initial page table. */ -load_cr3 (swapper_pg_dir); +load_cr3 (initial_page_table);/* Write 0x1234 to absolute memory location 0x472. The BIOS reads this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)#ifdef CONFIG_X86_32 memcpy (&boot_cpu_data, &new_cpu_data,sizeof (new_cpu_data));visws_early_detect (); + +/* +* copy kernel address range established so far and switch +* to the proper swapper page table +*/ +clone_pgd_range (swapper_pg_dir +KERNEL_PGD_BOUNDARY , + initial_page_table +KERNEL_PGD_BOUNDARY , +KERNEL_PGD_PTRS ); + +load_cr3 (swapper_pg_dir); +__flush_tlb_all ();#else printk (KERN_INFO "Command line: %s\n" , boot_command_line);#endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); -setup_trampoline_page_table (); +#ifdef CONFIG_X86_32 +/* sync back kernel address range */ +clone_pgd_range (initial_page_table +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , +KERNEL_PGD_PTRS ); +#endif tboot_probe ();
arch/x86/kernel/smpboot.c +6 -13
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)* most necessary things. */ -#ifdef CONFIG_X86_32 -/* -* Switch away from the trampoline page-table -* -* Do this before cpu_init() because it needs to access per-cpu -* data which may not be mapped in the trampoline page-table. -*/ -load_cr3 (swapper_pg_dir); -__flush_tlb_all (); -#endif -vmi_bringup ();cpu_init ();preempt_disable ();smp_callin (); +#ifdef CONFIG_X86_32 +/* switch away from the initial page table */ +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +#endif +/* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier ();/*
@@ -774,7 +769,6 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); - initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; err =do_boot_cpu (apicid, cpu); -if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +0 -16
@@ -38,19 +38,3 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } - -void __init setup_trampoline_page_table (void ) -{ -#ifdef CONFIG_X86_32 -/* Copy kernel address range */ -clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , - swapper_pg_dir +KERNEL_PGD_BOUNDARY , -KERNEL_PGD_PTRS ); - -/* Initialize low mappings */ -clone_pgd_range (trampoline_pg_dir, - swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, -KERNEL_PGD_BOUNDARY )); -#endif -}
arch/x86/mm/init_32.c +0 -45
57 lines not on this page · raw message
Signature · 5 lines
1.7.1
--
Regards/Gruss,
Boris.Borislav Petkov
maintainerPATCH 1/1 x86-32, mm: Add an initial page table for core bootstrapping
From: Borislav Petkov <[email protected]>
Date: Sun, Aug 29, 2010 at 10:32:05PM +0200
Quoted 8 lines
This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to
swapper_pg_dirand zap them later or create own swsusp PGD page solely for ACPI sleep needs.Signed-off-by Borislav Petkov <[email protected]>---
Quoted 7 lines
Ok, here's the initial version. This patch has survived a whole weekend of randconfig builds and has been boot/reboot/suspend/resume tested on 32-bit kvm guests and bare metal in all possible configurations of PAE and !PAE kernels with all possible VMSPLIT_* config options.
Patch is against tip/x86/urgent ontop of Hugh's
CONFIG_VMSPLIT_1Gand 2G_OPT fix.
Ok this one broke suspend-to-ram due to wrongly _not_ passing the _pointer_ to the initial_page_table as the pmode_cr3 pointer when resuming from ACPI sleep. Here's a better version.
--
From: Borislav Petkov <[email protected]>
Date: Sat, 28 Aug 2010 15:58:33 +0200
Subject: [PATCH] x86-32, mm: Add an initial page table for core bootstrapping
This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to swapper_pg_dir and zap them later or create own swsusp PGD page solely for ACPI sleep needs.
[ v1.1 Fix suspend-to-ram ]
--- arch/x86/include/asm/pgtable_32.h | 2 +- arch/x86/include/asm/tlbflush.h | 2 - arch/x86/include/asm/trampoline.h | 3 -- arch/x86/kernel/acpi/sleep.c | 7 ++++- arch/x86/kernel/head32.c | 1 + arch/x86/kernel/head_32.S | 55 +++++++++++++++++-------------------- arch/x86/kernel/reboot.c | 10 +----- arch/x86/kernel/setup.c | 18 +++++++++++- arch/x86/kernel/smpboot.c | 19 ++++--------- arch/x86/kernel/trampoline.c | 16 ----------- arch/x86/mm/init_32.c | 45 ------------------------------ 11 files changed, 58 insertions(+), 120 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -1
@@ -26,7 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; -extern pgd_t trampoline_pg_dir[1024 ]; +extern pgd_t initial_page_table[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/tlbflush.h +0 -2
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start, flush_tlb_all(); } -extern void zap_low_mappings (bool early); -#endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; -extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); -extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else -static inline void setup_trampoline_page_table (void ) {}static inline void reserve_trampoline_memory (void ) {}#endif /* CONFIG_X86_TRAMPOLINE */
arch/x86/kernel/acpi/sleep.c +6 -1
@@ -12,6 +12,11 @@#include <asm/segment.h> #include <asm/desc.h> +#ifdef CONFIG_X86_32 +#include <asm/pgtable.h> +#include <asm/pgtable_32.h> +#endif +#include "realmode/wakeup.h" #include "sleep.h"
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)#ifndef CONFIG_64BIT header->pmode_entry = (u32)&wakeup_pmode_return; - header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET); + header->pmode_cr3 = (u32 )__pa (&initial_page_table); saved_magic =0x12345678 ;#else /* CONFIG_64BIT */ header->trampoline_segment =setup_trampoline () >>4 ;
arch/x86/kernel/head32.c +1 -0
@@ -17,6 +17,7 @@#include <asm/apic.h> #include <asm/io_apic.h> #include <asm/bios_ebda.h> +#include <asm/tlbflush.h> static void __init i386_default_early_setup (void ) {
arch/x86/kernel/head_32.S +25 -30
122 lines not on this page · raw message
arch/x86/kernel/reboot.c +2 -8
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)CMOS_WRITE (0x00 ,0x8f );spin_unlock (&rtc_lock); -/* Remap the kernel at virtual address zero, as well as offset zero -from the kernel segment. This assumes the kernel segment starts at -virtual address PAGE_OFFSET. */ -memcpy (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , -sizeof (swapper_pg_dir [0 ]) *KERNEL_PGD_PTRS ); -/* -* Use `swapper_pg_dir' as our page directory. +* Switch back to the initial page table. */ -load_cr3 (swapper_pg_dir); +load_cr3 (initial_page_table);/* Write 0x1234 to absolute memory location 0x472. The BIOS reads this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)#ifdef CONFIG_X86_32 memcpy (&boot_cpu_data, &new_cpu_data,sizeof (new_cpu_data));visws_early_detect (); + +/* +* copy kernel address range established so far and switch +* to the proper swapper page table +*/ +clone_pgd_range (swapper_pg_dir +KERNEL_PGD_BOUNDARY , + initial_page_table +KERNEL_PGD_BOUNDARY , +KERNEL_PGD_PTRS ); + +load_cr3 (swapper_pg_dir); +__flush_tlb_all ();#else printk (KERN_INFO "Command line: %s\n" , boot_command_line);#endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); -setup_trampoline_page_table (); +#ifdef CONFIG_X86_32 +/* sync back kernel address range */ +clone_pgd_range (initial_page_table +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , +KERNEL_PGD_PTRS ); +#endif tboot_probe ();
arch/x86/kernel/smpboot.c +6 -13
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)* most necessary things. */ -#ifdef CONFIG_X86_32 -/* -* Switch away from the trampoline page-table -* -* Do this before cpu_init() because it needs to access per-cpu -* data which may not be mapped in the trampoline page-table. -*/ -load_cr3 (swapper_pg_dir); -__flush_tlb_all (); -#endif -vmi_bringup ();cpu_init ();preempt_disable ();smp_callin (); +#ifdef CONFIG_X86_32 +/* switch away from the initial page table */ +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +#endif +/* otherwise gcc will move up smp_processor_id before the cpu_init */ barrier ();/*
@@ -774,7 +769,6 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); - initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; err =do_boot_cpu (apicid, cpu); -if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +0 -16
@@ -38,19 +38,3 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } - -void __init setup_trampoline_page_table (void ) -{ -#ifdef CONFIG_X86_32 -/* Copy kernel address range */ -clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , - swapper_pg_dir +KERNEL_PGD_BOUNDARY , -KERNEL_PGD_PTRS ); - -/* Initialize low mappings */ -clone_pgd_range (trampoline_pg_dir, - swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, -KERNEL_PGD_BOUNDARY )); -#endif -}
arch/x86/mm/init_32.c +0 -45
57 lines not on this page · raw message
Signature · 5 lines
1.7.1
--
Regards/Gruss,
Boris.Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
In reply to H. Peter AnvinQuoted 3 lines · H. Peter Anvin
Agreed. I do have a concern about the kernel page tables not being synchronized into
trampoline_pg_dir(it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...
Great.
The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.
Joerg
H. Peter Anvin
reviewerRe: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 6 lines · Joerg Roedel
Quoted 3 lines · H. Peter Anvin
Agreed. I do have a concern about the kernel page tables not being synchronized into
trampoline_pg_dir(it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...
Quoted 8 lines
Great.
The synchronization problem might be real. The
cpu_initfunction uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change toswapper_pg_dirand the global tlb flush should probably be moved to the very beginning of thestart_secondaryfunction.
Are you going to submit an updated patch, then?
-hpa
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 10 lines · H. Peter Anvin
Quoted 6 lines · Joerg Roedel
The synchronization problem might be real. The
cpu_initfunction uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change toswapper_pg_dirand the global tlb flush should probably be moved to the very beginning of thestart_secondaryfunction.Are you going to submit an updated patch, then?
Yes, I will send it next week when I am back in office and gave it some testing.
Joerg
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: Joerg Roedel <[email protected]>
Date: Thu, Aug 12, 2010 at 03:04:14PM -0400
Quoted 10 lines · H. Peter Anvin
Quoted 9 lines · Joerg Roedel
Quoted 7 lines
The synchronization problem might be real. The
cpu_initfunction uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change toswapper_pg_dirand the global tlb flush should probably be moved to the very beginning of thestart_secondaryfunction.
Quoted 4 lines
Quoted 1 line
Are you going to submit an updated patch, then?
Yes, I will send it next week when I am back in office and gave it some testing.
currently testing...
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: Borislav Petkov <[email protected]>
Date: Fri, Aug 13, 2010 at 08:35:43AM -0400
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
The synchronization problem might be real. The
cpu_initfunction uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change toswapper_pg_dirand the global tlb flush should probably be moved to the very beginning of thestart_secondaryfunction.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Are you going to submit an updated patch, then?
Yes, I will send it next week when I am back in office and gave it some testing.
currently testing...
Updated versions survived almost 48h of uninterrupted hammering, I'm sending them as a reply to this message...
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
PATCH 1/2 x86-32: Fix crashes with CPU hotplug on AMD machines
From: Joerg Roedel <[email protected]>
This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:
1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.
2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.
3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.
4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.
5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.
6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.
7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.
There are two ways to fix this issue:
1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications
2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.
This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.
-v2: switch to swapper_pg_dir right after entering start_secondary() so that we are able to access percpu data which might not be mapped in the trampoline page table.
--- arch/x86/include/asm/pgtable_32.h | 1 + arch/x86/include/asm/trampoline.h | 3 +++ arch/x86/kernel/head_32.S | 8 +++++++- arch/x86/kernel/setup.c | 2 ++ arch/x86/kernel/smpboot.c | 32 +++++++++++++------------------- arch/x86/kernel/trampoline.c | 18 ++++++++++++++++++ 6 files changed, 44 insertions(+), 20 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
@@ -26,6 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; +extern pgd_t trampoline_pg_dir[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/trampoline.h +3 -0
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; +extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); +extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else static inline void reserve_trampoline_memory (void ) {}; +extern void __init setup_trampoline_page_table (void ) {};#endif /* CONFIG_X86_TRAMPOLINE */ #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif ENTRY(empty_zero_page) .fill 4096,1,0
arch/x86/kernel/setup.c +2 -0
@@ -1008,6 +1008,8 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); +setup_trampoline_page_table (); +tboot_probe ();#ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +13 -19
@@ -73,7 +73,6 @@#ifdef CONFIG_X86_32 u8 apicid_2_node[MAX_APICID ]; -static int low_mappings;#endif /* State of each CPU */
@@ -281,6 +280,18 @@ notrace static void __cpuinit start_secondary(void *unused)* fragile that we want to limit the things done here to the * most necessary things. */ + +#ifdef CONFIG_X86_32 +/* +* Switch away from the trampoline page-table +* +* Do this before cpu_init() because it needs to access per-cpu +* data which may not be mapped in the trampoline page-table. +*/ +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +#endif +vmi_bringup ();cpu_init ();preempt_disable ();
@@ -299,12 +310,6 @@ notrace static void __cpuinit start_secondary(void *unused) legacy_pic->chip->unmask (0 ); } -#ifdef CONFIG_X86_32 -while (low_mappings) -cpu_relax (); -__flush_tlb_all (); -#endif -/* This must be done before setting cpu_online_mask */ set_cpu_sibling_map (raw_smp_processor_id ());wmb ();
@@ -754,6 +759,7 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); + initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -894,20 +900,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; -#ifdef CONFIG_X86_32 -/* init low mem mapping */ -clone_pgd_range (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); -flush_tlb_all (); - low_mappings =1 ; - err =do_boot_cpu (apicid, cpu); -zap_low_mappings (false); - low_mappings =0 ; -#else - err =do_boot_cpu (apicid, cpu); -#endif if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +18 -0
@@ -1,6 +1,7 @@#include <linux/io.h> #include <asm/trampoline.h> +#include <asm/pgtable.h> #include <asm/e820.h> #if defined(CONFIG_X86_64 ) && defined(CONFIG_ACPI_SLEEP )
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } + +void __init setup_trampoline_page_table (void ) +{ +#ifdef CONFIG_X86_32 +/* Copy kernel address range */ +clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); + +/* Initialize low mappings */ +clone_pgd_range (trampoline_pg_dir, + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); +#endif +}
Signature · 6 lines
1.7.1 -- Regards/Gruss, Boris.
H. Peter Anvin
reviewerRe: [PATCH -v2 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
Quoted 4 lines · Borislav Petkov
#else
static inline voidreserve_trampoline_memory(void){};
+extern void__initsetup_trampoline_page_table(void){};
#endif /*CONFIG_X86_TRAMPOLINE*/
This breaks compilation in the !CONFIG_X86_TRAMPOLINE case; I presume you meant:
static inline void setup_trampoline_page_table(void) {}
... here too?
(No semicolon after a function body!)
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH -v2 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines
From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 18, 2010 at 02:41:17PM -0400
Quoted 6 lines · Borislav Petkov
Quoted 4 lines
#else
static inline voidreserve_trampoline_memory(void){};
+extern void__initsetup_trampoline_page_table(void){};
#endif /*CONFIG_X86_TRAMPOLINE*/
Quoted 8 lines
This breaks compilation in the !CONFIG_X86_TRAMPOLINE case; I presume you meant:
static inline void
setup_trampoline_page_table(void){}... here too?
(No semicolon after a function body!)
Yeah, thanks for catching that - this should be a static inline stub in the !CONFIG_X86_TRAMPOLINE case.
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
tip-bot for H. Peter Anvin
reviewer[tip:x86/urgent] x86-32: Fix dummy trampoline-related inline stubs
In reply to H. Peter AnvinCommit-ID: 8848a91068c018bc91f597038a0f41462a0f88a4 Gitweb: http://git.kernel.org/tip/8848a91068c018bc91f597038a0f41462a0f88a4 Author: H. Peter Anvin <[email protected]> AuthorDate: Wed, 18 Aug 2010 11:42:23 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Wed, 18 Aug 2010 12:42:24 -0700
x86-32: Fix dummy trampoline-related inline stubs
Fix dummy inline stubs for trampoline-related functions when no trampolines exist (until we get rid of the no-trampoline case entirely.)
LKML-Reference: <[email protected]>
--- arch/x86/include/asm/trampoline.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/include/asm/trampoline.h +2 -2
@@ -22,8 +22,8 @@ extern unsigned long setup_trampoline(void);extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else -static inline void reserve_trampoline_memory (void ) {}; -extern void __init setup_trampoline_page_table (void ) {}; +static inline void setup_trampoline_page_table (void ) {} +static inline void reserve_trampoline_memory (void ) {}#endif /* CONFIG_X86_TRAMPOLINE */ #endif /* __ASSEMBLY__ */
[tip:x86/urgent] x86-32: Separate 1:1 pagetables from swapper_pg_dir
In reply to Borislav PetkovCommit-ID: fd89a137924e0710078c3ae855e7cec1c43cb845 Gitweb: http://git.kernel.org/tip/fd89a137924e0710078c3ae855e7cec1c43cb845 Author: Joerg Roedel <[email protected]> AuthorDate: Mon, 16 Aug 2010 14:38:33 +0200 Committer: H. Peter Anvin <[email protected]> CommitDate: Wed, 18 Aug 2010 09:17:20 -0700
x86-32: Separate 1:1 pagetables from swapper_pg_dir
This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:
1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.
2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.
3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.
4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.
5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.
6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.
7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.
There are two ways to fix this issue:
1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications
2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.
This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.
-v2: switch to swapper_pg_dir right after entering start_secondary() so that we are able to access percpu data which might not be mapped in the trampoline page table.
LKML-Reference: <20100816123833.GB28147@aftab>
--- arch/x86/include/asm/pgtable_32.h | 1 + arch/x86/include/asm/trampoline.h | 3 +++ arch/x86/kernel/head_32.S | 8 +++++++- arch/x86/kernel/setup.c | 2 ++ arch/x86/kernel/smpboot.c | 32 +++++++++++++------------------- arch/x86/kernel/trampoline.c | 18 ++++++++++++++++++ 6 files changed, 44 insertions(+), 20 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
@@ -26,6 +26,7 @@ struct mm_struct;struct vm_area_struct ;extern pgd_t swapper_pg_dir[1024 ]; +extern pgd_t trampoline_pg_dir[1024 ];static inline void pgtable_cache_init (void ) { }static inline void check_pgt_cache (void ) { }
arch/x86/include/asm/trampoline.h +3 -0
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;extern unsigned long init_rsp;extern unsigned long initial_code; +extern unsigned long initial_page_table;extern unsigned long initial_gs;#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)extern unsigned long setup_trampoline (void ); +extern void __init setup_trampoline_page_table (void );extern void __init reserve_trampoline_memory (void );#else static inline void reserve_trampoline_memory (void ) {}; +extern void __init setup_trampoline_page_table (void ) {};#endif /* CONFIG_X86_TRAMPOLINE */ #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
@@ -334,7 +334,7 @@ ENTRY(startup_32_smp) /* * Enable paging */ - movl $pa(swapper_pg_dir),%eax + movl pa(initial_page_table), %eax movl %eax,%cr3 /* set the page table pointer.. */ movl %cr0,%eax orl $X86_CR0_PG,%eax
@@ -614,6 +614,8 @@ ignore_int: .align 4 ENTRY(initial_code) .long i386_start_kernel +ENTRY(initial_page_table) + .long pa(swapper_pg_dir) /* * BSS section
@@ -629,6 +631,10 @@ ENTRY(swapper_pg_dir) #endif swapper_pg_fixmap: .fill 1024,4,0 +#ifdef CONFIG_X86_TRAMPOLINE +ENTRY(trampoline_pg_dir) + .fill 1024,4,0 +#endif ENTRY(empty_zero_page) .fill 4096,1,0
arch/x86/kernel/setup.c +2 -0
@@ -1014,6 +1014,8 @@ void __init setup_arch(char **cmdline_p)paging_init (); x86_init.paging.pagetable_setup_done (swapper_pg_dir); +setup_trampoline_page_table (); +tboot_probe ();#ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +13 -19
@@ -73,7 +73,6 @@#ifdef CONFIG_X86_32 u8 apicid_2_node[MAX_APICID ]; -static int low_mappings;#endif /* State of each CPU */
@@ -281,6 +280,18 @@ notrace static void __cpuinit start_secondary(void *unused)* fragile that we want to limit the things done here to the * most necessary things. */ + +#ifdef CONFIG_X86_32 +/* +* Switch away from the trampoline page-table +* +* Do this before cpu_init() because it needs to access per-cpu +* data which may not be mapped in the trampoline page-table. +*/ +load_cr3 (swapper_pg_dir); +__flush_tlb_all (); +#endif +vmi_bringup ();cpu_init ();preempt_disable ();
@@ -299,12 +310,6 @@ notrace static void __cpuinit start_secondary(void *unused) legacy_pic->chip->unmask (0 ); } -#ifdef CONFIG_X86_32 -while (low_mappings) -cpu_relax (); -__flush_tlb_all (); -#endif -/* This must be done before setting cpu_online_mask */ set_cpu_sibling_map (raw_smp_processor_id ());wmb ();
@@ -750,6 +755,7 @@ do_rest:#ifdef CONFIG_X86_32 /* Stack for startup_32 can be just as for start_secondary onwards */ irq_ctx_init (cpu); + initial_page_table =__pa (&trampoline_pg_dir);#else clear_tsk_thread_flag (c_idle.idle,TIF_FORK ); initial_gs =per_cpu_offset (cpu);
@@ -897,20 +903,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ; -#ifdef CONFIG_X86_32 -/* init low mem mapping */ -clone_pgd_range (swapper_pg_dir, swapper_pg_dir +KERNEL_PGD_BOUNDARY , - min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY)); -flush_tlb_all (); - low_mappings =1 ; - err =do_boot_cpu (apicid, cpu); -zap_low_mappings (false); - low_mappings =0 ; -#else - err =do_boot_cpu (apicid, cpu); -#endif if (err) {pr_debug ("do_boot_cpu failed %d\n" , err);return -EIO ;
arch/x86/kernel/trampoline.c +18 -0
@@ -1,6 +1,7 @@#include <linux/io.h> #include <asm/trampoline.h> +#include <asm/pgtable.h> #include <asm/e820.h> #if defined(CONFIG_X86_64 ) && defined(CONFIG_ACPI_SLEEP )
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)memcpy (trampoline_base, trampoline_data,TRAMPOLINE_SIZE );return virt_to_phys (trampoline_base); } + +void __init setup_trampoline_page_table (void ) +{ +#ifdef CONFIG_X86_32 +/* Copy kernel address range */ +clone_pgd_range (trampoline_pg_dir +KERNEL_PGD_BOUNDARY , + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); + +/* Initialize low mappings */ +clone_pgd_range (trampoline_pg_dir, + swapper_pg_dir +KERNEL_PGD_BOUNDARY , + min_t(unsigned long, KERNEL_PGD_PTRS, +KERNEL_PGD_BOUNDARY )); +#endif +}
PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue
In reply to Borislav PetkovWhen testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.
Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.
--- arch/x86/Kconfig | 6 ++++++ arch/x86/kernel/smpboot.c | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 0 deletions(-)
arch/x86/Kconfig +6 -0
@@ -58,6 +58,7 @@ config X86 select ANON_INODES select HAVE_ARCH_KMEMCHECK select HAVE_USER_RETURN_NOTIFIER + select ARCH_CPU_PROBE_RELEASE config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS)
@@ -247,6 +248,11 @@ config ARCH_HWEIGHT_CFLAGS config KTIME_SCALAR def_bool X86_32 + +config ARCH_CPU_PROBE_RELEASE + def_bool y + depends on HOTPLUG_CPU + source "init/Kconfig" source "kernel/Kconfig.freezer"
arch/x86/kernel/smpboot.c +19 -0
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 }; static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);#define get_idle_for_cpu (x) (per_cpu(idle_thread_array, x))#define set_idle_for_cpu (x, p) (per_cpu(idle_thread_array, x) = (p)) + +/* +* We need this for trampoline_base protection from concurrent accesses when +* off- and onlining cores wildly. +*/ +static DEFINE_MUTEX (x86_cpu_hotplug_driver_mutex); + +void cpu_hotplug_driver_lock () +{ +mutex_lock (&x86_cpu_hotplug_driver_mutex); +} + +void cpu_hotplug_driver_unlock () +{ +mutex_unlock (&x86_cpu_hotplug_driver_mutex); +} + +ssize_t arch_cpu_probe (const char *buf,size_t count) {return -1 ; } +ssize_t arch_cpu_release (const char *buf,size_t count) {return -1 ; }#else static struct task_struct *idle_thread_array[NR_CPUS ]__cpuinitdata ;#define get_idle_for_cpu (x) (idle_thread_array[(x)])
Signature · 6 lines
1.7.1 -- Regards/Gruss, Boris.
H. Peter Anvin
reviewerRe: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue
arch/x86/Kconfig config X86 · line 62
Quoted 11 lines · Borislav Petkov
arch/x86/Kconfig +1
-0@@ -58,6 +58,7 @@ config X86 select ANON_INODES select HAVE_ARCH_KMEMCHECK select HAVE_USER_RETURN_NOTIFIER + select ARCH_CPU_PROBE_RELEASE
warning: (X86) selects ARCH_CPU_PROBE_RELEASE which has unmet direct dependencies (HOTPLUG_CPU)
... and build failure when compiling without CONFIG_HOTPLUG_CPU.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Re: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue
From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 18, 2010 at 03:03:33PM -0400
Quoted 15 lines · Borislav Petkov
arch/x86/Kconfig config X86 · line 62
Quoted 11 lines
arch/x86/Kconfig +1
-0@@ -58,6 +58,7 @@ config X86 select ANON_INODES select HAVE_ARCH_KMEMCHECK select HAVE_USER_RETURN_NOTIFIER + select ARCH_CPU_PROBE_RELEASEwarning: (X86) selects
ARCH_CPU_PROBE_RELEASEwhich has unmet direct dependencies (HOTPLUG_CPU)
well I got
+config ARCH_CPU_PROBE_RELEASE
+ def_bool y
+ depends on HOTPLUG_CPU
but I guess ARCH_CPU_PROBE_RELEASE gets selected still. Why isn't that "depends on" forcing it, I guess select is stronger.
Quoted 1 line
... and build failure when compiling without
CONFIG_HOTPLUG_CPU.
This hunk fixes the !CONFIG_HOTPLUG_CPU build here but I should've caught it earlier, shame on me. Sorry for the inconvenience, can you please drop this second patch for now, I'll send you a revised version tomorrow.
Thanks.
drivers/base/cpu.c +1 -1
@@ -261,7 +261,7 @@ int __init cpu_dev_init(void) } static struct sysdev_class_attribute *cpu_sysdev_class_attrs[] = { -#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE +#if defined(CONFIG_ARCH_CPU_PROBE_RELEASE ) && defined(CONFIG_HOTPLUG_CPU ) &attr_probe, &attr_release,#endif
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
H. Peter Anvin
reviewerRe: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue
Quoted 13 lines · Borislav Petkov
Quoted 3 lines
warning: (X86) selects
ARCH_CPU_PROBE_RELEASEwhich has unmet direct dependencies (HOTPLUG_CPU)well I got
(quoted diff) +3
-0+config ARCH_CPU_PROBE_RELEASE + def_bool y + depends on HOTPLUG_CPUbut I guess
ARCH_CPU_PROBE_RELEASEgets selected still. Why isn't that "depends on" forcing it, I guess select is stronger.
Quoted 7 lines
Quoted 1 line
... and build failure when compiling without
CONFIG_HOTPLUG_CPU.This hunk fixes the !CONFIG_HOTPLUG_CPU build here but I should've caught it earlier, shame on me. Sorry for the inconvenience, can you please drop this second patch for now, I'll send you a revised version tomorrow.
I would prefer to not continue to have a piece of Kconfig code which warns, so I'll hold off on this until I have your revised patch.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue
When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.
Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.
[ v2.1: fix !HOTPLUG_CPU build ]
--- arch/x86/Kconfig | 5 +++++ arch/x86/kernel/smpboot.c | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)
arch/x86/Kconfig +5 -0
@@ -245,6 +245,11 @@ config ARCH_HWEIGHT_CFLAGS config KTIME_SCALAR def_bool X86_32 + +config ARCH_CPU_PROBE_RELEASE + def_bool y + depends on HOTPLUG_CPU + source "init/Kconfig" source "kernel/Kconfig.freezer"
arch/x86/kernel/smpboot.c +19 -0
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 }; static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);#define get_idle_for_cpu (x) (per_cpu(idle_thread_array, x))#define set_idle_for_cpu (x, p) (per_cpu(idle_thread_array, x) = (p)) + +/* +* We need this for trampoline_base protection from concurrent accesses when +* off- and onlining cores wildly. +*/ +static DEFINE_MUTEX (x86_cpu_hotplug_driver_mutex); + +void cpu_hotplug_driver_lock () +{ +mutex_lock (&x86_cpu_hotplug_driver_mutex); +} + +void cpu_hotplug_driver_unlock () +{ +mutex_unlock (&x86_cpu_hotplug_driver_mutex); +} + +ssize_t arch_cpu_probe (const char *buf,size_t count) {return -1 ; } +ssize_t arch_cpu_release (const char *buf,size_t count) {return -1 ; }#else static struct task_struct *idle_thread_array[NR_CPUS ]__cpuinitdata ;#define get_idle_for_cpu (x) (idle_thread_array[(x)])
Signature · 11 lines
1.7.1 -- Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach General Managers: Alberto Bozzo, Andrew Bowd Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632
[tip:x86/urgent] x86, hotplug: Serialize CPU hotplug to avoid bringup concurrency issues
Commit-ID: d7c53c9e822a4fefa13a0cae76f3190bfd0d5c11 Gitweb: http://git.kernel.org/tip/d7c53c9e822a4fefa13a0cae76f3190bfd0d5c11 Author: Borislav Petkov <[email protected]> AuthorDate: Thu, 19 Aug 2010 20:10:29 +0200 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 19 Aug 2010 14:47:43 -0700
x86, hotplug: Serialize CPU hotplug to avoid bringup concurrency issues
When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.
Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.
[ v2.1: fix !HOTPLUG_CPU build ]
LKML-Reference: <20100819181029.GC17171@aftab>
--- arch/x86/Kconfig | 5 +++++ arch/x86/kernel/smpboot.c | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)
arch/x86/Kconfig +5 -0
@@ -245,6 +245,11 @@ config ARCH_HWEIGHT_CFLAGS config KTIME_SCALAR def_bool X86_32 + +config ARCH_CPU_PROBE_RELEASE + def_bool y + depends on HOTPLUG_CPU + source "init/Kconfig" source "kernel/Kconfig.freezer"
arch/x86/kernel/smpboot.c +19 -0
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 }; static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);#define get_idle_for_cpu (x) (per_cpu(idle_thread_array, x))#define set_idle_for_cpu (x, p) (per_cpu(idle_thread_array, x) = (p)) + +/* +* We need this for trampoline_base protection from concurrent accesses when +* off- and onlining cores wildly. +*/ +static DEFINE_MUTEX (x86_cpu_hotplug_driver_mutex); + +void cpu_hotplug_driver_lock () +{ +mutex_lock (&x86_cpu_hotplug_driver_mutex); +} + +void cpu_hotplug_driver_unlock () +{ +mutex_unlock (&x86_cpu_hotplug_driver_mutex); +} + +ssize_t arch_cpu_probe (const char *buf,size_t count) {return -1 ; } +ssize_t arch_cpu_release (const char *buf,size_t count) {return -1 ; }#else static struct task_struct *idle_thread_array[NR_CPUS ]__cpuinitdata ;#define get_idle_for_cpu (x) (idle_thread_array[(x)])
PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue
In reply to Borislav PetkovFrom: Borislav Petkov <[email protected]>
When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.
Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.
--- arch/x86/Kconfig | 6 ++++++ arch/x86/kernel/smpboot.c | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 0 deletions(-)
arch/x86/Kconfig +6 -0
@@ -58,6 +58,7 @@ config X86 select ANON_INODES select HAVE_ARCH_KMEMCHECK select HAVE_USER_RETURN_NOTIFIER + select ARCH_CPU_PROBE_RELEASE config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS)
@@ -247,6 +248,11 @@ config ARCH_HWEIGHT_CFLAGS config KTIME_SCALAR def_bool X86_32 + +config ARCH_CPU_PROBE_RELEASE + def_bool y + depends on HOTPLUG_CPU + source "init/Kconfig" source "kernel/Kconfig.freezer"
arch/x86/kernel/smpboot.c +19 -0
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 }; static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);#define get_idle_for_cpu (x) (per_cpu(idle_thread_array, x))#define set_idle_for_cpu (x, p) (per_cpu(idle_thread_array, x) = (p)) + +/* +* We need this for trampoline_base protection from concurrent accesses when +* off- and onlining cores wildly. +*/ +static DEFINE_MUTEX (x86_cpu_hotplug_driver_mutex); + +void cpu_hotplug_driver_lock () +{ +mutex_lock (&x86_cpu_hotplug_driver_mutex); +} + +void cpu_hotplug_driver_unlock () +{ +mutex_unlock (&x86_cpu_hotplug_driver_mutex); +} + +ssize_t arch_cpu_probe (const char *buf,size_t count) {return -1 ; } +ssize_t arch_cpu_release (const char *buf,size_t count) {return -1 ; }#else static struct task_struct *idle_thread_array[NR_CPUS ]__cpuinitdata ;#define get_idle_for_cpu (x) (idle_thread_array[(x)])
Signature · 1 line
1.6.4.4