Skip to contentSkip to conversation

Fix 32-bit CPU hotplug issue on AMD

PATCH In review Borislav Petkov 5 participants 31 messages Aug 4, 2010 – Sep 2, 2010

Subsystems (5)
x86 ACPI Driver core x86 MM +1 more subsystems
lore
Share & tools
lore
#0 lore raw

cover Fix 32-bit CPU hotplug issue on AMD

From: Borislav Petkov <[email protected]>

Hi,

the following two patches fix an issue which one of our tests triggers here on a 32-bit kernel while exercising the CPU hotplug path. More details are to be found in the respective commit messages.

Now, considering the seriousness of the issue, I'd very much like to backport the fixes to .32 and .34. And while the fixes are not oneliners, they're straightforward enough to not cause any problems IMHO. But I may be biased from staring at them for a while now so what do you guys, think, any objections to backporting those?

Thanks.

#1 lore raw

PATCH 1/2 x86-32: Fix crashes with CPU hotplug on AMD machines

From: Joerg Roedel <[email protected]>

This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:

1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.

2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.

3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.

4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.

5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.

6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.

7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.

There are two ways to fix this issue:

1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

---
 arch/x86/include/asm/pgtable_32.h |    1 +
 arch/x86/include/asm/trampoline.h |    3 +++
 arch/x86/kernel/head_32.S         |    8 +++++++-
 arch/x86/kernel/setup.c           |    2 ++
 arch/x86/kernel/smpboot.c         |   18 +++---------------
 arch/x86/kernel/trampoline.c      |   18 ++++++++++++++++++
 6 files changed, 34 insertions(+), 16 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
index 2984a25..f686f49 100644
@@ -26,6 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
+extern pgd_t trampoline_pg_dir[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/trampoline.h +3 -0
index cb507bb..8f78fdf 100644
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
+extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
+extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
 static inline void reserve_trampoline_memory(void) {};
+extern void __init setup_trampoline_page_table(void) {};
 #endif /* CONFIG_X86_TRAMPOLINE */
 
 #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
index 37c3d4b..75e3981 100644
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 ENTRY(empty_zero_page)
 	.fill 4096,1,0
 
arch/x86/kernel/setup.c +2 -0
index b4ae4ac..6600cfd 100644
@@ -1008,6 +1008,8 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
+	setup_trampoline_page_table();
+
 	tboot_probe();
 
 #ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +3 -15
index c4f33b2..6859a42 100644
@@ -73,7 +73,6 @@
 
 #ifdef CONFIG_X86_32
 u8 apicid_2_node[MAX_APICID];
-static int low_mappings;
 #endif
 
 /* State of each CPU */
@@ -300,8 +299,8 @@ notrace static void __cpuinit start_secondary(void *unused)
 	}
 
 #ifdef CONFIG_X86_32
-	while (low_mappings)
-		cpu_relax();
+	/* switch away from the trampoline page-table */
+	load_cr3(swapper_pg_dir);
 	__flush_tlb_all();
 #endif
 
@@ -754,6 +753,7 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
+	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -894,20 +894,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
-#ifdef CONFIG_X86_32
-	/* init low mem mapping */
-	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-	flush_tlb_all();
-	low_mappings = 1;
-
 	err = do_boot_cpu(apicid, cpu);
 
-	zap_low_mappings(false);
-	low_mappings = 0;
-#else
-	err = do_boot_cpu(apicid, cpu);
-#endif
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +18 -0
index c652ef6..a874495 100644
@@ -1,6 +1,7 @@
 #include <linux/io.h>
 
 #include <asm/trampoline.h>
+#include <asm/pgtable.h>
 #include <asm/e820.h>
 
 #if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP)
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
+
+void __init setup_trampoline_page_table(void)
+{
+#ifdef CONFIG_X86_32
+	/* Copy kernel address range */
+	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+
+	/* Initialize low mappings */
+	clone_pgd_range(trampoline_pg_dir,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+#endif
+}
Signature · 1 line
1.6.4.4
#2 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 10 lines · Borislav Petkov

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.

arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 

I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes trampoline_pg_dir; we can then set up and change to swapper_pg_dir almost immediately.

I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.

-hpa

#3 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 04, 2010 at 07:05:47PM -0400

Quoted 46 lines · Borislav Petkov
Quoted 10 lines

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.

arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 
Quoted 3 lines

I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes trampoline_pg_dir; we can then set up and change to swapper_pg_dir almost immediately.

Yeah, now we use swapper_pg_dir at all times and zap the low mappings. However, this is not perfectly clean, as this case in point shows how unrelated CPUs might establish TLB entries speculatively. Now imagine if they don't mcheck about it but silently and merrily continue on. In this particular case, there were no improper accesses due to the user/kernel permissions mismatch but imagine if suddenly kernel code started accessing userspace and this not through copy_to_user() et al.

So it really does make sense to have an initial page table and copy swapper_pg_dir from it. Which would be a perfect exercise for someone who would like to play with the boot code a bit more, ^hint hint^, if Joerg doesn't beat me to it.

But I'd suggest we get those fixes in now if there are no objections and later adjustments should come ontop after excessive testing. And what about backporting those fixes to .32 and .34, would you be ok with that? Greg, what about you?

Thanks.

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#4 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

In reply to H. Peter Anvin
Quoted 12 lines · H. Peter Anvin
Quoted 10 lines · Borislav Petkov

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

Quoted 4 lines

I would like to keep around a page directory with the low mappings around -- and not use it for kernel threads -- at all times *anyway*. This means we can remove any current hacks that we have to do around S3 entry and exit, for example.

Yeah, the page table with the low mappings is trampoline_pg_dir introduced in this patch :-)

Quoted 29 lines
arch/x86/kernel/head_32.S +7 -1
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 
Quoted 3 lines

I don't really see why this makes sense, though. It would make more sense that the initial page table we set up becomes trampoline_pg_dir; we can then set up and change to swapper_pg_dir almost immediately.

To make sure I understand correctly, you suggest to initialize tramponline_pg_dir in the boot sequence of the first cpu and fork swapper_pg_dir from it later on?

Quoted 2 lines

I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.

Yeah, may make sense. This patch already brings the 32 bit implementation closer to the 64 bit one. On 64 bit things are somewhat simpler because the tramponline page table can be defined at compile-time there (contains only 2 pgd_t entries) while on 32 bit we have to initialize it at runtime.

Joerg

Signature · 5 lines
AMD Operating System Research Center

Advanced Micro Devices GmbH Einsteinring 24 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Landkr. Muenchen; Registerger. Muenchen, HRB Nr. 43632
#5 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 5 lines · Joerg

To make sure I understand correctly, you suggest to initialize tramponline_pg_dir in the boot sequence of the first cpu and fork swapper_pg_dir from it later on?

Correct.

Quoted 8 lines
Quoted 2 lines

I realize this isn't how the 64-bit code works at the moment, but in a lot of ways I think it would be better if it did.

Yeah, may make sense. This patch already brings the 32 bit implementation closer to the 64 bit one. On 64 bit things are somewhat simpler because the tramponline page table can be defined at compile-time there (contains only 2 pgd_t entries) while on 32 bit we have to initialize it at runtime.

Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#6 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 4 lines · H. Peter Anvin

On 08/05/2010 12:45 AM, Roedel, Joerg wrote:
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.

Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.

Joerg

#7 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 6 lines · Joerg Roedel
Quoted 4 lines · H. Peter Anvin

On 08/05/2010 12:45 AM, Roedel, Joerg wrote:
Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.

Quoted 10 lines

Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#8 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 11:34:37AM -0400

Quoted 18 lines · Joerg Roedel
Quoted 6 lines · H. Peter Anvin
Quoted 4 lines · Joerg

Correct, again. It's unclear to me if we can get away with the very simple 64-bit approach -- in particular, not including all the 1:1 mappings in the kernel -- for all future users, though.

Quoted 10 lines

Yeah, We could probably use the same symbol names for the trampoline and the swapper page-table on 32 and 64 bit and merge the code that handles it between the architectures. Shouldn't be too difficult. We cook something up :-)
But do you mind to take this patch first? It fixes the occurence of an erratum on AMD hardware, it is a quite simple fix compared to the rework suggested. In fact, I would like to have a simple patch to fix the problem first (and backport it as necessary) and then go the way to rework the current code differences between 32 and 64 bit.

Quoted 3 lines

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the stack_start on the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#9 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 10 lines · Borislav Petkov
Quoted 4 lines

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the stack_start on the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...

Sounds good. This is obviously .37-or-higher material.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#10 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 11:47:55AM -0400

Quoted 6 lines · Borislav Petkov
Quoted 5 lines
Quoted 4 lines

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

Quoted 7 lines
Quoted 5 lines

I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the stack_start on the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...

Sounds good. This is obviously .37-or-higher material.

Definitely, this needs a lot of hammering before going upstream.

So here's what I got so far, I went and changed everything called swapper_* when used to prep the initial page table to initial_*. Then in setup_arch I switch to the swapper_pg_dir after copying the kernel address mappings into it so all other places touching swapper_pg_dir can remain unchanged. Then, initial_page_table is still used for booting the APs.

The nice effect is that we drop all that swsusp_pg_dir for ACPI_SLEEP and unifying the do_boot_cpu() part of start_secondary() even more.

So this boots fine on 32-bit PAE but reboots the machine on !PAE when the AP does

	/* Set up the stack pointer */
	lss stack_start,%esp

	pushl $0
	popfl

in head_32.S. I've debugged this by adding endless loops in the manner of

@@@ -274,6 -274,6 +274,7 @@@ ENTRY(startup_32_smp
movl %eax,%es
movl %eax,%fs
movl %eax,%gs
+ movl $0xdeadbeef, %ebx
#endif /* CONFIG_SMP */

so that this happens only on the AP and then did

+667:
+ cmpl $0xdeadbeef, %ebx
+ je 667b
+

So when the loop is before the "pushl.. popfl" sequence above, the BSP would print "CPU stuck??" for the AP and continue booting by not bringing the AP online. If I put the endless loop after that sequence, the machine reboots. This is also nicely reproducible in kvm for even faster and safer experimenting with the code.

Thanks.

--

arch/x86/include/asm/pgtable_32.h +1 -0
index 2984a25..8abde9e 100644
@@ -26,6 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
+extern pgd_t initial_page_table[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/tlbflush.h +0 -2
index 7f3eba0..169be89 100644
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start,
 	flush_tlb_all();
 }
 
-extern void zap_low_mappings(bool early);
-
 #endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/kernel/acpi/sleep.c +6 -1
index fcc3c61..48d89d6 100644
@@ -12,6 +12,11 @@
 #include <asm/segment.h>
 #include <asm/desc.h>
 
+#ifdef CONFIG_X86_32
+#include <asm/pgtable.h>
+#include <asm/pgtable_32.h>
+#endif
+
 #include "realmode/wakeup.h"
 #include "sleep.h"
 
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)
 
 #ifndef CONFIG_64BIT
 	header->pmode_entry = (u32)&wakeup_pmode_return;
-	header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET);
+	header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET);
 	saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 	header->trampoline_segment = setup_trampoline() >> 4;
arch/x86/kernel/head_32.S +21 -19
index 37c3d4b..80b0961 100644
@@ -177,7 +177,7 @@ default_entry:
 #ifdef CONFIG_X86_PAE
 
 	/*
-	 * In PAE mode swapper_pg_dir is statically defined to contain enough
+	 * In PAE mode initial_page_table is statically defined to contain enough
 	 * entries to cover the VMSPLIT option (that is the top 1, 2 or 3
 	 * entries). The identity mapping is handled by pointing two PGD
 	 * entries to the first kernel PMD.
@@ -191,7 +191,7 @@ default_entry:
 	xorl %ebx,%ebx				/* %ebx is kept at zero */
 
 	movl $pa(__brk_base), %edi
-	movl $pa(swapper_pg_pmd), %edx
+	movl $pa(initial_pg_pmd), %edx
 	movl $PTE_IDENT_ATTR, %eax
 10:
 	leal PDE_IDENT_ATTR(%edi),%ecx		/* Create PMD entry */
@@ -220,14 +220,14 @@ default_entry:
 	movl %eax, pa(max_pfn_mapped)
 
 	/* Do early initialization of the fixmap area */
-	movl $pa(swapper_pg_fixmap)+PDE_IDENT_ATTR,%eax
-	movl %eax,pa(swapper_pg_pmd+0x1000*KPMDS-8)
+	movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
+	movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8)
 #else	/* Not PAE */
 
 page_pde_offset = (__PAGE_OFFSET >> 20);
 
 	movl $pa(__brk_base), %edi
-	movl $pa(swapper_pg_dir), %edx
+	movl $pa(initial_page_table), %edx
 	movl $PTE_IDENT_ATTR, %eax
 10:
 	leal PDE_IDENT_ATTR(%edi),%ecx		/* Create PDE entry */
@@ -251,8 +251,8 @@ page_pde_offset = (__PAGE_OFFSET >> 20);
 	movl %eax, pa(max_pfn_mapped)
 
 	/* Do early initialization of the fixmap area */
-	movl $pa(swapper_pg_fixmap)+PDE_IDENT_ATTR,%eax
-	movl %eax,pa(swapper_pg_dir+0xffc)
+	movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
+	movl %eax,pa(initial_page_table+0xffc)
 #endif
 	jmp 3f
 /*
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl $pa(initial_page_table),%eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -615,16 +615,18 @@ ENTRY(initial_code)
 __PAGE_ALIGNED_BSS
 	.align PAGE_SIZE_asm
 #ifdef CONFIG_X86_PAE
-swapper_pg_pmd:
+initial_pg_pmd:
 	.fill 1024*KPMDS,4,0
 #else
-ENTRY(swapper_pg_dir)
+ENTRY(initial_page_table)
 	.fill 1024,4,0
 #endif
-swapper_pg_fixmap:
+initial_pg_fixmap:
 	.fill 1024,4,0
 ENTRY(empty_zero_page)
 	.fill 4096,1,0
+ENTRY(swapper_pg_dir)
+	.fill 1024,4,0
 
 /*
  * This starts the data section.
@@ -633,20 +635,20 @@ ENTRY(empty_zero_page)
 __PAGE_ALIGNED_DATA
 	/* Page-aligned for the benefit of paravirt? */
 	.align PAGE_SIZE_asm
-ENTRY(swapper_pg_dir)
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR),0	/* low identity map */
+ENTRY(initial_page_table)
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR),0	/* low identity map */
 # if KPMDS == 3
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR),0
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x1000),0
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x2000),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0
 # elif KPMDS == 2
 	.long	0,0
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR),0
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR+0x1000),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
 # elif KPMDS == 1
 	.long	0,0
 	.long	0,0
-	.long	pa(swapper_pg_pmd+PGD_IDENT_ATTR),0
+	.long	pa(initial_pg_pmd+PGD_IDENT_ATTR),0
 # else
 #  error "Kernel PMDs should be 1, 2 or 3"
 # endif
arch/x86/kernel/setup.c +9 -0
index b4ae4ac..ba19c9e 100644
@@ -727,6 +727,15 @@ void __init setup_arch(char **cmdline_p)
 	int k8 = 0;
 
 #ifdef CONFIG_X86_32
+       /* Copy kernel address range */
+       clone_pgd_range(swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+                       initial_page_table + KERNEL_PGD_BOUNDARY,
+                       min_t(unsigned long, KERNEL_PGD_PTRS,
+                             KERNEL_PGD_BOUNDARY));
+
+       load_cr3(swapper_pg_dir);
+       __flush_tlb_all();
+
 	memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
 	visws_early_detect();
 #else
arch/x86/kernel/smpboot.c +1 -16
index c4f33b2..d1bd555 100644
@@ -73,7 +73,6 @@
 
 #ifdef CONFIG_X86_32
 u8 apicid_2_node[MAX_APICID];
-static int low_mappings;
 #endif
 
 /* State of each CPU */
@@ -300,8 +299,7 @@ notrace static void __cpuinit start_secondary(void *unused)
 	}
 
 #ifdef CONFIG_X86_32
-	while (low_mappings)
-		cpu_relax();
+	load_cr3(swapper_pg_dir);
 	__flush_tlb_all();
 #endif
 
@@ -894,20 +892,7 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
-#ifdef CONFIG_X86_32
-	/* init low mem mapping */
-	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-	flush_tlb_all();
-	low_mappings = 1;
-
 	err = do_boot_cpu(apicid, cpu);
-
-	zap_low_mappings(false);
-	low_mappings = 0;
-#else
-	err = do_boot_cpu(apicid, cpu);
-#endif
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/mm/init_32.c +0 -45

57 lines not on this page · raw message

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
maintainer
#11 lore raw

Initial working version (Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines)

In reply to H. Peter Anvin

From: "H. Peter Anvin" <[email protected]>
Date: Thu, Aug 12, 2010 at 08:47:55AM -0700

Quoted 7 lines
Quoted 5 lines

I've been working on what you suggested and the patch boots on a 32-bit PAE kernel but keeps rebooting on non-PAE after setting the stack_start on the AP. I could send it out later for you to look at, if you like - you should be able to spot what I'm missing :)...

Sounds good. This is obviously .37-or-higher material.

Ok, here we go, the version below boots a !PAE and a PAE kernel fine in kvm. The approach taken is, IMHO, the least intrusive in trying to touch code assuming swapper_pg_dir as little as possible and use initial_page_table for boot/acpi sleep/reboot only.

For example, paths like init_memory_mapping -> kernel_physical_mapping_init -> ... which fill in the kernel mappings into swapper_pg_dir are left unchanged by switching to swapper_pg_dir at the beginning of setup_arch() BTW, I guess this could be done even earlier since I copy the mappings from initial_page_table so there's no difference between the two at the switch point.

Later, after paging_init(), synching back all swapper_pg_dir changes into initial_page_table is done for the APs to boot and have a stack mapping in the initial_page_table so that they don't pagefault into a triple fault (this was the problem with the previous version, btw.)

Please take a look and let me know if I've forgotten something before I start hammering on it on a real hardware.

Thanks.

---

arch/x86/include/asm/pgtable_32.h +1 -1
index f686f49..8abde9e 100644
@@ -26,7 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
-extern pgd_t trampoline_pg_dir[1024];
+extern pgd_t initial_page_table[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/tlbflush.h +0 -2
index 7f3eba0..169be89 100644
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start,
 	flush_tlb_all();
 }
 
-extern void zap_low_mappings(bool early);
-
 #endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
index 4dde797..f4500fb 100644
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
-extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
-extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
-static inline void setup_trampoline_page_table(void) {}
 static inline void reserve_trampoline_memory(void) {}
 #endif /* CONFIG_X86_TRAMPOLINE */
 
arch/x86/kernel/acpi/sleep.c +6 -1
index 33cec15..d04500e 100644
@@ -12,6 +12,11 @@
 #include <asm/segment.h>
 #include <asm/desc.h>
 
+#ifdef CONFIG_X86_32
+#include <asm/pgtable.h>
+#include <asm/pgtable_32.h>
+#endif
+
 #include "realmode/wakeup.h"
 #include "sleep.h"
 
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)
 
 #ifndef CONFIG_64BIT
 	header->pmode_entry = (u32)&wakeup_pmode_return;
-	header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET);
+	header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET);
 	saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 	header->trampoline_segment = setup_trampoline() >> 4;
arch/x86/kernel/head32.c +1 -0
index 784360c..8b9c201 100644
@@ -17,6 +17,7 @@
 #include <asm/apic.h>
 #include <asm/io_apic.h>
 #include <asm/bios_ebda.h>
+#include <asm/tlbflush.h>
 
 static void __init i386_default_early_setup(void)
 {
arch/x86/kernel/head_32.S +21 -25

112 lines not on this page · raw message

arch/x86/kernel/reboot.c +2 -8
index e3af342..ce5566e 100644
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)
 	CMOS_WRITE(0x00, 0x8f);
 	spin_unlock(&rtc_lock);
 
-	/* Remap the kernel at virtual address zero, as well as offset zero
-	   from the kernel segment.  This assumes the kernel segment starts at
-	   virtual address PAGE_OFFSET. */
-	memcpy(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		sizeof(swapper_pg_dir [0]) * KERNEL_PGD_PTRS);
-
 	/*
-	 * Use `swapper_pg_dir' as our page directory.
+	 * Switch back to the initial page table.
 	 */
-	load_cr3(swapper_pg_dir);
+	load_cr3(initial_page_table);
 
 	/* Write 0x1234 to absolute memory location 0x472.  The BIOS reads
 	   this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
index c3a4fbb..32322df 100644
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)
 #ifdef CONFIG_X86_32
 	memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
 	visws_early_detect();
+
+	/*
+	 * copy kernel address range established so far and switch
+	 * to the proper swapper page table
+	 */
+	clone_pgd_range(swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			initial_page_table + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
+
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
 #else
 	printk(KERN_INFO "Command line: %s\n", boot_command_line);
 #endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
-	setup_trampoline_page_table();
+#ifdef CONFIG_X86_32
+	/* sync back kernel address range */
+	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
+#endif
 
 	tboot_probe();
 
arch/x86/kernel/smpboot.c +6 -13
index 8b3bfc4..0d86d0b 100644
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)
 	 * most necessary things.
 	 */
 
-#ifdef CONFIG_X86_32
-	/*
-	 * Switch away from the trampoline page-table
-	 *
-	 * Do this before cpu_init() because it needs to access per-cpu
-	 * data which may not be mapped in the trampoline page-table.
-	 */
-	load_cr3(swapper_pg_dir);
-	__flush_tlb_all();
-#endif
-
 	vmi_bringup();
 	cpu_init();
 	preempt_disable();
 	smp_callin();
 
+#ifdef CONFIG_X86_32
+	/* switch away from the initial page table */
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
+#endif
+
 	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 	barrier();
 	/*
@@ -774,7 +769,6 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
-	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
 	err = do_boot_cpu(apicid, cpu);
-
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +0 -17
index a874495..f1488a3 100644
@@ -38,20 +38,3 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
-
-void __init setup_trampoline_page_table(void)
-{
-#ifdef CONFIG_X86_32
-	/* Copy kernel address range */
-	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			min_t(unsigned long, KERNEL_PGD_PTRS,
-			      KERNEL_PGD_BOUNDARY));
-
-	/* Initialize low mappings */
-	clone_pgd_range(trampoline_pg_dir,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			min_t(unsigned long, KERNEL_PGD_PTRS,
-			      KERNEL_PGD_BOUNDARY));
-#endif
-}
arch/x86/mm/init_32.c +0 -45

57 lines not on this page · raw message

Signature · 2 lines
Regards/Gruss,
    Boris.
maintainer
#12 lore raw

PATCH 1/1 x86-32, mm: Add an initial page table for core bootstrapping

This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to swapper_pg_dir and zap them later or create own swsusp PGD page solely for ACPI sleep needs.

---

Ok, here's the initial version. This patch has survived a whole weekend of randconfig builds and has been boot/reboot/suspend/resume tested on 32-bit kvm guests and bare metal in all possible configurations of PAE and !PAE kernels with all possible VMSPLIT_* config options.

Patch is against tip/x86/urgent ontop of Hugh's CONFIG_VMSPLIT_1G and 2G_OPT fix.

 arch/x86/include/asm/pgtable_32.h |    2 +-
 arch/x86/include/asm/tlbflush.h   |    2 -
 arch/x86/include/asm/trampoline.h |    3 --
 arch/x86/kernel/acpi/sleep.c      |    7 ++++-
 arch/x86/kernel/head32.c          |    1 +
 arch/x86/kernel/head_32.S         |   55 +++++++++++++++++--------------------
 arch/x86/kernel/reboot.c          |   10 +-----
 arch/x86/kernel/setup.c           |   18 +++++++++++-
 arch/x86/kernel/smpboot.c         |   19 ++++---------
 arch/x86/kernel/trampoline.c      |   16 -----------
 arch/x86/mm/init_32.c             |   45 ------------------------------
 11 files changed, 58 insertions(+), 120 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -1
index f686f49..8abde9e 100644
@@ -26,7 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
-extern pgd_t trampoline_pg_dir[1024];
+extern pgd_t initial_page_table[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/tlbflush.h +0 -2
index 7f3eba0..169be89 100644
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start,
 	flush_tlb_all();
 }
 
-extern void zap_low_mappings(bool early);
-
 #endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
index 4dde797..f4500fb 100644
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
-extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
-extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
-static inline void setup_trampoline_page_table(void) {}
 static inline void reserve_trampoline_memory(void) {}
 #endif /* CONFIG_X86_TRAMPOLINE */
 
arch/x86/kernel/acpi/sleep.c +6 -1
index 33cec15..d04500e 100644
@@ -12,6 +12,11 @@
 #include <asm/segment.h>
 #include <asm/desc.h>
 
+#ifdef CONFIG_X86_32
+#include <asm/pgtable.h>
+#include <asm/pgtable_32.h>
+#endif
+
 #include "realmode/wakeup.h"
 #include "sleep.h"
 
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)
 
 #ifndef CONFIG_64BIT
 	header->pmode_entry = (u32)&wakeup_pmode_return;
-	header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET);
+	header->pmode_cr3 = (u32)(initial_page_table - __PAGE_OFFSET);
 	saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 	header->trampoline_segment = setup_trampoline() >> 4;
arch/x86/kernel/head32.c +1 -0
index 784360c..8b9c201 100644
@@ -17,6 +17,7 @@
 #include <asm/apic.h>
 #include <asm/io_apic.h>
 #include <asm/bios_ebda.h>
+#include <asm/tlbflush.h>
 
 static void __init i386_default_early_setup(void)
 {
arch/x86/kernel/head_32.S +25 -30

122 lines not on this page · raw message

arch/x86/kernel/reboot.c +2 -8
index e3af342..ce5566e 100644
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)
 	CMOS_WRITE(0x00, 0x8f);
 	spin_unlock(&rtc_lock);
 
-	/* Remap the kernel at virtual address zero, as well as offset zero
-	   from the kernel segment.  This assumes the kernel segment starts at
-	   virtual address PAGE_OFFSET. */
-	memcpy(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		sizeof(swapper_pg_dir [0]) * KERNEL_PGD_PTRS);
-
 	/*
-	 * Use `swapper_pg_dir' as our page directory.
+	 * Switch back to the initial page table.
 	 */
-	load_cr3(swapper_pg_dir);
+	load_cr3(initial_page_table);
 
 	/* Write 0x1234 to absolute memory location 0x472.  The BIOS reads
 	   this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
index c3a4fbb..b315b37 100644
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)
 #ifdef CONFIG_X86_32
 	memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
 	visws_early_detect();
+
+	/*
+	 * copy kernel address range established so far and switch
+	 * to the proper swapper page table
+	 */
+	clone_pgd_range(swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			initial_page_table + KERNEL_PGD_BOUNDARY,
+			KERNEL_PGD_PTRS);
+
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
 #else
 	printk(KERN_INFO "Command line: %s\n", boot_command_line);
 #endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
-	setup_trampoline_page_table();
+#ifdef CONFIG_X86_32
+	/* sync back kernel address range */
+	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			KERNEL_PGD_PTRS);
+#endif
 
 	tboot_probe();
 
arch/x86/kernel/smpboot.c +6 -13
index 8b3bfc4..0d86d0b 100644
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)
 	 * most necessary things.
 	 */
 
-#ifdef CONFIG_X86_32
-	/*
-	 * Switch away from the trampoline page-table
-	 *
-	 * Do this before cpu_init() because it needs to access per-cpu
-	 * data which may not be mapped in the trampoline page-table.
-	 */
-	load_cr3(swapper_pg_dir);
-	__flush_tlb_all();
-#endif
-
 	vmi_bringup();
 	cpu_init();
 	preempt_disable();
 	smp_callin();
 
+#ifdef CONFIG_X86_32
+	/* switch away from the initial page table */
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
+#endif
+
 	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 	barrier();
 	/*
@@ -774,7 +769,6 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
-	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
 	err = do_boot_cpu(apicid, cpu);
-
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +0 -16
index e2a5952..f1488a3 100644
@@ -38,19 +38,3 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
-
-void __init setup_trampoline_page_table(void)
-{
-#ifdef CONFIG_X86_32
-	/* Copy kernel address range */
-	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			KERNEL_PGD_PTRS);
-
-	/* Initialize low mappings */
-	clone_pgd_range(trampoline_pg_dir,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			min_t(unsigned long, KERNEL_PGD_PTRS,
-			      KERNEL_PGD_BOUNDARY));
-#endif
-}
arch/x86/mm/init_32.c +0 -45

57 lines not on this page · raw message

Signature · 5 lines
1.7.1

-- 
Regards/Gruss,
    Boris.
maintainer
#13 lore raw

PATCH 1/1 x86-32, mm: Add an initial page table for core bootstrapping

From: Borislav Petkov <[email protected]>
Date: Sun, Aug 29, 2010 at 10:32:05PM +0200

Quoted 8 lines

This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to swapper_pg_dir and zap them later or create own swsusp PGD page solely for ACPI sleep needs.

---

Quoted 7 lines

Ok, here's the initial version. This patch has survived a whole weekend of randconfig builds and has been boot/reboot/suspend/resume tested on 32-bit kvm guests and bare metal in all possible configurations of PAE and !PAE kernels with all possible VMSPLIT_* config options.

Patch is against tip/x86/urgent ontop of Hugh's CONFIG_VMSPLIT_1G and 2G_OPT fix.

Ok this one broke suspend-to-ram due to wrongly _not_ passing the _pointer_ to the initial_page_table as the pmode_cr3 pointer when resuming from ACPI sleep. Here's a better version.

--
From: Borislav Petkov <[email protected]>
Date: Sat, 28 Aug 2010 15:58:33 +0200
Subject: [PATCH] x86-32, mm: Add an initial page table for core bootstrapping

This patch adds an initial page table with low mappings used exclusively for booting APs/resuming after ACPI suspend/machine restart. After this, there's no need to add low mappings to swapper_pg_dir and zap them later or create own swsusp PGD page solely for ACPI sleep needs.

[ v1.1 Fix suspend-to-ram ]

---
 arch/x86/include/asm/pgtable_32.h |    2 +-
 arch/x86/include/asm/tlbflush.h   |    2 -
 arch/x86/include/asm/trampoline.h |    3 --
 arch/x86/kernel/acpi/sleep.c      |    7 ++++-
 arch/x86/kernel/head32.c          |    1 +
 arch/x86/kernel/head_32.S         |   55 +++++++++++++++++--------------------
 arch/x86/kernel/reboot.c          |   10 +-----
 arch/x86/kernel/setup.c           |   18 +++++++++++-
 arch/x86/kernel/smpboot.c         |   19 ++++---------
 arch/x86/kernel/trampoline.c      |   16 -----------
 arch/x86/mm/init_32.c             |   45 ------------------------------
 11 files changed, 58 insertions(+), 120 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -1
index f686f49..8abde9e 100644
@@ -26,7 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
-extern pgd_t trampoline_pg_dir[1024];
+extern pgd_t initial_page_table[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/tlbflush.h +0 -2
index 7f3eba0..169be89 100644
@@ -172,6 +172,4 @@ static inline void flush_tlb_kernel_range(unsigned long start,
 	flush_tlb_all();
 }
 
-extern void zap_low_mappings(bool early);
-
 #endif /* _ASM_X86_TLBFLUSH_H */
arch/x86/include/asm/trampoline.h +0 -3
index 4dde797..f4500fb 100644
@@ -13,16 +13,13 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
-extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
-extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
-static inline void setup_trampoline_page_table(void) {}
 static inline void reserve_trampoline_memory(void) {}
 #endif /* CONFIG_X86_TRAMPOLINE */
 
arch/x86/kernel/acpi/sleep.c +6 -1
index 33cec15..b35e1ab 100644
@@ -12,6 +12,11 @@
 #include <asm/segment.h>
 #include <asm/desc.h>
 
+#ifdef CONFIG_X86_32
+#include <asm/pgtable.h>
+#include <asm/pgtable_32.h>
+#endif
+
 #include "realmode/wakeup.h"
 #include "sleep.h"
 
@@ -90,7 +95,7 @@ int acpi_save_state_mem(void)
 
 #ifndef CONFIG_64BIT
 	header->pmode_entry = (u32)&wakeup_pmode_return;
-	header->pmode_cr3 = (u32)(swsusp_pg_dir - __PAGE_OFFSET);
+	header->pmode_cr3 = (u32)__pa(&initial_page_table);
 	saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 	header->trampoline_segment = setup_trampoline() >> 4;
arch/x86/kernel/head32.c +1 -0
index 784360c..8b9c201 100644
@@ -17,6 +17,7 @@
 #include <asm/apic.h>
 #include <asm/io_apic.h>
 #include <asm/bios_ebda.h>
+#include <asm/tlbflush.h>
 
 static void __init i386_default_early_setup(void)
 {
arch/x86/kernel/head_32.S +25 -30

122 lines not on this page · raw message

arch/x86/kernel/reboot.c +2 -8
index e3af342..ce5566e 100644
@@ -371,16 +371,10 @@ void machine_real_restart(const unsigned char *code, int length)
 	CMOS_WRITE(0x00, 0x8f);
 	spin_unlock(&rtc_lock);
 
-	/* Remap the kernel at virtual address zero, as well as offset zero
-	   from the kernel segment.  This assumes the kernel segment starts at
-	   virtual address PAGE_OFFSET. */
-	memcpy(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		sizeof(swapper_pg_dir [0]) * KERNEL_PGD_PTRS);
-
 	/*
-	 * Use `swapper_pg_dir' as our page directory.
+	 * Switch back to the initial page table.
 	 */
-	load_cr3(swapper_pg_dir);
+	load_cr3(initial_page_table);
 
 	/* Write 0x1234 to absolute memory location 0x472.  The BIOS reads
 	   this on booting to tell it to "Bypass memory test (also warm
arch/x86/kernel/setup.c +17 -1
index c3a4fbb..b315b37 100644
@@ -730,6 +730,17 @@ void __init setup_arch(char **cmdline_p)
 #ifdef CONFIG_X86_32
 	memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
 	visws_early_detect();
+
+	/*
+	 * copy kernel address range established so far and switch
+	 * to the proper swapper page table
+	 */
+	clone_pgd_range(swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			initial_page_table + KERNEL_PGD_BOUNDARY,
+			KERNEL_PGD_PTRS);
+
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
 #else
 	printk(KERN_INFO "Command line: %s\n", boot_command_line);
 #endif
@@ -1014,7 +1025,12 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
-	setup_trampoline_page_table();
+#ifdef CONFIG_X86_32
+	/* sync back kernel address range */
+	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			KERNEL_PGD_PTRS);
+#endif
 
 	tboot_probe();
 
arch/x86/kernel/smpboot.c +6 -13
index 8b3bfc4..0d86d0b 100644
@@ -300,22 +300,17 @@ notrace static void __cpuinit start_secondary(void *unused)
 	 * most necessary things.
 	 */
 
-#ifdef CONFIG_X86_32
-	/*
-	 * Switch away from the trampoline page-table
-	 *
-	 * Do this before cpu_init() because it needs to access per-cpu
-	 * data which may not be mapped in the trampoline page-table.
-	 */
-	load_cr3(swapper_pg_dir);
-	__flush_tlb_all();
-#endif
-
 	vmi_bringup();
 	cpu_init();
 	preempt_disable();
 	smp_callin();
 
+#ifdef CONFIG_X86_32
+	/* switch away from the initial page table */
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
+#endif
+
 	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 	barrier();
 	/*
@@ -774,7 +769,6 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
-	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -923,7 +917,6 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
 	err = do_boot_cpu(apicid, cpu);
-
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +0 -16
index e2a5952..f1488a3 100644
@@ -38,19 +38,3 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
-
-void __init setup_trampoline_page_table(void)
-{
-#ifdef CONFIG_X86_32
-	/* Copy kernel address range */
-	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			KERNEL_PGD_PTRS);
-
-	/* Initialize low mappings */
-	clone_pgd_range(trampoline_pg_dir,
-			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-			min_t(unsigned long, KERNEL_PGD_PTRS,
-			      KERNEL_PGD_BOUNDARY));
-#endif
-}
arch/x86/mm/init_32.c +0 -45

57 lines not on this page · raw message

Signature · 5 lines
1.7.1

-- 
Regards/Gruss,
    Boris.
#14 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

In reply to H. Peter Anvin
Quoted 3 lines · H. Peter Anvin

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

Great.

The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.

Joerg

#15 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 6 lines · Joerg Roedel
Quoted 3 lines · H. Peter Anvin

Agreed. I do have a concern about the kernel page tables not being synchronized into trampoline_pg_dir (it's only an issue for 32-bit !PAE), so at the very least we need to keep an eye out for it...

Quoted 8 lines

Great.

The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.

Are you going to submit an updated patch, then?

-hpa

#16 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 10 lines · H. Peter Anvin
Quoted 6 lines · Joerg Roedel

The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.

Are you going to submit an updated patch, then?

Yes, I will send it next week when I am back in office and gave it some testing.

Joerg

#17 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: Joerg Roedel <[email protected]>
Date: Thu, Aug 12, 2010 at 03:04:14PM -0400

Quoted 10 lines · H. Peter Anvin
Quoted 9 lines · Joerg Roedel
Quoted 7 lines

The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.

Quoted 4 lines
Quoted 1 line

Are you going to submit an updated patch, then?

Yes, I will send it next week when I am back in office and gave it some testing.

currently testing...

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#18 lore raw

Re: [PATCH 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: Borislav Petkov <[email protected]>
Date: Fri, Aug 13, 2010 at 08:35:43AM -0400

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

The synchronization problem might be real. The cpu_init function uses per-cpu variables which might not be present in the early synced tramponline page table (iirc). The change to swapper_pg_dir and the global tlb flush should probably be moved to the very beginning of the start_secondary function.

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

Are you going to submit an updated patch, then?

Yes, I will send it next week when I am back in office and gave it some testing.

currently testing...

Updated versions survived almost 48h of uninterrupted hammering, I'm sending them as a reply to this message...

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#19 lore raw

PATCH 1/2 x86-32: Fix crashes with CPU hotplug on AMD machines

From: Joerg Roedel <[email protected]>

This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:

1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.

2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.

3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.

4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.

5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.

6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.

7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.

There are two ways to fix this issue:

1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

-v2: switch to swapper_pg_dir right after entering start_secondary() so that we are able to access percpu data which might not be mapped in the trampoline page table.

---
 arch/x86/include/asm/pgtable_32.h |    1 +
 arch/x86/include/asm/trampoline.h |    3 +++
 arch/x86/kernel/head_32.S         |    8 +++++++-
 arch/x86/kernel/setup.c           |    2 ++
 arch/x86/kernel/smpboot.c         |   32 +++++++++++++-------------------
 arch/x86/kernel/trampoline.c      |   18 ++++++++++++++++++
 6 files changed, 44 insertions(+), 20 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
index 2984a25..f686f49 100644
@@ -26,6 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
+extern pgd_t trampoline_pg_dir[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/trampoline.h +3 -0
index cb507bb..8f78fdf 100644
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
+extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
+extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
 static inline void reserve_trampoline_memory(void) {};
+extern void __init setup_trampoline_page_table(void) {};
 #endif /* CONFIG_X86_TRAMPOLINE */
 
 #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
index 37c3d4b..75e3981 100644
@@ -328,7 +328,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -608,6 +608,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -623,6 +625,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 ENTRY(empty_zero_page)
 	.fill 4096,1,0
 
arch/x86/kernel/setup.c +2 -0
index b4ae4ac..6600cfd 100644
@@ -1008,6 +1008,8 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
+	setup_trampoline_page_table();
+
 	tboot_probe();
 
 #ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +13 -19
index c4f33b2..2d148a9 100644
@@ -73,7 +73,6 @@
 
 #ifdef CONFIG_X86_32
 u8 apicid_2_node[MAX_APICID];
-static int low_mappings;
 #endif
 
 /* State of each CPU */
@@ -281,6 +280,18 @@ notrace static void __cpuinit start_secondary(void *unused)
 	 * fragile that we want to limit the things done here to the
 	 * most necessary things.
 	 */
+
+#ifdef CONFIG_X86_32
+	/*
+	 * Switch away from the trampoline page-table
+	 *
+	 * Do this before cpu_init() because it needs to access per-cpu
+	 * data which may not be mapped in the trampoline page-table.
+	 */
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
+#endif
+
 	vmi_bringup();
 	cpu_init();
 	preempt_disable();
@@ -299,12 +310,6 @@ notrace static void __cpuinit start_secondary(void *unused)
 		legacy_pic->chip->unmask(0);
 	}
 
-#ifdef CONFIG_X86_32
-	while (low_mappings)
-		cpu_relax();
-	__flush_tlb_all();
-#endif
-
 	/* This must be done before setting cpu_online_mask */
 	set_cpu_sibling_map(raw_smp_processor_id());
 	wmb();
@@ -754,6 +759,7 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
+	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -894,20 +900,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
-#ifdef CONFIG_X86_32
-	/* init low mem mapping */
-	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-	flush_tlb_all();
-	low_mappings = 1;
-
 	err = do_boot_cpu(apicid, cpu);
 
-	zap_low_mappings(false);
-	low_mappings = 0;
-#else
-	err = do_boot_cpu(apicid, cpu);
-#endif
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +18 -0
index c652ef6..a874495 100644
@@ -1,6 +1,7 @@
 #include <linux/io.h>
 
 #include <asm/trampoline.h>
+#include <asm/pgtable.h>
 #include <asm/e820.h>
 
 #if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP)
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
+
+void __init setup_trampoline_page_table(void)
+{
+#ifdef CONFIG_X86_32
+	/* Copy kernel address range */
+	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+
+	/* Initialize low mappings */
+	clone_pgd_range(trampoline_pg_dir,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+#endif
+}
Signature · 6 lines
1.7.1


-- 
Regards/Gruss,
Boris.
#20 lore raw

Re: [PATCH -v2 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

Quoted 4 lines · Borislav Petkov

#else
static inline void reserve_trampoline_memory(void) {};
+extern void __init setup_trampoline_page_table(void) {};
#endif /* CONFIG_X86_TRAMPOLINE */

This breaks compilation in the !CONFIG_X86_TRAMPOLINE case; I presume you meant:

static inline void setup_trampoline_page_table(void) {}

... here too?

(No semicolon after a function body!)

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#21 lore raw

Re: [PATCH -v2 1/2] x86-32: Fix crashes with CPU hotplug on AMD machines

From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 18, 2010 at 02:41:17PM -0400

Quoted 6 lines · Borislav Petkov
Quoted 4 lines

#else
static inline void reserve_trampoline_memory(void) {};
+extern void __init setup_trampoline_page_table(void) {};
#endif /* CONFIG_X86_TRAMPOLINE */

Quoted 8 lines

This breaks compilation in the !CONFIG_X86_TRAMPOLINE case; I presume you meant:

static inline void setup_trampoline_page_table(void) {}

... here too?

(No semicolon after a function body!)

Yeah, thanks for catching that - this should be a static inline stub in the !CONFIG_X86_TRAMPOLINE case.

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#22 lore raw

[tip:x86/urgent] x86-32: Fix dummy trampoline-related inline stubs

In reply to H. Peter Anvin
Commit-ID:  8848a91068c018bc91f597038a0f41462a0f88a4
Gitweb:     http://git.kernel.org/tip/8848a91068c018bc91f597038a0f41462a0f88a4
Author:     H. Peter Anvin <[email protected]>
AuthorDate: Wed, 18 Aug 2010 11:42:23 -0700
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Wed, 18 Aug 2010 12:42:24 -0700

x86-32: Fix dummy trampoline-related inline stubs

Fix dummy inline stubs for trampoline-related functions when no trampolines exist (until we get rid of the no-trampoline case entirely.)

Cc Joerg Roedel <[email protected]>
Cc Borislav Petkov <[email protected]>

LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/trampoline.h |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/include/asm/trampoline.h +2 -2
index 8f78fdf..4dde797 100644
@@ -22,8 +22,8 @@ extern unsigned long setup_trampoline(void);
 extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
-static inline void reserve_trampoline_memory(void) {};
-extern void __init setup_trampoline_page_table(void) {};
+static inline void setup_trampoline_page_table(void) {}
+static inline void reserve_trampoline_memory(void) {}
 #endif /* CONFIG_X86_TRAMPOLINE */
 
 #endif /* __ASSEMBLY__ */
#23 lore raw

[tip:x86/urgent] x86-32: Separate 1:1 pagetables from swapper_pg_dir

In reply to Borislav Petkov
Commit-ID:  fd89a137924e0710078c3ae855e7cec1c43cb845
Gitweb:     http://git.kernel.org/tip/fd89a137924e0710078c3ae855e7cec1c43cb845
Author:     Joerg Roedel <[email protected]>
AuthorDate: Mon, 16 Aug 2010 14:38:33 +0200
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Wed, 18 Aug 2010 09:17:20 -0700

x86-32: Separate 1:1 pagetables from swapper_pg_dir

This patch fixes machine crashes which occur when heavily exercising the CPU hotplug codepaths on a 32-bit kernel. These crashes are caused by AMD Erratum 383 and result in a fatal machine check exception. Here's the scenario:

1. On 32-bit, the swapper_pg_dir page table is used as the initial page table for booting a secondary CPU.

2. To make this work, swapper_pg_dir needs a direct mapping of physical memory in it (the low mappings). By adding those low, large page (2M) mappings (PAE kernel), we create the necessary conditions for Erratum 383 to occur.

3. Other CPUs which do not participate in the off- and onlining game may use swapper_pg_dir while the low mappings are present (when leave_mm is called). For all steps below, the CPU referred to is a CPU that is using swapper_pg_dir, and not the CPU which is being onlined.

4. The presence of the low mappings in swapper_pg_dir can result in TLB entries for addresses below __PAGE_OFFSET to be established speculatively. These TLB entries are marked global and large.

5. When the CPU with such TLB entry switches to another page table, this TLB entry remains because it is global.

6. The process then generates an access to an address covered by the above TLB entry but there is a permission mismatch - the TLB entry covers a large global page not accessible to userspace.

7. Due to this permission mismatch a new 4kb, user TLB entry gets established. Further, Erratum 383 provides for a small window of time where both TLB entries are present. This results in an uncorrectable machine check exception signalling a TLB multimatch which panics the machine.

There are two ways to fix this issue:

1. Always do a global TLB flush when a new cr3 is loaded and the old page table was swapper_pg_dir. I consider this a hack hard to understand and with performance implications

2. Do not use swapper_pg_dir to boot secondary CPUs like 64-bit does.

This patch implements solution 2. It introduces a trampoline_pg_dir which has the same layout as swapper_pg_dir with low_mappings. This page table is used as the initial page table of the booting CPU. Later in the bringup process, it switches to swapper_pg_dir and does a global TLB flush. This fixes the crashes in our test cases.

-v2: switch to swapper_pg_dir right after entering start_secondary() so that we are able to access percpu data which might not be mapped in the trampoline page table.

LKML-Reference: <20100816123833.GB28147@aftab>

---
 arch/x86/include/asm/pgtable_32.h |    1 +
 arch/x86/include/asm/trampoline.h |    3 +++
 arch/x86/kernel/head_32.S         |    8 +++++++-
 arch/x86/kernel/setup.c           |    2 ++
 arch/x86/kernel/smpboot.c         |   32 +++++++++++++-------------------
 arch/x86/kernel/trampoline.c      |   18 ++++++++++++++++++
 6 files changed, 44 insertions(+), 20 deletions(-)
arch/x86/include/asm/pgtable_32.h +1 -0
index 2984a25..f686f49 100644
@@ -26,6 +26,7 @@ struct mm_struct;
 struct vm_area_struct;
 
 extern pgd_t swapper_pg_dir[1024];
+extern pgd_t trampoline_pg_dir[1024];
 
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
arch/x86/include/asm/trampoline.h +3 -0
index cb507bb..8f78fdf 100644
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;
 
 extern unsigned long init_rsp;
 extern unsigned long initial_code;
+extern unsigned long initial_page_table;
 extern unsigned long initial_gs;
 
 #define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
 
 extern unsigned long setup_trampoline(void);
+extern void __init setup_trampoline_page_table(void);
 extern void __init reserve_trampoline_memory(void);
 #else
 static inline void reserve_trampoline_memory(void) {};
+extern void __init setup_trampoline_page_table(void) {};
 #endif /* CONFIG_X86_TRAMPOLINE */
 
 #endif /* __ASSEMBLY__ */
arch/x86/kernel/head_32.S +7 -1
index ff4c453..fa8c1b8 100644
@@ -334,7 +334,7 @@ ENTRY(startup_32_smp)
 /*
  * Enable paging
  */
-	movl $pa(swapper_pg_dir),%eax
+	movl pa(initial_page_table), %eax
 	movl %eax,%cr3		/* set the page table pointer.. */
 	movl %cr0,%eax
 	orl  $X86_CR0_PG,%eax
@@ -614,6 +614,8 @@ ignore_int:
 .align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(initial_page_table)
+	.long pa(swapper_pg_dir)
 
 /*
  * BSS section
@@ -629,6 +631,10 @@ ENTRY(swapper_pg_dir)
 #endif
 swapper_pg_fixmap:
 	.fill 1024,4,0
+#ifdef CONFIG_X86_TRAMPOLINE
+ENTRY(trampoline_pg_dir)
+	.fill 1024,4,0
+#endif
 ENTRY(empty_zero_page)
 	.fill 4096,1,0
 
arch/x86/kernel/setup.c +2 -0
index b008e78..c3a4fbb 100644
@@ -1014,6 +1014,8 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	x86_init.paging.pagetable_setup_done(swapper_pg_dir);
 
+	setup_trampoline_page_table();
+
 	tboot_probe();
 
 #ifdef CONFIG_X86_64
arch/x86/kernel/smpboot.c +13 -19
index a5e928b..abf4a86 100644
@@ -73,7 +73,6 @@
 
 #ifdef CONFIG_X86_32
 u8 apicid_2_node[MAX_APICID];
-static int low_mappings;
 #endif
 
 /* State of each CPU */
@@ -281,6 +280,18 @@ notrace static void __cpuinit start_secondary(void *unused)
 	 * fragile that we want to limit the things done here to the
 	 * most necessary things.
 	 */
+
+#ifdef CONFIG_X86_32
+	/*
+	 * Switch away from the trampoline page-table
+	 *
+	 * Do this before cpu_init() because it needs to access per-cpu
+	 * data which may not be mapped in the trampoline page-table.
+	 */
+	load_cr3(swapper_pg_dir);
+	__flush_tlb_all();
+#endif
+
 	vmi_bringup();
 	cpu_init();
 	preempt_disable();
@@ -299,12 +310,6 @@ notrace static void __cpuinit start_secondary(void *unused)
 		legacy_pic->chip->unmask(0);
 	}
 
-#ifdef CONFIG_X86_32
-	while (low_mappings)
-		cpu_relax();
-	__flush_tlb_all();
-#endif
-
 	/* This must be done before setting cpu_online_mask */
 	set_cpu_sibling_map(raw_smp_processor_id());
 	wmb();
@@ -750,6 +755,7 @@ do_rest:
 #ifdef CONFIG_X86_32
 	/* Stack for startup_32 can be just as for start_secondary onwards */
 	irq_ctx_init(cpu);
+	initial_page_table = __pa(&trampoline_pg_dir);
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
@@ -897,20 +903,8 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 
-#ifdef CONFIG_X86_32
-	/* init low mem mapping */
-	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
-		min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-	flush_tlb_all();
-	low_mappings = 1;
-
 	err = do_boot_cpu(apicid, cpu);
 
-	zap_low_mappings(false);
-	low_mappings = 0;
-#else
-	err = do_boot_cpu(apicid, cpu);
-#endif
 	if (err) {
 		pr_debug("do_boot_cpu failed %d\n", err);
 		return -EIO;
arch/x86/kernel/trampoline.c +18 -0
index c652ef6..a874495 100644
@@ -1,6 +1,7 @@
 #include <linux/io.h>
 
 #include <asm/trampoline.h>
+#include <asm/pgtable.h>
 #include <asm/e820.h>
 
 #if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP)
@@ -37,3 +38,20 @@ unsigned long __trampinit setup_trampoline(void)
 	memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
 	return virt_to_phys(trampoline_base);
 }
+
+void __init setup_trampoline_page_table(void)
+{
+#ifdef CONFIG_X86_32
+	/* Copy kernel address range */
+	clone_pgd_range(trampoline_pg_dir + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+
+	/* Initialize low mappings */
+	clone_pgd_range(trampoline_pg_dir,
+			swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+			min_t(unsigned long, KERNEL_PGD_PTRS,
+			      KERNEL_PGD_BOUNDARY));
+#endif
+}
#24 lore raw

PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue

In reply to Borislav Petkov

When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.

Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.

---
 arch/x86/Kconfig          |    6 ++++++
 arch/x86/kernel/smpboot.c |   19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 0 deletions(-)
arch/x86/Kconfig +6 -0
index dcb0593..1598663 100644
@@ -58,6 +58,7 @@ config X86
 	select ANON_INODES
 	select HAVE_ARCH_KMEMCHECK
 	select HAVE_USER_RETURN_NOTIFIER
+	select ARCH_CPU_PROBE_RELEASE
 
 config INSTRUCTION_DECODER
 	def_bool (KPROBES || PERF_EVENTS)
@@ -247,6 +248,11 @@ config ARCH_HWEIGHT_CFLAGS
 
 config KTIME_SCALAR
 	def_bool X86_32
+
+config ARCH_CPU_PROBE_RELEASE
+	def_bool y
+	depends on HOTPLUG_CPU
+
 source "init/Kconfig"
 source "kernel/Kconfig.freezer"
 
arch/x86/kernel/smpboot.c +19 -0
index 2d148a9..733ad2b 100644
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 };
 static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
 #define get_idle_for_cpu(x)      (per_cpu(idle_thread_array, x))
 #define set_idle_for_cpu(x, p)   (per_cpu(idle_thread_array, x) = (p))
+
+/*
+ * We need this for trampoline_base protection from concurrent accesses when
+ * off- and onlining cores wildly.
+ */
+static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
+
+void cpu_hotplug_driver_lock()
+{
+	mutex_lock(&x86_cpu_hotplug_driver_mutex);
+}
+
+void cpu_hotplug_driver_unlock()
+{
+	mutex_unlock(&x86_cpu_hotplug_driver_mutex);
+}
+
+ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
+ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
 #else
 static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
 #define get_idle_for_cpu(x)      (idle_thread_array[(x)])
Signature · 6 lines
1.7.1


-- 
Regards/Gruss,
Boris.
#25 lore raw

Re: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue

arch/x86/Kconfig config X86 · line 62

Quoted 11 lines · Borislav Petkov
arch/x86/Kconfig +1 -0
index dcb0593..1598663 100644
@@ -58,6 +58,7 @@ config X86
 	select ANON_INODES
 	select HAVE_ARCH_KMEMCHECK
 	select HAVE_USER_RETURN_NOTIFIER
+	select ARCH_CPU_PROBE_RELEASE
 

warning: (X86) selects ARCH_CPU_PROBE_RELEASE which has unmet direct dependencies (HOTPLUG_CPU)

... and build failure when compiling without CONFIG_HOTPLUG_CPU.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#26 lore raw

Re: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue

From: "H. Peter Anvin" <[email protected]>
Date: Wed, Aug 18, 2010 at 03:03:33PM -0400

Quoted 15 lines · Borislav Petkov

arch/x86/Kconfig config X86 · line 62

Quoted 11 lines
arch/x86/Kconfig +1 -0
index dcb0593..1598663 100644
@@ -58,6 +58,7 @@ config X86
 	select ANON_INODES
 	select HAVE_ARCH_KMEMCHECK
 	select HAVE_USER_RETURN_NOTIFIER
+	select ARCH_CPU_PROBE_RELEASE
 

warning: (X86) selects ARCH_CPU_PROBE_RELEASE which has unmet direct dependencies (HOTPLUG_CPU)

well I got

+config ARCH_CPU_PROBE_RELEASE
+ def_bool y
+ depends on HOTPLUG_CPU

but I guess ARCH_CPU_PROBE_RELEASE gets selected still. Why isn't that "depends on" forcing it, I guess select is stronger.

Quoted 1 line

... and build failure when compiling without CONFIG_HOTPLUG_CPU.

This hunk fixes the !CONFIG_HOTPLUG_CPU build here but I should've caught it earlier, shame on me. Sorry for the inconvenience, can you please drop this second patch for now, I'll send you a revised version tomorrow.

Thanks.

drivers/base/cpu.c +1 -1
index 251acea..273cfb9 100644
@@ -261,7 +261,7 @@ int __init cpu_dev_init(void)
 }
 
 static struct sysdev_class_attribute *cpu_sysdev_class_attrs[] = {
-#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
+#if defined(CONFIG_ARCH_CPU_PROBE_RELEASE) && defined(CONFIG_HOTPLUG_CPU)
        &attr_probe,
        &attr_release,
 #endif
Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#27 lore raw

Re: [PATCH -v2 2/2] x86, cpu hotplug: Fix cpu bringup concurrency issue

Quoted 13 lines · Borislav Petkov
Quoted 3 lines

warning: (X86) selects ARCH_CPU_PROBE_RELEASE which has unmet direct dependencies (HOTPLUG_CPU)

well I got

(quoted diff) +3 -0
+config ARCH_CPU_PROBE_RELEASE
+       def_bool y
+       depends on HOTPLUG_CPU

but I guess ARCH_CPU_PROBE_RELEASE gets selected still. Why isn't that "depends on" forcing it, I guess select is stronger.

Quoted 7 lines
Quoted 1 line

... and build failure when compiling without CONFIG_HOTPLUG_CPU.

This hunk fixes the !CONFIG_HOTPLUG_CPU build here but I should've caught it earlier, shame on me. Sorry for the inconvenience, can you please drop this second patch for now, I'll send you a revised version tomorrow.

I would prefer to not continue to have a piece of Kconfig code which warns, so I'll hold off on this until I have your revised patch.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#28 lore raw

PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue

When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.

Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.

[ v2.1: fix !HOTPLUG_CPU build ]

---
 arch/x86/Kconfig          |    5 +++++
 arch/x86/kernel/smpboot.c |   19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 0 deletions(-)
arch/x86/Kconfig +5 -0
index a84fc34..ac7827f 100644
@@ -245,6 +245,11 @@ config ARCH_HWEIGHT_CFLAGS
 
 config KTIME_SCALAR
 	def_bool X86_32
+
+config ARCH_CPU_PROBE_RELEASE
+	def_bool y
+	depends on HOTPLUG_CPU
+
 source "init/Kconfig"
 source "kernel/Kconfig.freezer"
 
arch/x86/kernel/smpboot.c +19 -0
index abf4a86..8b3bfc4 100644
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 };
 static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
 #define get_idle_for_cpu(x)      (per_cpu(idle_thread_array, x))
 #define set_idle_for_cpu(x, p)   (per_cpu(idle_thread_array, x) = (p))
+
+/*
+ * We need this for trampoline_base protection from concurrent accesses when
+ * off- and onlining cores wildly.
+ */
+static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
+
+void cpu_hotplug_driver_lock()
+{
+        mutex_lock(&x86_cpu_hotplug_driver_mutex);
+}
+
+void cpu_hotplug_driver_unlock()
+{
+        mutex_unlock(&x86_cpu_hotplug_driver_mutex);
+}
+
+ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
+ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
 #else
 static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
 #define get_idle_for_cpu(x)      (idle_thread_array[(x)])
Signature · 11 lines
1.7.1

-- 
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#29 lore raw

[tip:x86/urgent] x86, hotplug: Serialize CPU hotplug to avoid bringup concurrency issues

Commit-ID:  d7c53c9e822a4fefa13a0cae76f3190bfd0d5c11
Gitweb:     http://git.kernel.org/tip/d7c53c9e822a4fefa13a0cae76f3190bfd0d5c11
Author:     Borislav Petkov <[email protected]>
AuthorDate: Thu, 19 Aug 2010 20:10:29 +0200
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 19 Aug 2010 14:47:43 -0700

x86, hotplug: Serialize CPU hotplug to avoid bringup concurrency issues

When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.

Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.

[ v2.1: fix !HOTPLUG_CPU build ]

LKML-Reference: <20100819181029.GC17171@aftab>

---
 arch/x86/Kconfig          |    5 +++++
 arch/x86/kernel/smpboot.c |   19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 0 deletions(-)
arch/x86/Kconfig +5 -0
index a84fc34..ac7827f 100644
@@ -245,6 +245,11 @@ config ARCH_HWEIGHT_CFLAGS
 
 config KTIME_SCALAR
 	def_bool X86_32
+
+config ARCH_CPU_PROBE_RELEASE
+	def_bool y
+	depends on HOTPLUG_CPU
+
 source "init/Kconfig"
 source "kernel/Kconfig.freezer"
 
arch/x86/kernel/smpboot.c +19 -0
index abf4a86..8b3bfc4 100644
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 };
 static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
 #define get_idle_for_cpu(x)      (per_cpu(idle_thread_array, x))
 #define set_idle_for_cpu(x, p)   (per_cpu(idle_thread_array, x) = (p))
+
+/*
+ * We need this for trampoline_base protection from concurrent accesses when
+ * off- and onlining cores wildly.
+ */
+static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
+
+void cpu_hotplug_driver_lock()
+{
+        mutex_lock(&x86_cpu_hotplug_driver_mutex);
+}
+
+void cpu_hotplug_driver_unlock()
+{
+        mutex_unlock(&x86_cpu_hotplug_driver_mutex);
+}
+
+ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
+ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
 #else
 static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
 #define get_idle_for_cpu(x)      (idle_thread_array[(x)])
#30 lore raw

PATCH 2/2 x86, cpu hotplug: Fix cpu bringup concurrency issue

In reply to Borislav Petkov

From: Borislav Petkov <[email protected]>

When testing cpu hotplug code on 32-bit we kept hitting the "CPU%d:
Stuck ??" message due to multiple cores concurrently accessing the cpu_callin_mask, among others.

Since these codepaths are not protected from concurrent access due to the fact that there's no sane reason for making an already complex code unnecessarily more complex - we hit the issue only when insanely switching cores off- and online - serialize hotplugging cores on the sysfs level and be done with it.

---
 arch/x86/Kconfig          |    6 ++++++
 arch/x86/kernel/smpboot.c |   19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 0 deletions(-)
arch/x86/Kconfig +6 -0
index dcb0593..1598663 100644
@@ -58,6 +58,7 @@ config X86
 	select ANON_INODES
 	select HAVE_ARCH_KMEMCHECK
 	select HAVE_USER_RETURN_NOTIFIER
+	select ARCH_CPU_PROBE_RELEASE
 
 config INSTRUCTION_DECODER
 	def_bool (KPROBES || PERF_EVENTS)
@@ -247,6 +248,11 @@ config ARCH_HWEIGHT_CFLAGS
 
 config KTIME_SCALAR
 	def_bool X86_32
+
+config ARCH_CPU_PROBE_RELEASE
+	def_bool y
+	depends on HOTPLUG_CPU
+
 source "init/Kconfig"
 source "kernel/Kconfig.freezer"
 
arch/x86/kernel/smpboot.c +19 -0
index 6859a42..9e8e61b 100644
@@ -90,6 +90,25 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 };
 static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
 #define get_idle_for_cpu(x)      (per_cpu(idle_thread_array, x))
 #define set_idle_for_cpu(x, p)   (per_cpu(idle_thread_array, x) = (p))
+
+/*
+ * We need this for trampoline_base protection from concurrent accesses when
+ * off- and onlining cores wildly.
+ */
+static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
+
+void cpu_hotplug_driver_lock()
+{
+        mutex_lock(&x86_cpu_hotplug_driver_mutex);
+}
+
+void cpu_hotplug_driver_unlock()
+{
+        mutex_unlock(&x86_cpu_hotplug_driver_mutex);
+}
+
+ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
+ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
 #else
 static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
 #define get_idle_for_cpu(x)      (idle_thread_array[(x)])
Signature · 1 line
1.6.4.4