Skip to contentSkip to conversation

powerpc: Avoid nmi_enter/nmi_exit in real mode interrupt.

PATCH Superseded Mahesh Salgaonkar 5 participants 8 messages Feb 14, 2024 – Apr 10, 2024

Subsystems (2)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v4 1/1 powerpc: Avoid nmi_enter/nmi_exit in real mode interrupt.

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

---
Changes in v4:
- Fix coding style issues.

Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at https://lore.kernel.org/linuxppc-dev/[email protected]/

Changes in v2:
- Rebase to upstream master
- Use jump_labels, if CONFIG_JUMP_LABEL is enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/

---
 arch/powerpc/include/asm/interrupt.h | 10 ++++++++++
 arch/powerpc/include/asm/percpu.h    | 10 ++++++++++
 arch/powerpc/kernel/setup_64.c       |  3 +++
 3 files changed, 23 insertions(+)
arch/powerpc/include/asm/interrupt.h +10 -0
index a4196ab1d0167..0b96464ff0339 100644
@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
 	if (IS_ENABLED(CONFIG_KASAN))
 		return;
 
+	/*
+	 * Likewise, do not use it in real mode if percpu first chunk is not
+	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
+	 * are chances where percpu allocation can come from vmalloc area.
+	 */
+	if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk)
+		return;
+
 	/* Otherwise, it should be safe to call it */
 	nmi_enter();
 }
@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
 		// no nmi_exit for a pseries hash guest taking a real mode exception
 	} else if (IS_ENABLED(CONFIG_KASAN)) {
 		// no nmi_exit for KASAN in real mode
+	} else if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk) {
+		// no nmi_exit if percpu first chunk is not embedded
 	} else {
 		nmi_exit();
 	}
arch/powerpc/include/asm/percpu.h +10 -0
index 8e5b7d0b851c6..e24063eb0b33b 100644
@@ -15,6 +15,16 @@
 #endif /* CONFIG_SMP */
 #endif /* __powerpc64__ */
 
+#ifdef CONFIG_PPC64
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
+
+#define is_embed_first_chunk	\
+		(static_key_enabled(&__percpu_embed_first_chunk.key))
+#else
+#define is_embed_first_chunk	true
+#endif /* CONFIG_PPC64 */
+
 #include <asm-generic/percpu.h>
 
 #include <asm/paca.h>
arch/powerpc/kernel/setup_64.c +3 -0
index 2f19d5e944852..e04f0ff69d4b6 100644
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)
 
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
+DEFINE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
 
 void __init setup_per_cpu_areas(void)
 {
@@ -869,6 +870,8 @@ void __init setup_per_cpu_areas(void)
 			pr_warn("PERCPU: %s allocator failed (%d), "
 				"falling back to page size\n",
 				pcpu_fc_names[pcpu_chosen_fc], rc);
+		else
+			static_key_enable(&__percpu_embed_first_chunk.key);
 	}
 
 	if (rc < 0)
Signature · 1 line
2.43.0
Reviewed-by on v4 1/1
#1 lore raw

Le 14/02/2024 à 10:51, Mahesh Salgaonkar a écrit :

Quoted 20 lines

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Quoted 4 lines

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

Quoted 90 lines

---
Changes in v4:
- Fix coding style issues.

Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at https://lore.kernel.org/linuxppc-dev/[email protected]/

Changes in v2:
- Rebase to upstream master
- Use jump_labels, if CONFIG_JUMP_LABEL is enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/

---
  arch/powerpc/include/asm/interrupt.h | 10 ++++++++++
  arch/powerpc/include/asm/percpu.h    | 10 ++++++++++
  arch/powerpc/kernel/setup_64.c       |  3 +++
  3 files changed, 23 insertions(+)
arch/powerpc/include/asm/interrupt.h +10 -0
index a4196ab1d0167..0b96464ff0339 100644
@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
  	if (IS_ENABLED(CONFIG_KASAN))
  		return;
 
+	/*
+	 * Likewise, do not use it in real mode if percpu first chunk is not
+	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
+	 * are chances where percpu allocation can come from vmalloc area.
+	 */
+	if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk)
+		return;
+
  	/* Otherwise, it should be safe to call it */
  	nmi_enter();
  }
@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
  		// no nmi_exit for a pseries hash guest taking a real mode exception
  	} else if (IS_ENABLED(CONFIG_KASAN)) {
  		// no nmi_exit for KASAN in real mode
+	} else if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk) {
+		// no nmi_exit if percpu first chunk is not embedded
  	} else {
  		nmi_exit();
  	}
arch/powerpc/include/asm/percpu.h +10 -0
index 8e5b7d0b851c6..e24063eb0b33b 100644
@@ -15,6 +15,16 @@
  #endif /* CONFIG_SMP */
  #endif /* __powerpc64__ */
 
+#ifdef CONFIG_PPC64
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
+
+#define is_embed_first_chunk	\
+		(static_key_enabled(&__percpu_embed_first_chunk.key))
+#else
+#define is_embed_first_chunk	true
+#endif /* CONFIG_PPC64 */
+
  #include <asm-generic/percpu.h>
 
  #include <asm/paca.h>
arch/powerpc/kernel/setup_64.c +3 -0
index 2f19d5e944852..e04f0ff69d4b6 100644
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)
 
  unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
  EXPORT_SYMBOL(__per_cpu_offset);
+DEFINE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
 
  void __init setup_per_cpu_areas(void)
  {
@@ -869,6 +870,8 @@ void __init setup_per_cpu_areas(void)
  			pr_warn("PERCPU: %s allocator failed (%d), "
  				"falling back to page size\n",
  				pcpu_fc_names[pcpu_chosen_fc], rc);
+		else
+			static_key_enable(&__percpu_embed_first_chunk.key);
  	}
 
  	if (rc < 0)
Tested-by on v4 1/1
#2 lore raw
In reply to Mahesh Salgaonkar
Quoted 26 lines · Mahesh Salgaonkar

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel
crash when invoked during real mode interrupt handling (e.g. early HMI/MCE
interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI()
wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when
percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu
allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation
to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364]
Quoted 7 lines

machine_check_early_common+0x134/0x1f8

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu
first chunk is not embedded.

Thanks for the Patch.
I have tested the patch and the fix works fine. selftests/powerpc/mce/inject-ra-err testcase is working as expected after enabling percpu_alloc=page with the patch applied.

Output with Patch:

# ./inject-ra-err 
test: inject-ra-err
tags: git_version:unknown
success: inject-ra-err
#
Quoted 106 lines

---
Changes in v4:
- Fix coding style issues.

Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at
https://lore.kernel.org/linuxppc-dev/[email protected]/

Changes in v2:
- Rebase to upstream master
- Use jump_labels, if CONFIG_JUMP_LABEL is enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at
https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/

---
 arch/powerpc/include/asm/interrupt.h | 10 ++++++++++
 arch/powerpc/include/asm/percpu.h    | 10 ++++++++++
 arch/powerpc/kernel/setup_64.c       |  3 +++
 3 files changed, 23 insertions(+)
arch/powerpc/include/asm/interrupt.h +0 -0
index a4196ab1d0167..0b96464ff0339 100644
@@ -336,6 +336,14 @@ static inline void

interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte

arch/powerpc/include/asm/interrupt.h +2 -0
 	if (IS_ENABLED(CONFIG_KASAN))
 		return;
 
+	/*
+	 * Likewise, do not use it in real mode if percpu first chunk

is not

arch/powerpc/include/asm/interrupt.h +1 -0
+	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled

there

arch/powerpc/include/asm/interrupt.h +1 -0
+	 * are chances where percpu allocation can come from vmalloc

area.

arch/powerpc/include/asm/interrupt.h +2 -0
+	 */
+	if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) &&

!is_embed_first_chunk)

arch/powerpc/include/asm/interrupt.h +2 -0
+		return;
+
 	/* Otherwise, it should be safe to call it */
 	nmi_enter();
 }
@@ -351,6 +359,8 @@ static inline void

interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
// no nmi_exit for a pseries hash guest taking a real
mode exception

arch/powerpc/include/asm/interrupt.h +1 -0
 	} else if (IS_ENABLED(CONFIG_KASAN)) {
 		// no nmi_exit for KASAN in real mode
+	} else if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) &&

!is_embed_first_chunk) {

arch/powerpc/include/asm/interrupt.h +1 -0
+		// no nmi_exit if percpu first chunk is not embedded
 	} else {
 		nmi_exit();
 	}
arch/powerpc/include/asm/percpu.h +10 -0
index 8e5b7d0b851c6..e24063eb0b33b 100644
@@ -15,6 +15,16 @@
 #endif /* CONFIG_SMP */
 #endif /* __powerpc64__ */
 
+#ifdef CONFIG_PPC64
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
+
+#define is_embed_first_chunk	\
+		(static_key_enabled(&__percpu_embed_first_chunk.key))
+#else
+#define is_embed_first_chunk	true
+#endif /* CONFIG_PPC64 */
+
 #include <asm-generic/percpu.h>
 
 #include <asm/paca.h>
arch/powerpc/kernel/setup_64.c +3 -0
index 2f19d5e944852..e04f0ff69d4b6 100644
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)
 
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
+DEFINE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
 
 void __init setup_per_cpu_areas(void)
 {
@@ -869,6 +870,8 @@ void __init setup_per_cpu_areas(void)
 			pr_warn("PERCPU: %s allocator failed (%d), "
 				"falling back to page size\n",
 				pcpu_fc_names[pcpu_chosen_fc], rc);
+		else
+			static_key_enable(&__percpu_embed_first_chunk.k
 ey);
 	}
 

if (rc < 0)

#3 lore raw
In reply to Mahesh Salgaonkar

Hi Mahesh,

Quoted 20 lines · Mahesh Salgaonkar

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Quoted 2 lines

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

My system (powernv) doesn't even boot with percpu_alloc=page.

AFAIK the only reason we added support for it was to handle 4K kernels with HPT. See commit eb553f16973a ("powerpc/64/mm: implement page mapping percpu first chunk allocator").

So I wonder if we should change the Kconfig to only offer it as an option in that case, and change the logic in setup_per_cpu_areas() to only use it as a last resort.

I guess we probably still need this commit though, even if just for 4K HPT.

Quoted 26 lines

---
Changes in v4:
- Fix coding style issues.

Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at https://lore.kernel.org/linuxppc-dev/[email protected]/

Changes in v2:
- Rebase to upstream master
- Use jump_labels, if CONFIG_JUMP_LABEL is enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/

---
 arch/powerpc/include/asm/interrupt.h | 10 ++++++++++
 arch/powerpc/include/asm/percpu.h    | 10 ++++++++++
 arch/powerpc/kernel/setup_64.c       |  3 +++
 3 files changed, 23 insertions(+)
arch/powerpc/include/asm/interrupt.h +0 -0
index a4196ab1d0167..0b96464ff0339 100644

arch/powerpc/include/asm/interrupt.h static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte · line 344

Quoted 9 lines
arch/powerpc/include/asm/interrupt.h +6 -0
@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
 	if (IS_ENABLED(CONFIG_KASAN))
 		return;
 
+	/*
+	 * Likewise, do not use it in real mode if percpu first chunk is not
+	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
+	 * are chances where percpu allocation can come from vmalloc area.
+	 */
+	if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk)

I think this would be clearer if it was inverted, eg:

        if (percpu_first_chunk_is_paged)
               return;

That way you shouldn't need to check CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK here. Instead it can be part of the ifdef in the header.

Quoted 17 lines
arch/powerpc/include/asm/interrupt.h +2 -0
@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
 		// no nmi_exit for a pseries hash guest taking a real mode exception
 	} else if (IS_ENABLED(CONFIG_KASAN)) {
 		// no nmi_exit for KASAN in real mode
+	} else if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk) {
+		// no nmi_exit if percpu first chunk is not embedded
 	} else {
 		nmi_exit();
 	}
arch/powerpc/include/asm/percpu.h +0 -0
index 8e5b7d0b851c6..e24063eb0b33b 100644
@@ -15,6 +15,16 @@
 #endif /* CONFIG_SMP */
 #endif /* __powerpc64__ */
 

arch/powerpc/include/asm/percpu.h line 27

Quoted 10 lines
arch/powerpc/include/asm/percpu.h +10 -0
+#ifdef CONFIG_PPC64
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
+
+#define is_embed_first_chunk	\
+		(static_key_enabled(&__percpu_embed_first_chunk.key))
+#else
+#define is_embed_first_chunk	true
+#endif /* CONFIG_PPC64 */
+

Something like:

#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK #include <linux/jump_label.h> DECLARE_STATIC_KEY_FALSE(__percpu_first_chunk_is_paged);

#define percpu_first_chunk_is_paged \
(static_key_enabled(&__percpu_first_chunk_is_paged.key))
#else
#define percpu_first_chunk_is_paged false
#endif /* CONFIG_PPC64 */

Quoted 13 lines
arch/powerpc/kernel/setup_64.c +1 -0
index 2f19d5e944852..e04f0ff69d4b6 100644
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)
 
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
+DEFINE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
 
 void __init setup_per_cpu_areas(void)
 {

arch/powerpc/kernel/setup_64.c void __init setup_per_cpu_areas(void) · line 877

Quoted 8 lines
arch/powerpc/kernel/setup_64.c +2 -0
@@ -869,6 +870,8 @@ void __init setup_per_cpu_areas(void)
 			pr_warn("PERCPU: %s allocator failed (%d), "
 				"falling back to page size\n",
 				pcpu_fc_names[pcpu_chosen_fc], rc);
+		else
+			static_key_enable(&__percpu_embed_first_chunk.key);
 	}
 
 	if (rc < 0)

Finally, the current patch breaks the microwatt build:

  $ make microwatt_defconfig ; make -s -j (nproc)
  make[1]: Entering directory '/home/michael/linux/.build'
    GEN     Makefile
  #
  # configuration written to .config
  #
  make[1]: Leaving directory '/home/michael/linux/.build'
  ld: arch/powerpc/kernel/traps.o:(.toc+0x0): undefined reference to `__percpu_embed_first_chunk'
  ld: arch/powerpc/kernel/mce.o:(.toc+0x0): undefined reference to `__percpu_embed_first_chunk'
  make[3]: *** [../scripts/Makefile.vmlinux:37: vmlinux] Error 1

I guess because it has CONFIG_JUMP_LABEL=n?

cheers

#4 lore raw
Quoted 23 lines · Michael Ellerman

Hi Mahesh,

Quoted 20 lines · Mahesh Salgaonkar

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Quoted 5 lines
Quoted 2 lines

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

My system (powernv) doesn't even boot with percpu_alloc=page.

Can you share the crash details?

Quoted 11 lines

AFAIK the only reason we added support for it was to handle 4K kernels with HPT. See commit eb553f16973a ("powerpc/64/mm: implement page mapping percpu first chunk allocator").

So I wonder if we should change the Kconfig to only offer it as an option in that case, and change the logic in setup_per_cpu_areas() to only use it as a last resort.

I guess we probably still need this commit though, even if just for 4K HPT.

We have also observed some error when we have large gap between the start memory of NUMA nodes. That made the percpu offset really large causing boot failures even on 64K.

-aneesh

#5 lore raw
Quoted 22 lines · Aneesh Kumar K V
Quoted 21 lines · Michael Ellerman
Quoted 20 lines · Mahesh Salgaonkar

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

My system (powernv) doesn't even boot with percpu_alloc=page.

Can you share the crash details?

Yes but it's not pretty :)

  [    1.725257][  T714] systemd-journald[714]: Collecting audit messages is disabled.
  [    1.729401][    T1] systemd[1]: Finished systemd-tmpfiles-setup-dev.service - Create Static Device Nodes in /dev.
  [^[[0;32m  OK  ^[[0m] Finished ^[[0;1;39msystemd-tmpfiles-…reate Static Device Nodes in /dev.
  [    1.773902][   C22] Disabling lock debugging due to kernel taint
  [    1.773905][   C23] Oops: Machine check, sig: 7 [#1]
  [    1.773911][   C23] LE PAGE_SIZE=64K MMU=Radix SMP NR_CPUS=2048 NUMA PowerNV
  [    1.773916][   C23] Modules linked in:
  [    1.773920][   C23] CPU: 23 PID: 0 Comm: swapper/23 Tainted: G   M               6.8.0-rc7-02500-g23515c370cbb #1
  [    1.773924][   C23] Hardware name: 8335-GTH POWER9 0x4e1202 opal:skiboot-v6.5.3-35-g1851b2a06 PowerNV
  [    1.773926][   C23] NIP:  0000000000000000 LR: 0000000000000000 CTR: 0000000000000000
  [    1.773929][   C23] REGS: c000000fffa6ef50 TRAP: 0000   Tainted: G   M                (6.8.0-rc7-02500-g23515c370cbb)
  [    1.773932][   C23] MSR:  0000000000000000 <>  CR: 00000000  XER: 00000000
  [    1.773937][   C23] CFAR: 0000000000000000 IRQMASK: 3 
  [    1.773937][   C23] GPR00: 0000000000000000 c000000fffa6efe0 c000000fffa6efb0 0000000000000000 
  [    1.773937][   C23] GPR04: c00000000003d8c0 c000000001f5f000 0000000000000000 0000000000000103 
  [    1.773937][   C23] GPR08: 0000000000000003 653a0d962a590300 0000000000000000 0000000000000000 
  [    1.773937][   C23] GPR12: c000000fffa6f280 0000000000000000 c0000000000084a4 0000000000000000 
  [    1.773937][   C23] GPR16: 0000000053474552 0000000000000000 c00000000003d8c0 c000000fffa6f280 
  [    1.773937][   C23] GPR20: c000000001f5f000 c000000fffa6f340 c000000fffa6f2e8 0000000000000000 
  [    1.773937][   C23] GPR24: 0007fffffecf0000 c0000000065bbb80 0000000000550102 c000000002172b20 
  [    1.773937][   C23] GPR28: 0000000000000000 0000000053474552 0000000000000000 c000000ffffc6d80 
  [    1.773982][   C23] NIP [0000000000000000] 0x0
  [    1.773988][   C23] LR [0000000000000000] 0x0
  [    1.773990][   C23] Call Trace:
  [    1.773991][   C23] [c000000fffa6efe0] [c000000001f5f000] .TOC.+0x0/0xa1000 (unreliable)
  [    1.773999][   C23] Code: XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX 
  [    1.774021][   C23] ---[ end trace 0000000000000000 ]---

Something has gone badly wrong.

That was a test kernel with some other commits, but nothing that should cause that. Removing percpu_alloc=page fix it.

It's based on fddff98e83b4b4d54470902ea0d520c4d423ca3b.

Quoted 8 lines
Quoted 8 lines

AFAIK the only reason we added support for it was to handle 4K kernels with HPT. See commit eb553f16973a ("powerpc/64/mm: implement page mapping percpu first chunk allocator").

So I wonder if we should change the Kconfig to only offer it as an option in that case, and change the logic in setup_per_cpu_areas() to only use it as a last resort.

Quoted 5 lines
Quoted 3 lines

I guess we probably still need this commit though, even if just for 4K HPT.

We have also observed some error when we have large gap between the start memory of NUMA nodes. That made the percpu offset really large causing boot failures even on 64K.

Yeah, I have vague memories of that :)

cheers

#6 lore raw
Quoted 61 lines · Michael Ellerman
Quoted 22 lines · Aneesh Kumar K V
Quoted 21 lines · Michael Ellerman
Quoted 20 lines · Mahesh Salgaonkar

nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.

Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.

With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:

[    1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110
[    1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0
[    1.215719] --- interrupt: 200
[    1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable)
[    1.215722] [c000000fffd731b0] [0000000000000000] 0x0
[    1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.

My system (powernv) doesn't even boot with percpu_alloc=page.

Can you share the crash details?

Yes but it's not pretty :)

  [    1.725257][  T714] systemd-journald[714]: Collecting audit messages is disabled.
  [    1.729401][    T1] systemd[1]: Finished systemd-tmpfiles-setup-dev.service - Create Static Device Nodes in /dev.
  [^[[0;32m  OK  ^[[0m] Finished ^[[0;1;39msystemd-tmpfiles-…reate Static Device Nodes in /dev.
  [    1.773902][   C22] Disabling lock debugging due to kernel taint
  [    1.773905][   C23] Oops: Machine check, sig: 7 [#1]
  [    1.773911][   C23] LE PAGE_SIZE=64K MMU=Radix SMP NR_CPUS=2048 NUMA PowerNV
  [    1.773916][   C23] Modules linked in:
  [    1.773920][   C23] CPU: 23 PID: 0 Comm: swapper/23 Tainted: G   M               6.8.0-rc7-02500-g23515c370cbb #1
  [    1.773924][   C23] Hardware name: 8335-GTH POWER9 0x4e1202 opal:skiboot-v6.5.3-35-g1851b2a06 PowerNV
  [    1.773926][   C23] NIP:  0000000000000000 LR: 0000000000000000 CTR: 0000000000000000
  [    1.773929][   C23] REGS: c000000fffa6ef50 TRAP: 0000   Tainted: G   M                (6.8.0-rc7-02500-g23515c370cbb)
  [    1.773932][   C23] MSR:  0000000000000000 <>  CR: 00000000  XER: 00000000
  [    1.773937][   C23] CFAR: 0000000000000000 IRQMASK: 3 
  [    1.773937][   C23] GPR00: 0000000000000000 c000000fffa6efe0 c000000fffa6efb0 0000000000000000 
  [    1.773937][   C23] GPR04: c00000000003d8c0 c000000001f5f000 0000000000000000 0000000000000103 
  [    1.773937][   C23] GPR08: 0000000000000003 653a0d962a590300 0000000000000000 0000000000000000 
  [    1.773937][   C23] GPR12: c000000fffa6f280 0000000000000000 c0000000000084a4 0000000000000000 
  [    1.773937][   C23] GPR16: 0000000053474552 0000000000000000 c00000000003d8c0 c000000fffa6f280 
  [    1.773937][   C23] GPR20: c000000001f5f000 c000000fffa6f340 c000000fffa6f2e8 0000000000000000 
  [    1.773937][   C23] GPR24: 0007fffffecf0000 c0000000065bbb80 0000000000550102 c000000002172b20 
  [    1.773937][   C23] GPR28: 0000000000000000 0000000053474552 0000000000000000 c000000ffffc6d80 
  [    1.773982][   C23] NIP [0000000000000000] 0x0
  [    1.773988][   C23] LR [0000000000000000] 0x0
  [    1.773990][   C23] Call Trace:
  [    1.773991][   C23] [c000000fffa6efe0] [c000000001f5f000] .TOC.+0x0/0xa1000 (unreliable)
  [    1.773999][   C23] Code: XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX 
  [    1.774021][   C23] ---[ end trace 0000000000000000 ]---
Quoted 4 lines

Something has gone badly wrong.

That was a test kernel with some other commits, but nothing that should cause that. Removing percpu_alloc=page fix it.

So, when I try this without my patch "Avoid nmi_enter/nmi_exit in real mode interrupt", I see this getting recreated. However, I was not able to recrate this even once with my changes. Are you able to see this crash with my patch ?

Thanks,
-Mahesh.

#7 lore raw
In reply to Michael Ellerman
Quoted 17 lines · Michael Ellerman
Quoted 5 lines
arch/powerpc/include/asm/interrupt.h +0 -0
index a4196ab1d0167..0b96464ff0339 100644

arch/powerpc/include/asm/interrupt.h static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte · line 344

Quoted 9 lines
arch/powerpc/include/asm/interrupt.h +6 -0
@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
 	if (IS_ENABLED(CONFIG_KASAN))
 		return;
 
+	/*
+	 * Likewise, do not use it in real mode if percpu first chunk is not
+	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
+	 * are chances where percpu allocation can come from vmalloc area.
+	 */
+	if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk)

I think this would be clearer if it was inverted, eg:

Quoted 2 lines
        if (percpu_first_chunk_is_paged)
               return;

Agree.

Quoted 38 lines

That way you shouldn't need to check CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK here. Instead it can be part of the ifdef in the header.

Quoted 17 lines
arch/powerpc/include/asm/interrupt.h +2 -0
@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
 		// no nmi_exit for a pseries hash guest taking a real mode exception
 	} else if (IS_ENABLED(CONFIG_KASAN)) {
 		// no nmi_exit for KASAN in real mode
+	} else if (IS_ENABLED(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) && !is_embed_first_chunk) {
+		// no nmi_exit if percpu first chunk is not embedded
 	} else {
 		nmi_exit();
 	}
arch/powerpc/include/asm/percpu.h +0 -0
index 8e5b7d0b851c6..e24063eb0b33b 100644
@@ -15,6 +15,16 @@
 #endif /* CONFIG_SMP */
 #endif /* __powerpc64__ */
 

arch/powerpc/include/asm/percpu.h line 27

Quoted 10 lines
arch/powerpc/include/asm/percpu.h +10 -0
+#ifdef CONFIG_PPC64
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
+
+#define is_embed_first_chunk	\
+		(static_key_enabled(&__percpu_embed_first_chunk.key))
+#else
+#define is_embed_first_chunk	true
+#endif /* CONFIG_PPC64 */
+

Something like:

#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK #include <linux/jump_label.h> DECLARE_STATIC_KEY_FALSE(__percpu_first_chunk_is_paged);

Quoted 5 lines

#define percpu_first_chunk_is_paged \
(static_key_enabled(&__percpu_first_chunk_is_paged.key))
#else
#define percpu_first_chunk_is_paged false
#endif /* CONFIG_PPC64 */

Sure, will fix it.

Quoted 36 lines
Quoted 14 lines
arch/powerpc/kernel/setup_64.c +1 -0
index 2f19d5e944852..e04f0ff69d4b6 100644
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)
 
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
+DEFINE_STATIC_KEY_FALSE(__percpu_embed_first_chunk);
 
 void __init setup_per_cpu_areas(void)
 {

arch/powerpc/kernel/setup_64.c void __init setup_per_cpu_areas(void) · line 877

Quoted 8 lines
arch/powerpc/kernel/setup_64.c +2 -0
@@ -869,6 +870,8 @@ void __init setup_per_cpu_areas(void)
 			pr_warn("PERCPU: %s allocator failed (%d), "
 				"falling back to page size\n",
 				pcpu_fc_names[pcpu_chosen_fc], rc);
+		else
+			static_key_enable(&__percpu_embed_first_chunk.key);
 	}
 
 	if (rc < 0)

Finally, the current patch breaks the microwatt build:

  $ make microwatt_defconfig ; make -s -j (nproc)
  make[1]: Entering directory '/home/michael/linux/.build'
    GEN     Makefile
  #
  # configuration written to .config
  #
  make[1]: Leaving directory '/home/michael/linux/.build'
  ld: arch/powerpc/kernel/traps.o:(.toc+0x0): undefined reference to `__percpu_embed_first_chunk'
  ld: arch/powerpc/kernel/mce.o:(.toc+0x0): undefined reference to `__percpu_embed_first_chunk'
  make[3]: *** [../scripts/Makefile.vmlinux:37: vmlinux] Error 1
Quoted 1 line

I guess because it has CONFIG_JUMP_LABEL=n?

Even with CONFIG_JUMP_LABEL=n it should still work. Let me take look and fix this for microwatt build.

Thanks for your review.
-Mahesh.

Quoted 2 lines

cheers

Signature · 1 line
Mahesh J Salgaonkar