powerpc: Avoid nmi_enter/nmi_exit in real mode interrupt.
Subsystems (2)
PATCH v3 1/1 powerpc: Avoid nmi_enter/nmi_exit in real mode interrupt.
nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.
Early HMI/MCE handlers are called through DEFINE_INTERRUPT_HANDLER_NMI() wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However with CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there are chances where percpu allocation can come from the vmalloc area.
With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in machine_check_early:
[ 1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110 [ 1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0 [ 1.215719] --- interrupt: 200 [ 1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable) [ 1.215722] [c000000fffd731b0] [0000000000000000] 0x0 [ 1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8
Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.
---
Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at https://lore.kernel.org/linuxppc-dev/[email protected]/
Changes in v2:
- Rebase to upstream master
- Use jump_labels, if CONFIG_JUMP_LABEL is enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/
--- arch/powerpc/include/asm/interrupt.h | 10 ++++++++++ arch/powerpc/include/asm/percpu.h | 10 ++++++++++ arch/powerpc/kernel/setup_64.c | 4 ++++ 3 files changed, 24 insertions(+)
arch/powerpc/include/asm/interrupt.h +10 -0
@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inteif (IS_ENABLED (CONFIG_KASAN ))return ; +/* +* Likewise, do not use it in real mode if percpu first chunk is not +* embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there +* are chances where percpu allocation can come from vmalloc area. +*/ +if (IS_ENABLED (CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK ) && !is_embed_first_chunk) +return ; +/* Otherwise, it should be safe to call it */ nmi_enter (); }
@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter// no nmi_exit for a pseries hash guest taking a real mode exception }else if (IS_ENABLED (CONFIG_KASAN )) {// no nmi_exit for KASAN in real mode + }else if (IS_ENABLED (CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK ) && !is_embed_first_chunk) { +// no nmi_exit if percpu first chunk is not embedded }else {nmi_exit (); }
arch/powerpc/include/asm/percpu.h +10 -0
@@ -15,6 +15,16 @@#endif /* CONFIG_SMP */ #endif /* __powerpc64__ */ +#ifdef CONFIG_PPC64 +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_FALSE (__percpu_embed_first_chunk); + +#define is_embed_first_chunk \ + (static_key_enabled(&__percpu_embed_first_chunk.key)) +#else +#define is_embed_first_chunk (true) +#endif /* CONFIG_PPC64 */ +#include <asm-generic/percpu.h> #include <asm/paca.h>
arch/powerpc/kernel/setup_64.c +4 -0
@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)unsigned long __per_cpu_offset[NR_CPUS ]__read_mostly ;EXPORT_SYMBOL (__per_cpu_offset); +DEFINE_STATIC_KEY_FALSE (__percpu_embed_first_chunk);void __init setup_per_cpu_areas (void ) {
@@ -869,6 +870,9 @@ void __init setup_per_cpu_areas(void)pr_warn ("PERCPU: %s allocator failed (%d), " "falling back to page size\n" , pcpu_fc_names[pcpu_chosen_fc], rc); + else { +static_key_enable (&__percpu_embed_first_chunk.key); + } }if (rc <0 )
Signature · 1 line
2.43.0
Le 14/02/2024 à 10:16, Mahesh Salgaonkar a écrit :
Quoted 77 lines
[Vous ne recevez pas souvent de courriers de [email protected]. Découvrez pourquoi ceci est important à https://aka.ms/LearnAboutSenderIdentification ]
nmi_enter()/nmi_exit() touches per cpu variables which can lead to kernel crash when invoked during real mode interrupt handling (e.g. early HMI/MCE interrupt handler) if percpu allocation comes from vmalloc area.
Early HMI/MCE handlers are called through
DEFINE_INTERRUPT_HANDLER_NMI()wrapper which invokes nmi_enter/nmi_exit calls. We don't see any issue when percpu allocation is from the embedded first chunk. However withCONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNKenabled there are chances where percpu allocation can come from the vmalloc area.With kernel command line "percpu_alloc=page" we can force percpu allocation to come from vmalloc area and can see kernel crash in
machine_check_early:[ 1.215714] NIP [c000000000e49eb4] rcu_nmi_enter+0x24/0x110 [ 1.215717] LR [c0000000000461a0] machine_check_early+0xf0/0x2c0 [ 1.215719] --- interrupt: 200 [ 1.215720] [c000000fffd73180] [0000000000000000] 0x0 (unreliable) [ 1.215722] [c000000fffd731b0] [0000000000000000] 0x0 [ 1.215724] [c000000fffd73210] [c000000000008364] machine_check_early_common+0x134/0x1f8Fix this by avoiding use of nmi_enter()/nmi_exit() in real mode if percpu first chunk is not embedded.
Signed-off-by Mahesh Salgaonkar <[email protected]>---
Changes in v3:
- Address comments from Christophe Leroy to avoid using #ifdefs in the
code
- v2 at https://lore.kernel.org/linuxppc-dev/[email protected]/Changes in v2:
- Rebase to upstream master
- Usejump_labels, ifCONFIG_JUMP_LABELis enabled, to avoid redoing the
embed first chunk test at each interrupt entry.
- v1 is at https://lore.kernel.org/linuxppc-dev/164578465828.74956.6065296024817333750.stgit@jupiter/--- arch/powerpc/include/asm/interrupt.h | 10 ++++++++++ arch/powerpc/include/asm/percpu.h | 10 ++++++++++ arch/powerpc/kernel/setup_64.c | 4 ++++ 3 files changed, 24 insertions(+)arch/powerpc/include/asm/interrupt.h +10
-0@@ -336,6 +336,14 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inteif (IS_ENABLED (CONFIG_KASAN ))return ; +/* +* Likewise, do not use it in real mode if percpu first chunk is not +* embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there +* are chances where percpu allocation can come from vmalloc area. +*/ +if (IS_ENABLED (CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK ) && !is_embed_first_chunk) +return ; +/* Otherwise, it should be safe to call it */ nmi_enter (); }@@ -351,6 +359,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter// no nmi_exit for a pseries hash guest taking a real mode exception }else if (IS_ENABLED (CONFIG_KASAN )) {// no nmi_exit for KASAN in real mode + }else if (IS_ENABLED (CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK ) && !is_embed_first_chunk) { +// no nmi_exit if percpu first chunk is not embedded }else {nmi_exit (); }arch/powerpc/include/asm/percpu.h +0
-0@@ -15,6 +15,16 @@#endif /* CONFIG_SMP */
arch/powerpc/include/asm/percpu.h line 25
Quoted 10 lines
arch/powerpc/include/asm/percpu.h +8
-0#endif /* __powerpc64__ */ +#ifdef CONFIG_PPC64 +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_FALSE (__percpu_embed_first_chunk); + +#define is_embed_first_chunk \ + (static_key_enabled(&__percpu_embed_first_chunk.key)) +#else +#define is_embed_first_chunk (true)
Parenthesis aroud true are not necessary
Quoted 18 lines
arch/powerpc/include/asm/percpu.h +2
-0+#endif /* CONFIG_PPC64 */ +#include <asm-generic/percpu.h> #include <asm/paca.h> arch/powerpc/kernel/setup_64.c +1
-0@@ -834,6 +834,7 @@ static __init int pcpu_cpu_to_node(int cpu)unsigned long __per_cpu_offset[NR_CPUS ]__read_mostly ;EXPORT_SYMBOL (__per_cpu_offset); +DEFINE_STATIC_KEY_FALSE (__percpu_embed_first_chunk);void __init setup_per_cpu_areas (void ) {
arch/powerpc/kernel/setup_64.c void __init setup_per_cpu_areas(void) · line 875
Quoted 6 lines
arch/powerpc/kernel/setup_64.c +3
-0@@ -869,6 +870,9 @@ void __init setup_per_cpu_areas(void)pr_warn ("PERCPU: %s allocator failed (%d), " "falling back to page size\n" , pcpu_fc_names[pcpu_chosen_fc], rc); + else { +static_key_enable (&__percpu_embed_first_chunk.key); + }
Braces are unnecessary for single line, in addition it adds unbalanced bracing as the first leg doesn't have braces, see
https://docs.kernel.org/process/coding-style.html#placing-braces-and-spaces
Quoted 6 lines
arch/powerpc/kernel/setup_64.c +0
-1}if (rc <0 ) --2.43.0