powerpc/crash: save cpu register data in crash_smp_send_stop()
PATCH 1/1 powerpc/crash: save cpu register data in crash_smp_send_stop()
Capture register data for secondary CPUs in crash_smp_send_stop() instead of doing it much later in crash_kexec_prepare_cpus() function with another set of NMI IPIs to secondary CPUs. This change avoids unnecessarily tricky post processing of data to get the right backtrace for these CPUs.
--- arch/powerpc/kernel/smp.c | 40 -------------------------- arch/powerpc/kexec/crash.c | 58 ++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 58 deletions(-)
arch/powerpc/kernel/smp.c +0 -40
@@ -60,7 +60,6 @@#include <asm/cpu_has_feature.h> #include <asm/ftrace.h> #include <asm/kup.h> -#include <asm/fadump.h> #ifdef DEBUG #include <asm/udbg.h>
@@ -620,45 +619,6 @@ void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) }#endif -#ifdef CONFIG_NMI_IPI -static void crash_stop_this_cpu(struct pt_regs *regs) -#else -static void crash_stop_this_cpu(void *dummy) -#endif -{ - /* - * Just busy wait here and avoid marking CPU as offline to ensure - * register data is captured appropriately. - */ - while (1) - cpu_relax(); -} - -void crash_smp_send_stop(void) -{ - static bool stopped = false; - - /* - * In case of fadump, register data for all CPUs is captured by f/w - * on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before - * this rtas call to avoid tricky post processing of those CPUs' - * backtraces. - */ - if (should_fadump_crash()) - return; - - if (stopped) - return; - - stopped = true; - -#ifdef CONFIG_NMI_IPI - smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_stop_this_cpu, 1000000); -#else - smp_call_function(crash_stop_this_cpu, NULL, 0); -#endif /* CONFIG_NMI_IPI */ -} -#ifdef CONFIG_NMI_IPI static void nmi_stop_this_cpu (struct pt_regs *regs) {
arch/powerpc/kexec/crash.c +40 -18
94 lines not on this page · raw message
Signature · 1 line
2.35.1
Nicholas Piggin
reviewerExcerpts from Hari Bathini's message of May 7, 2022 2:39 am:
Quoted 5 lines
Capture register data for secondary CPUs in
crash_smp_send_stop()instead of doing it much later incrash_kexec_prepare_cpus()function with another set of NMI IPIs to secondary CPUs. This change avoids unnecessarily tricky post processing of data to get the right backtrace for these CPUs.
Is the tricky post processing done in crash tools? Is it buggy in some situations or just fragile code you want to deprecate? Seems like a good goal either way
I assume the desire to stop secondaries ASAP is not just to get register data but also to limit the amount of damage they might cause to the crash process. Can they take interrupts or trigger the hard lockup watchdog, for example?
arch/powerpc/kernel/smp.c void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) · line 648
Quoted 12 lines
arch/powerpc/kernel/smp.c +0
-12-void crash_smp_send_stop (void ) -{ -static bool stopped = false; - -/* -* In case of fadump, register data for all CPUs is captured by f/w -* on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before -* this rtas call to avoid tricky post processing of those CPUs' -* backtraces. -*/ -if (should_fadump_crash ()) -return ;
This is not actually code you changed, but I wonder if it's wrong, if fadump is enabled then panic runs without stopping secondaries? Doesn't seem quite right.
arch/powerpc/kernel/smp.c void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) · line 660
Quoted 12 lines
arch/powerpc/kernel/smp.c +0
-12- -if (stopped) -return ; - - stopped = true; - -#ifdef CONFIG_NMI_IPI -smp_send_nmi_ipi (NMI_IPI_ALL_OTHERS , crash_stop_this_cpu,1000000 ); -#else -smp_call_function (crash_stop_this_cpu,NULL ,0 ); -#endif /* CONFIG_NMI_IPI */ -}
Now if kexec is not configured do we lose our crash_smp_send_stop function, or is it only ever called if kexec is enabled?
Quoted 62 lines
arch/powerpc/kernel/smp.c +0
-1-#ifdef CONFIG_NMI_IPI static void nmi_stop_this_cpu (struct pt_regs *regs) {arch/powerpc/kexec/crash.c +27
-2@@ -25,6 +25,7 @@#include <asm/setjmp.h> #include <asm/debug.h> #include <asm/interrupt.h> +#include <asm/fadump.h> /* * The primary CPU waits a while for all secondary CPUs to enter. This is to @@ -102,7 +103,7 @@ void crash_ipi_callback(struct pt_regs *regs)/* NOTREACHED */ } -static void crash_kexec_prepare_cpus (int cpu) +static void crash_kexec_prepare_cpus (void ) {unsigned int msecs;volatile unsigned int ncpus =num_online_cpus () -1 ;/* Excluding the panic cpu */ @@ -203,7 +204,7 @@ void crash_kexec_secondary(struct pt_regs *regs)#else /* ! CONFIG_SMP */ -static void crash_kexec_prepare_cpus (int cpu) +static void crash_kexec_prepare_cpus (void ) {/* * move the secondaries to us so that we can copy @@ -249,6 +250,42 @@ static void __maybe_unused crash_kexec_wait_realmode(int cpu)static inline void crash_kexec_wait_realmode (int cpu) {}#endif /* CONFIG_SMP && CONFIG_PPC64 */ +void crash_smp_send_stop (void ) +{ +static int cpus_stopped; + +/* +* In case of fadump, register data for all CPUs is captured by f/w +* on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before +* this rtas call to avoid tricky post processing of those CPUs' +* backtraces. +*/ +if (should_fadump_crash ()) +return ; + +if (cpus_stopped) +return ; + + cpus_stopped =1 ; + +/* Avoid hardlocking with irresponsive CPU holding logbuf_lock */ +printk_deferred_enter (); + +/* +* This function is only called after the system +* has panicked or is otherwise in a critical state.
arch/powerpc/kexec/crash.c static void __maybe_unused crash_kexec_wait_realmode(int cpu) · line 286
Quoted 10 lines
arch/powerpc/kexec/crash.c +10
-0+* The minimum amount of code to allow a kexec'd kernel +* to run successfully needs to happen here. +* +* In practice this means stopping other cpus in +* an SMP system. +* The kernel is broken so disable interrupts. +*/ +hard_irq_disable (); + +crash_kexec_prepare_cpus ();
This seems to move a bit of the kexec code around so this runs before notifiers in the panic path now. Maybe that's okay, I don't know this code too well, but how feasible would it be to have crash_stop_this_cpu() call crash_save_cpu()? And keeping the second IPI.
I do like the idea of removing the second IPI if possible, but that could be done later by moving the logic into crash_save_cpu() (it could just poll on a flag until the primary releases it to the next phase, rather than have the primary send another IPI).
Thanks,
Nick
Quoted 40 lines
arch/powerpc/kexec/crash.c +3
-16+} +/* * Register a function to be called on shutdown. Only use this if you * can't reset your device in the second kernel. @@ -312,21 +349,6 @@ void default_machine_crash_shutdown(struct pt_regs *regs)unsigned int i;int (*old_handler)(struct pt_regs *regs); -/* Avoid hardlocking with irresponsive CPU holding logbuf_lock */ -printk_deferred_enter (); - -/* -* This function is only called after the system -* has panicked or is otherwise in a critical state. -* The minimum amount of code to allow a kexec'd kernel -* to run successfully needs to happen here. -* -* In practice this means stopping other cpus in -* an SMP system. -* The kernel is broken so disable interrupts. -*/ -hard_irq_disable (); -/* * Make a note of crashing cpu. Will be used in machine_kexec * such that another IPI will not be sent. @@ -340,7 +362,7 @@ void default_machine_crash_shutdown(struct pt_regs *regs)if (TRAP (regs) ==INTERRUPT_SYSTEM_RESET )mdelay (PRIMARY_TIMEOUT ); -crash_kexec_prepare_cpus (crashing_cpu); +crash_smp_send_stop ();crash_save_cpu (regs, crashing_cpu);Signature · 1 line
2.35.1
Hi Nick,
Thanks for the review..
Quoted 8 lines · Nicholas Piggin
Excerpts from Hari Bathini's message of May 7, 2022 2:39 am:
Quoted 5 lines
Capture register data for secondary CPUs in
crash_smp_send_stop()instead of doing it much later incrash_kexec_prepare_cpus()function with another set of NMI IPIs to secondary CPUs. This change avoids unnecessarily tricky post processing of data to get the right backtrace for these CPUs.Is the tricky post processing done in crash tools?
Yeah. In tools like crash-utility that try to make sense of the register data captured.
Quoted 3 lines
Is it buggy in
some situations or just fragile code you want to deprecate? Seems like a good goal either way
The post processing may need looking up the emergency stack and
eventually tracing back to the regular stack. No code in crash-utility
to handle it currently though. This meant no proper backtrace, with crash-utility, for only cases like "crash_kexec_post_notifiers". But default cases will start having improper backtraces as well with
series [0] moving around crash_smp_send_stop().
Quoted 4 lines
I assume the desire to stop secondaries ASAP is not just to get register data but also to limit the amount of damage they might cause to the crash process. Can they take interrupts or trigger the hard lockup watchdog, for example?
True. Intention is to stop secondaries ASAP and make dump capture as smooth as possible...
Quoted 8 lines
Quoted 8 lines
arch/powerpc/kernel/smp.c +0
-8-void crash_smp_send_stop (void ) -{ -static bool stopped = false; - -/* -* In case of fadump, register data for all CPUs is captured by f/w -* on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before -* this rtas call to avoid tricky post processing of those CPUs'
Quoted 8 lines
arch/powerpc/kernel/smp.c void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) · line 648
Quoted 4 lines
arch/powerpc/kernel/smp.c +0
-4-* backtraces. -*/ -if (should_fadump_crash ()) -return ;This is not actually code you changed, but I wonder if it's wrong, if fadump is enabled then panic runs without stopping secondaries? Doesn't seem quite right.
So far, haven't seen any problem. F/W seems to handle it alright without having to stop the secondaries before the ibm,os-term call. But I do agree that stopping secondaries before calling into rtas sounds like the right thing to do, even if that meant processing more than one stack to get the proper backtrace..
Quoted 7 lines
Quoted 7 lines
arch/powerpc/kernel/smp.c +0
-7- -if (stopped) -return ; - - stopped = true; - -#ifdef CONFIG_NMI_IPI
Quoted 8 lines
arch/powerpc/kernel/smp.c void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) · line 660
Quoted 5 lines
arch/powerpc/kernel/smp.c +0
-5-smp_send_nmi_ipi (NMI_IPI_ALL_OTHERS , crash_stop_this_cpu,1000000 ); -#else -smp_call_function (crash_stop_this_cpu,NULL ,0 ); -#endif /* CONFIG_NMI_IPI */ -}Now if kexec is not configured do we lose our
crash_smp_send_stopfunction, or is it only ever called if kexec is enabled?
crash_smp_send_stop() is proposed to be called for both kdump and non-kdump cases as well with [1].
Quoted 70 lines
Quoted 70 lines
arch/powerpc/kernel/smp.c +0
-1-#ifdef CONFIG_NMI_IPI static void nmi_stop_this_cpu (struct pt_regs *regs) {arch/powerpc/kexec/crash.c +35
-2@@ -25,6 +25,7 @@#include <asm/setjmp.h> #include <asm/debug.h> #include <asm/interrupt.h> +#include <asm/fadump.h> /* * The primary CPU waits a while for all secondary CPUs to enter. This is to @@ -102,7 +103,7 @@ void crash_ipi_callback(struct pt_regs *regs)/* NOTREACHED */ } -static void crash_kexec_prepare_cpus (int cpu) +static void crash_kexec_prepare_cpus (void ) {unsigned int msecs;volatile unsigned int ncpus =num_online_cpus () -1 ;/* Excluding the panic cpu */ @@ -203,7 +204,7 @@ void crash_kexec_secondary(struct pt_regs *regs)#else /* ! CONFIG_SMP */ -static void crash_kexec_prepare_cpus (int cpu) +static void crash_kexec_prepare_cpus (void ) {/* * move the secondaries to us so that we can copy @@ -249,6 +250,42 @@ static void __maybe_unused crash_kexec_wait_realmode(int cpu)static inline void crash_kexec_wait_realmode (int cpu) {}#endif /* CONFIG_SMP && CONFIG_PPC64 */ +void crash_smp_send_stop (void ) +{ +static int cpus_stopped; + +/* +* In case of fadump, register data for all CPUs is captured by f/w +* on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before +* this rtas call to avoid tricky post processing of those CPUs' +* backtraces. +*/ +if (should_fadump_crash ()) +return ; + +if (cpus_stopped) +return ; + + cpus_stopped =1 ; + +/* Avoid hardlocking with irresponsive CPU holding logbuf_lock */ +printk_deferred_enter (); + +/* +* This function is only called after the system +* has panicked or is otherwise in a critical state. +* The minimum amount of code to allow a kexec'd kernel +* to run successfully needs to happen here. +* +* In practice this means stopping other cpus in +* an SMP system. +* The kernel is broken so disable interrupts. +*/ +hard_irq_disable ();
Quoted 8 lines
arch/powerpc/kexec/crash.c static void __maybe_unused crash_kexec_wait_realmode(int cpu) · line 286
Quoted 2 lines
arch/powerpc/kexec/crash.c +2
-0+ +crash_kexec_prepare_cpus ();This seems to move a bit of the kexec code around so this runs before notifiers in the panic path now. Maybe that's okay, I don't know this code too well, but how feasible would it be to have
crash_stop_this_cpu()callcrash_save_cpu()? And keeping the second IPI.
Yeah. With the series [0] being proposed, it makes sense to move crash_save_cpu() call to crash_stop_this_cpu() itself.
Quoted 4 lines
I do like the idea of removing the second IPI if possible, but that could be done later by moving the logic into
crash_save_cpu()(it could just poll on a flag until the primary releases it to the next phase, rather than have the primary send another IPI).
Also, polling for flag in crash_stop_this_cpu()/crash_save_cpu() and avoiding the second IPI sounds right. Will work on it.
Sorry about the delay in response. I lost track of this one.
Thanks
Hari
[0] https://lore.kernel.org/linuxppc-dev/[email protected]/
[1] https://lore.kernel.org/linuxppc-dev/[email protected]/