powerpc/mce: Avoid using irq_work_queue() in realmode
PATCH v3 1/2 powerpc/mce: Avoid using irq_work_queue() in realmode
In realmode mce handler we use irq_work_queue() to defer the processing of mce events, irq_work_queue() can only be called when translation is enabled because it touches memory outside RMA, hence we enable translation before calling irq_work_queue and disable on return, though it is not safe to do in realmode.
To avoid this, program the decrementer and call the event processing functions from timer handler.
---
V2:
* Use arch_irq_work_raise to raise decrementer interrupt.
* Avoid having atomic variable.
V3:
* Fix build error.
Reported by kernel test bot.
--- arch/powerpc/include/asm/machdep.h | 2 + arch/powerpc/include/asm/mce.h | 2 + arch/powerpc/include/asm/paca.h | 1 + arch/powerpc/kernel/mce.c | 51 +++++++++++------------- arch/powerpc/kernel/time.c | 3 ++ arch/powerpc/platforms/pseries/pseries.h | 1 + arch/powerpc/platforms/pseries/ras.c | 31 +------------- arch/powerpc/platforms/pseries/setup.c | 1 + 8 files changed, 34 insertions(+), 58 deletions(-)
arch/powerpc/include/asm/machdep.h +2 -0
@@ -99,6 +99,8 @@ struct machdep_calls {/* Called during machine check exception to retrive fixup address. */ bool (*mce_check_early_recovery)(struct pt_regs *regs); +void (*machine_check_log_err)(void ); +/* Motherboard/chipset features. This is a kind of general purpose * hook used to control some machine specific features (like reset * lines, chip power control, etc...).
arch/powerpc/include/asm/mce.h +2 -0
@@ -235,8 +235,10 @@ extern void machine_check_print_event_info(struct machine_check_event *evt,unsigned long addr_to_pfn (struct pt_regs *regs,unsigned long addr);extern void mce_common_process_ue (struct pt_regs *regs,struct mce_error_info *mce_err); +void machine_check_raise_dec_intr (void );int mce_register_notifier (struct notifier_block *nb);int mce_unregister_notifier (struct notifier_block *nb); +void mce_run_late_handlers (void );#ifdef CONFIG_PPC_BOOK3S_64 void flush_and_reload_slb (void );void flush_erat (void );
arch/powerpc/include/asm/paca.h +1 -0
@@ -280,6 +280,7 @@ struct paca_struct {#endif #ifdef CONFIG_PPC_BOOK3S_64 struct mce_info *mce_info; +u32 mces_to_process;#endif /* CONFIG_PPC_BOOK3S_64 */ }____cacheline_aligned ;
arch/powerpc/kernel/mce.c +23 -28
108 lines not on this page · raw message
arch/powerpc/kernel/time.c +3 -0
@@ -594,6 +594,9 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)if (test_irq_work_pending ()) {clear_irq_work_pending (); +#ifdef CONFIG_PPC_BOOK3S_64 +mce_run_late_handlers (); +#endif irq_work_run (); }
arch/powerpc/platforms/pseries/pseries.h +1 -0
@@ -21,6 +21,7 @@ struct pt_regs;extern int pSeries_system_reset_exception (struct pt_regs *regs);extern int pSeries_machine_check_exception (struct pt_regs *regs);extern long pseries_machine_check_realmode (struct pt_regs *regs); +void pSeries_machine_check_log_err (void );#ifdef CONFIG_SMP extern void smp_init_pseries (void );
arch/powerpc/platforms/pseries/ras.c +1 -30
@@ -23,11 +23,6 @@ static DEFINE_SPINLOCK(ras_log_buf_lock);static int ras_check_exception_token; -static void mce_process_errlog_event (struct irq_work *work); -static struct irq_work mce_errlog_process_work = { - .func = mce_process_errlog_event, -}; -#define EPOW_SENSOR_TOKEN 9#define EPOW_SENSOR_INDEX 0
@@ -729,40 +724,16 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) error_type = mce_log->error_type; disposition =mce_handle_err_realmode (disposition, error_type); - -/* -* Enable translation as we will be accessing per-cpu variables -* in save_mce_event() which may fall outside RMO region, also -* leave it enabled because subsequently we will be queuing work -* to workqueues where again per-cpu variables accessed, besides -* fwnmi_release_errinfo() crashes when called in realmode on -* pseries. -* Note: All the realmode handling like flushing SLB entries for -* SLB multihit is done by now. -*/ out: - msr =mfmsr (); -mtmsr (msr |MSR_IR |MSR_DR ); - disposition =mce_handle_err_virtmode (regs, errp, mce_log, disposition); - -/* -* Queue irq work to log this rtas event later. -* irq_work_queue uses per-cpu variables, so do this in virt -* mode as well. -*/ -irq_work_queue (&mce_errlog_process_work); - -mtmsr (msr); -return disposition; }/* * Process MCE rtas errlog event. */ -static void mce_process_errlog_event (struct irq_work *work) +void pSeries_machine_check_log_err (void ) {struct rtas_error_log *err;
arch/powerpc/platforms/pseries/setup.c +1 -0
@@ -1084,6 +1084,7 @@ define_machine(pseries) { .system_reset_exception = pSeries_system_reset_exception, .machine_check_early = pseries_machine_check_realmode, .machine_check_exception = pSeries_machine_check_exception, + .machine_check_log_err = pSeries_machine_check_log_err, #ifdef CONFIG_KEXEC_CORE .machine_kexec = pSeries_machine_kexec, .kexec_cpu_down = pseries_kexec_cpu_down,
Signature · 1 line
2.31.1
PATCH v3 2/2 pseries/mce: Refactor the pseries mce handling code
Now that we are no longer switching on the mmu in realmode mce handler, Revert the commit 4ff753feab02("powerpc/pseries:
Avoid using addr_to_pfn in real mode") partially, which introduced functions mce_handle_err_virtmode/realmode() to separate mce handler code which needed translation to enabled.
--- arch/powerpc/platforms/pseries/ras.c | 122 +++++++++++---------------- 1 file changed, 49 insertions(+), 73 deletions(-)
arch/powerpc/platforms/pseries/ras.c +49 -73
@@ -511,58 +511,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs)return 0 ;/* need to perform reset */ } -static int mce_handle_err_realmode (int disposition,u8 error_type) -{ -#ifdef CONFIG_PPC_BOOK3S_64 -if (disposition ==RTAS_DISP_NOT_RECOVERED ) { -switch (error_type) { -case MC_ERROR_TYPE_ERAT : -flush_erat (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -case MC_ERROR_TYPE_SLB : -/* -* Store the old slb content in paca before flushing. -* Print this when we go to virtual mode. -* There are chances that we may hit MCE again if there -* is a parity error on the SLB entry we trying to read -* for saving. Hence limit the slb saving to single -* level of recursion. -*/ -if (local_paca->in_mce ==1 ) -slb_save_contents (local_paca->mce_faulty_slbs); -flush_and_reload_slb (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -default : -break ; - } - }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { -/* Platform corrected itself but could be degraded */ -pr_err ("MCE: limited recovery, system may be degraded\n" ); - disposition =RTAS_DISP_FULLY_RECOVERED ; - } -#endif -return disposition; -} - -static int mce_handle_err_virtmode (struct pt_regs *regs, -struct rtas_error_log *errp, -struct pseries_mc_errorlog *mce_log, -int disposition) +static int mce_handle_error (struct pt_regs *regs,struct rtas_error_log *errp) {struct mce_error_info mce_err = {0 }; +unsigned long eaddr =0 , paddr =0 ; +struct pseries_errorlog *pseries_log; +struct pseries_mc_errorlog *mce_log; +int disposition =rtas_error_disposition (errp);int initiator =rtas_error_initiator (errp);int severity =rtas_error_severity (errp); -unsigned long eaddr =0 , paddr =0 ;u8 error_type, err_sub_type; -if (!mce_log) - goto out; - - error_type = mce_log->error_type; - err_sub_type =rtas_mc_error_sub_type (mce_log); -if (initiator ==RTAS_INITIATOR_UNKNOWN ) mce_err.initiator =MCE_INITIATOR_UNKNOWN ;else if (initiator ==RTAS_INITIATOR_CPU )
@@ -588,6 +547,8 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.severity =MCE_SEV_SEVERE ; else if (severity == RTAS_SEVERITY_ERROR) mce_err.severity = MCE_SEV_SEVERE; + else if (severity == RTAS_SEVERITY_FATAL) + mce_err.severity = MCE_SEV_FATAL; else mce_err.severity = MCE_SEV_FATAL;
@@ -599,7 +560,18 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ; mce_err.error_class =MCE_ECLASS_UNKNOWN ; -switch (error_type) { +if (!rtas_error_extended (errp)) + goto out; + + pseries_log =get_pseries_errorlog (errp,PSERIES_ELOG_SECT_ID_MCE ); +if (!pseries_log) + goto out; + + mce_log = (struct pseries_mc_errorlog *)pseries_log->data; + error_type = mce_log->error_type; + err_sub_type =rtas_mc_error_sub_type (mce_log); + +switch (mce_log->error_type) {case MC_ERROR_TYPE_UE : mce_err.error_type =MCE_ERROR_TYPE_UE ;mce_common_process_ue (regs, &mce_err);
@@ -692,41 +664,45 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;case MC_ERROR_TYPE_I_CACHE : - mce_err.error_type =MCE_ERROR_TYPE_ICACHE ; + mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;case MC_ERROR_TYPE_UNKNOWN :default : mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ;break ; } + +#ifdef CONFIG_PPC_BOOK3S_64 +if (disposition ==RTAS_DISP_NOT_RECOVERED ) { +switch (error_type) { +case MC_ERROR_TYPE_SLB : +case MC_ERROR_TYPE_ERAT : +/* +* Store the old slb content in paca before flushing. +* Print this when we go to virtual mode. +* There are chances that we may hit MCE again if there +* is a parity error on the SLB entry we trying to read +* for saving. Hence limit the slb saving to single +* level of recursion. +*/ +if (local_paca->in_mce ==1 ) +slb_save_contents (local_paca->mce_faulty_slbs); +flush_and_reload_slb (); + disposition =RTAS_DISP_FULLY_RECOVERED ; +break ; +default : +break ; + } + }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { +/* Platform corrected itself but could be degraded */ +pr_err ("MCE: limited recovery, system may be degraded\n" ); + disposition =RTAS_DISP_FULLY_RECOVERED ; + } +#endif out:save_mce_event (regs, disposition ==RTAS_DISP_FULLY_RECOVERED , - &mce_err, regs->nip, eaddr, paddr); -return disposition; -} + &mce_err, regs->nip, eaddr, paddr); -static int mce_handle_error (struct pt_regs *regs,struct rtas_error_log *errp) -{ -struct pseries_errorlog *pseries_log; -struct pseries_mc_errorlog *mce_log =NULL ; -int disposition =rtas_error_disposition (errp); -unsigned long msr; -u8 error_type; - -if (!rtas_error_extended (errp)) - goto out; - - pseries_log =get_pseries_errorlog (errp,PSERIES_ELOG_SECT_ID_MCE ); -if (!pseries_log) - goto out; - - mce_log = (struct pseries_mc_errorlog *)pseries_log->data; - error_type = mce_log->error_type; - - disposition =mce_handle_err_realmode (disposition, error_type); -out: - disposition =mce_handle_err_virtmode (regs, errp, mce_log, - disposition);return disposition; }
Signature · 1 line
2.31.1
Nicholas Piggin
reviewerRe: [PATCH v3 2/2] pseries/mce: Refactor the pseries mce handling code
Excerpts from Ganesh Goudar's message of November 24, 2021 7:55 pm:
Quoted 80 lines
Now that we are no longer switching on the mmu in realmode mce handler, Revert the commit 4ff753feab02("powerpc/pseries:
Avoid usingaddr_to_pfnin real mode") partially, which introduced functions mce_handle_err_virtmode/realmode() to separate mce handler code which needed translation to enabled.Signed-off-by Ganesh Goudar <[email protected]>--- arch/powerpc/platforms/pseries/ras.c | 122 +++++++++++---------------- 1 file changed, 49 insertions(+), 73 deletions(-)arch/powerpc/platforms/pseries/ras.c +5
-46@@ -511,58 +511,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs)return 0 ;/* need to perform reset */ } -static int mce_handle_err_realmode (int disposition,u8 error_type) -{ -#ifdef CONFIG_PPC_BOOK3S_64 -if (disposition ==RTAS_DISP_NOT_RECOVERED ) { -switch (error_type) { -case MC_ERROR_TYPE_ERAT : -flush_erat (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -case MC_ERROR_TYPE_SLB : -/* -* Store the old slb content in paca before flushing. -* Print this when we go to virtual mode. -* There are chances that we may hit MCE again if there -* is a parity error on the SLB entry we trying to read -* for saving. Hence limit the slb saving to single -* level of recursion. -*/ -if (local_paca->in_mce ==1 ) -slb_save_contents (local_paca->mce_faulty_slbs); -flush_and_reload_slb (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -default : -break ; - } - }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { -/* Platform corrected itself but could be degraded */ -pr_err ("MCE: limited recovery, system may be degraded\n" ); - disposition =RTAS_DISP_FULLY_RECOVERED ; - } -#endif -return disposition; -} - -static int mce_handle_err_virtmode (struct pt_regs *regs, -struct rtas_error_log *errp, -struct pseries_mc_errorlog *mce_log, -int disposition) +static int mce_handle_error (struct pt_regs *regs,struct rtas_error_log *errp) {struct mce_error_info mce_err = {0 }; +unsigned long eaddr =0 , paddr =0 ; +struct pseries_errorlog *pseries_log; +struct pseries_mc_errorlog *mce_log; +int disposition =rtas_error_disposition (errp);int initiator =rtas_error_initiator (errp);int severity =rtas_error_severity (errp); -unsigned long eaddr =0 , paddr =0 ;u8 error_type, err_sub_type; -if (!mce_log) - goto out; - - error_type = mce_log->error_type; - err_sub_type =rtas_mc_error_sub_type (mce_log); -if (initiator ==RTAS_INITIATOR_UNKNOWN ) mce_err.initiator =MCE_INITIATOR_UNKNOWN ;else if (initiator ==RTAS_INITIATOR_CPU )
arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 554
Quoted 8 lines
arch/powerpc/platforms/pseries/ras.c +2
-0@@ -588,6 +547,8 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.severity =MCE_SEV_SEVERE ; else if (severity == RTAS_SEVERITY_ERROR) mce_err.severity = MCE_SEV_SEVERE; + else if (severity == RTAS_SEVERITY_FATAL) + mce_err.severity = MCE_SEV_FATAL; else mce_err.severity = MCE_SEV_FATAL;
What's this hunk for?
Quoted 21 lines
arch/powerpc/platforms/pseries/ras.c +12
-1@@ -599,7 +560,18 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ; mce_err.error_class =MCE_ECLASS_UNKNOWN ; -switch (error_type) { +if (!rtas_error_extended (errp)) + goto out; + + pseries_log =get_pseries_errorlog (errp,PSERIES_ELOG_SECT_ID_MCE ); +if (!pseries_log) + goto out; + + mce_log = (struct pseries_mc_errorlog *)pseries_log->data; + error_type = mce_log->error_type; + err_sub_type =rtas_mc_error_sub_type (mce_log); + +switch (mce_log->error_type) {case MC_ERROR_TYPE_UE : mce_err.error_type =MCE_ERROR_TYPE_UE ;mce_common_process_ue (regs, &mce_err);
arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 668
Quoted 6 lines
arch/powerpc/platforms/pseries/ras.c +1
-1@@ -692,41 +664,45 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;case MC_ERROR_TYPE_I_CACHE : - mce_err.error_type =MCE_ERROR_TYPE_ICACHE ; + mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;
And this one. Doesn't look right.
Quoted 22 lines
arch/powerpc/platforms/pseries/ras.c +17
-0case MC_ERROR_TYPE_UNKNOWN :default : mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ;break ; } + +#ifdef CONFIG_PPC_BOOK3S_64 +if (disposition ==RTAS_DISP_NOT_RECOVERED ) { +switch (error_type) { +case MC_ERROR_TYPE_SLB : +case MC_ERROR_TYPE_ERAT : +/* +* Store the old slb content in paca before flushing. +* Print this when we go to virtual mode. +* There are chances that we may hit MCE again if there +* is a parity error on the SLB entry we trying to read +* for saving. Hence limit the slb saving to single +* level of recursion. +*/ +if (local_paca->in_mce ==1 ) +slb_save_contents (local_paca->mce_faulty_slbs); +flush_and_reload_slb ();
arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 700
Quoted 10 lines
arch/powerpc/platforms/pseries/ras.c +10
-0+ disposition =RTAS_DISP_FULLY_RECOVERED ; +break ; +default : +break ; + } + }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { +/* Platform corrected itself but could be degraded */ +pr_err ("MCE: limited recovery, system may be degraded\n" ); + disposition =RTAS_DISP_FULLY_RECOVERED ; + }
I would prefer if you just keep the mce_handle_err_realmode function (can rename it if you want). It's actually changed a bit since the patch being reverted so we don't want to undo that.
Thanks,
Nick
Re: [PATCH v3 2/2] pseries/mce: Refactor the pseries mce handling code
Quoted 147 lines · Nicholas Piggin
Excerpts from Ganesh Goudar's message of November 24, 2021 7:55 pm:
Quoted 80 lines
Now that we are no longer switching on the mmu in realmode mce handler, Revert the commit 4ff753feab02("powerpc/pseries:
Avoid usingaddr_to_pfnin real mode") partially, which introduced functions mce_handle_err_virtmode/realmode() to separate mce handler code which needed translation to enabled.Signed-off-by Ganesh Goudar<[email protected]>--- arch/powerpc/platforms/pseries/ras.c | 122 +++++++++++---------------- 1 file changed, 49 insertions(+), 73 deletions(-)arch/powerpc/platforms/pseries/ras.c +5
-46@@ -511,58 +511,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs)return 0 ;/* need to perform reset */ } -static int mce_handle_err_realmode (int disposition,u8 error_type) -{ -#ifdef CONFIG_PPC_BOOK3S_64 -if (disposition ==RTAS_DISP_NOT_RECOVERED ) { -switch (error_type) { -case MC_ERROR_TYPE_ERAT : -flush_erat (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -case MC_ERROR_TYPE_SLB : -/* -* Store the old slb content in paca before flushing. -* Print this when we go to virtual mode. -* There are chances that we may hit MCE again if there -* is a parity error on the SLB entry we trying to read -* for saving. Hence limit the slb saving to single -* level of recursion. -*/ -if (local_paca->in_mce ==1 ) -slb_save_contents (local_paca->mce_faulty_slbs); -flush_and_reload_slb (); - disposition =RTAS_DISP_FULLY_RECOVERED ; -break ; -default : -break ; - } - }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { -/* Platform corrected itself but could be degraded */ -pr_err ("MCE: limited recovery, system may be degraded\n" ); - disposition =RTAS_DISP_FULLY_RECOVERED ; - } -#endif -return disposition; -} - -static int mce_handle_err_virtmode (struct pt_regs *regs, -struct rtas_error_log *errp, -struct pseries_mc_errorlog *mce_log, -int disposition) +static int mce_handle_error (struct pt_regs *regs,struct rtas_error_log *errp) {struct mce_error_info mce_err = {0 }; +unsigned long eaddr =0 , paddr =0 ; +struct pseries_errorlog *pseries_log; +struct pseries_mc_errorlog *mce_log; +int disposition =rtas_error_disposition (errp);int initiator =rtas_error_initiator (errp);int severity =rtas_error_severity (errp); -unsigned long eaddr =0 , paddr =0 ;u8 error_type, err_sub_type; -if (!mce_log) - goto out; - - error_type = mce_log->error_type; - err_sub_type =rtas_mc_error_sub_type (mce_log); -if (initiator ==RTAS_INITIATOR_UNKNOWN ) mce_err.initiator =MCE_INITIATOR_UNKNOWN ;else if (initiator ==RTAS_INITIATOR_CPU )arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 554
Quoted 8 lines
arch/powerpc/platforms/pseries/ras.c +2
-0@@ -588,6 +547,8 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.severity =MCE_SEV_SEVERE ; else if (severity == RTAS_SEVERITY_ERROR) mce_err.severity = MCE_SEV_SEVERE; + else if (severity == RTAS_SEVERITY_FATAL) + mce_err.severity = MCE_SEV_FATAL; else mce_err.severity = MCE_SEV_FATAL;What's this hunk for?
Quoted 21 lines
arch/powerpc/platforms/pseries/ras.c +12
-1@@ -599,7 +560,18 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ; mce_err.error_class =MCE_ECLASS_UNKNOWN ; -switch (error_type) { +if (!rtas_error_extended (errp)) + goto out; + + pseries_log =get_pseries_errorlog (errp,PSERIES_ELOG_SECT_ID_MCE ); +if (!pseries_log) + goto out; + + mce_log = (struct pseries_mc_errorlog *)pseries_log->data; + error_type = mce_log->error_type; + err_sub_type =rtas_mc_error_sub_type (mce_log); + +switch (mce_log->error_type) {case MC_ERROR_TYPE_UE : mce_err.error_type =MCE_ERROR_TYPE_UE ;mce_common_process_ue (regs, &mce_err);arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 668
Quoted 6 lines
arch/powerpc/platforms/pseries/ras.c +1
-1@@ -692,41 +664,45 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;case MC_ERROR_TYPE_I_CACHE : - mce_err.error_type =MCE_ERROR_TYPE_ICACHE ; + mce_err.error_type =MCE_ERROR_TYPE_DCACHE ;break ;And this one. Doesn't look right.
Quoted 27 lines
arch/powerpc/platforms/pseries/ras.c +22
-0case MC_ERROR_TYPE_UNKNOWN :default : mce_err.error_type =MCE_ERROR_TYPE_UNKNOWN ;break ; } + +#ifdef CONFIG_PPC_BOOK3S_64 +if (disposition ==RTAS_DISP_NOT_RECOVERED ) { +switch (error_type) { +case MC_ERROR_TYPE_SLB : +case MC_ERROR_TYPE_ERAT : +/* +* Store the old slb content in paca before flushing. +* Print this when we go to virtual mode. +* There are chances that we may hit MCE again if there +* is a parity error on the SLB entry we trying to read +* for saving. Hence limit the slb saving to single +* level of recursion. +*/ +if (local_paca->in_mce ==1 ) +slb_save_contents (local_paca->mce_faulty_slbs); +flush_and_reload_slb (); + disposition =RTAS_DISP_FULLY_RECOVERED ; +break ; +default : +break ; + }
Quoted 8 lines
arch/powerpc/platforms/pseries/ras.c static int mce_handle_err_virtmode(struct pt_regs *regs, · line 700
Quoted 5 lines
arch/powerpc/platforms/pseries/ras.c +5
-0+ }else if (disposition ==RTAS_DISP_LIMITED_RECOVERY ) { +/* Platform corrected itself but could be degraded */ +pr_err ("MCE: limited recovery, system may be degraded\n" ); + disposition =RTAS_DISP_FULLY_RECOVERED ; + }I would prefer if you just keep the
mce_handle_err_realmodefunction (can rename it if you want). It's actually changed a bit since the patch being reverted so we don't want to undo that.
Ok, I will leave it as is for now, we can change it later.
Quoted 3 lines
Thanks,
Nick
Nicholas Piggin
reviewerExcerpts from Ganesh Goudar's message of November 24, 2021 7:54 pm:
Quoted 95 lines
In realmode mce handler we use
irq_work_queue()to defer the processing of mce events,irq_work_queue()can only be called when translation is enabled because it touches memory outside RMA, hence we enable translation before callingirq_work_queueand disable on return, though it is not safe to do in realmode.To avoid this, program the decrementer and call the event processing functions from timer handler.
Signed-off-by Ganesh Goudar <[email protected]>---
V2:
* Usearch_irq_work_raiseto raise decrementer interrupt.
* Avoid having atomic variable.V3:
* Fix build error.
Reported by kernel test bot.--- arch/powerpc/include/asm/machdep.h | 2 + arch/powerpc/include/asm/mce.h | 2 + arch/powerpc/include/asm/paca.h | 1 + arch/powerpc/kernel/mce.c | 51 +++++++++++------------- arch/powerpc/kernel/time.c | 3 ++ arch/powerpc/platforms/pseries/pseries.h | 1 + arch/powerpc/platforms/pseries/ras.c | 31 +------------- arch/powerpc/platforms/pseries/setup.c | 1 + 8 files changed, 34 insertions(+), 58 deletions(-)arch/powerpc/include/asm/machdep.h +2
-0@@ -99,6 +99,8 @@ struct machdep_calls {/* Called during machine check exception to retrive fixup address. */ bool (*mce_check_early_recovery)(struct pt_regs *regs); +void (*machine_check_log_err)(void ); +/* Motherboard/chipset features. This is a kind of general purpose * hook used to control some machine specific features (like reset * lines, chip power control, etc...). arch/powerpc/include/asm/mce.h +2
-0@@ -235,8 +235,10 @@ extern void machine_check_print_event_info(struct machine_check_event *evt,unsigned long addr_to_pfn (struct pt_regs *regs,unsigned long addr);extern void mce_common_process_ue (struct pt_regs *regs,struct mce_error_info *mce_err); +void machine_check_raise_dec_intr (void );int mce_register_notifier (struct notifier_block *nb);int mce_unregister_notifier (struct notifier_block *nb); +void mce_run_late_handlers (void );#ifdef CONFIG_PPC_BOOK3S_64 void flush_and_reload_slb (void );void flush_erat (void );arch/powerpc/include/asm/paca.h +1
-0@@ -280,6 +280,7 @@ struct paca_struct {#endif #ifdef CONFIG_PPC_BOOK3S_64 struct mce_info *mce_info; +u32 mces_to_process;#endif /* CONFIG_PPC_BOOK3S_64 */ }____cacheline_aligned ;arch/powerpc/kernel/mce.c +0
-10@@ -28,19 +28,9 @@#include "setup.h" -static void machine_check_process_queued_event (struct irq_work *work); -static void machine_check_ue_irq_work (struct irq_work *work);static void machine_check_ue_event (struct machine_check_event *evt);static void machine_process_ue_event (struct work_struct *work); -static struct irq_work mce_event_process_work = { - .func = machine_check_process_queued_event, -}; - -static struct irq_work mce_ue_event_irq_work = { - .func = machine_check_ue_irq_work, -}; - static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);static BLOCKING_NOTIFIER_HEAD (mce_notifier_list);
arch/powerpc/kernel/mce.c static void mce_set_error_info(struct machine_check_event *mce, · line 86
Quoted 8 lines
arch/powerpc/kernel/mce.c +5
-0@@ -89,6 +79,12 @@ static void mce_set_error_info(struct machine_check_event *mce, } } +/* Raise decrementer interrupt */ +void machine_check_raise_dec_intr (void ) +{ +arch_irq_work_raise (); +}
It would be better if the name specifically related to irq work, which is more than just dec interrupt. It might be good to set mces_to_process here as well.
I would name it something like mce_irq_work_queue, and the paca variable to mce_pending_irq_work...
arch/powerpc/kernel/mce.c static void machine_check_process_queued_event(struct irq_work *work) · line 359
Quoted 10 lines
arch/powerpc/kernel/mce.c +10
-0+void mce_run_late_handlers (void ) +{ +if (unlikely (local_paca->mces_to_process)) { +if (ppc_md.machine_check_log_err) + ppc_md.machine_check_log_err (); +machine_check_process_queued_event (); +machine_check_ue_work (); + local_paca->mces_to_process--; + } +}
The problem with a counter is that you're clearing the irq work pending in the timer interrupt, so you'll never call in here again to clear that (until something else sets irq work).
But as far as I can see it does not need to be a counter, just a flag. The machine check calls will process multiple events, right? (and the current irq_work queue does not queue the same work multiple times).
Oh. That's actually bad, isn't it? Our irq work should be per-CPU because the callbacks are mainly only operating on the local paca queued events, so we have a longstanding bug there AFAIKS. Your patch will solve it if everything is converted over.
Quoted 9 lines
arch/powerpc/kernel/mce.c +1
-0+void machine_check_print_event_info (struct machine_check_event *evt,bool user_mode,bool in_guest) {arch/powerpc/kernel/time.c +0
-0
arch/powerpc/kernel/time.c DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) · line 599
Quoted 6 lines
arch/powerpc/kernel/time.c +3
-0@@ -594,6 +594,9 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)if (test_irq_work_pending ()) {clear_irq_work_pending (); +#ifdef CONFIG_PPC_BOOK3S_64 +mce_run_late_handlers (); +#endif
Maybe create a no-op inline function for others and call unconditionally here. I wonder if the name could be better, we have lots of handlers, of varying earliness. real-mode, then virt mode NMI context, then IRQ context, then workqueue context.
mce_run_irq_context_handlers() might not be much better though.
Quoted 89 lines
arch/powerpc/kernel/time.c +0
-0irq_work_run (); }arch/powerpc/platforms/pseries/pseries.h +1
-0@@ -21,6 +21,7 @@ struct pt_regs;extern int pSeries_system_reset_exception (struct pt_regs *regs);extern int pSeries_machine_check_exception (struct pt_regs *regs);extern long pseries_machine_check_realmode (struct pt_regs *regs); +void pSeries_machine_check_log_err (void );#ifdef CONFIG_SMP extern void smp_init_pseries (void );arch/powerpc/platforms/pseries/ras.c +1
-30@@ -23,11 +23,6 @@ static DEFINE_SPINLOCK(ras_log_buf_lock);static int ras_check_exception_token; -static void mce_process_errlog_event (struct irq_work *work); -static struct irq_work mce_errlog_process_work = { - .func = mce_process_errlog_event, -}; -#define EPOW_SENSOR_TOKEN 9#define EPOW_SENSOR_INDEX 0@@ -729,40 +724,16 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) error_type = mce_log->error_type; disposition =mce_handle_err_realmode (disposition, error_type); - -/* -* Enable translation as we will be accessing per-cpu variables -* in save_mce_event() which may fall outside RMO region, also -* leave it enabled because subsequently we will be queuing work -* to workqueues where again per-cpu variables accessed, besides -* fwnmi_release_errinfo() crashes when called in realmode on -* pseries. -* Note: All the realmode handling like flushing SLB entries for -* SLB multihit is done by now. -*/ out: - msr =mfmsr (); -mtmsr (msr |MSR_IR |MSR_DR ); - disposition =mce_handle_err_virtmode (regs, errp, mce_log, disposition); - -/* -* Queue irq work to log this rtas event later. -* irq_work_queue uses per-cpu variables, so do this in virt -* mode as well. -*/ -irq_work_queue (&mce_errlog_process_work); - -mtmsr (msr); -return disposition; }/* * Process MCE rtas errlog event. */ -static void mce_process_errlog_event (struct irq_work *work) +void pSeries_machine_check_log_err (void ) {struct rtas_error_log *err;arch/powerpc/platforms/pseries/setup.c +1
-0@@ -1084,6 +1084,7 @@ define_machine(pseries) { .system_reset_exception = pSeries_system_reset_exception, .machine_check_early = pseries_machine_check_realmode, .machine_check_exception = pSeries_machine_check_exception, + .machine_check_log_err = pSeries_machine_check_log_err, #ifdef CONFIG_KEXEC_CORE .machine_kexec = pSeries_machine_kexec, .kexec_cpu_down = pseries_kexec_cpu_down,Signature · 1 line
2.31.1
Quoted 99 lines · Nicholas Piggin
Excerpts from Ganesh Goudar's message of November 24, 2021 7:54 pm:
Quoted 98 lines
In realmode mce handler we use
irq_work_queue()to defer the processing of mce events,irq_work_queue()can only be called when translation is enabled because it touches memory outside RMA, hence we enable translation before callingirq_work_queueand disable on return, though it is not safe to do in realmode.To avoid this, program the decrementer and call the event processing functions from timer handler.
Signed-off-by Ganesh Goudar<[email protected]>---
V2:
* Usearch_irq_work_raiseto raise decrementer interrupt.
* Avoid having atomic variable.V3:
* Fix build error.
Reported by kernel test bot.--- arch/powerpc/include/asm/machdep.h | 2 + arch/powerpc/include/asm/mce.h | 2 + arch/powerpc/include/asm/paca.h | 1 + arch/powerpc/kernel/mce.c | 51 +++++++++++------------- arch/powerpc/kernel/time.c | 3 ++ arch/powerpc/platforms/pseries/pseries.h | 1 + arch/powerpc/platforms/pseries/ras.c | 31 +------------- arch/powerpc/platforms/pseries/setup.c | 1 + 8 files changed, 34 insertions(+), 58 deletions(-)arch/powerpc/include/asm/machdep.h +2
-0@@ -99,6 +99,8 @@ struct machdep_calls {/* Called during machine check exception to retrive fixup address. */ bool (*mce_check_early_recovery)(struct pt_regs *regs); +void (*machine_check_log_err)(void ); +/* Motherboard/chipset features. This is a kind of general purpose * hook used to control some machine specific features (like reset * lines, chip power control, etc...). arch/powerpc/include/asm/mce.h +2
-0@@ -235,8 +235,10 @@ extern void machine_check_print_event_info(struct machine_check_event *evt,unsigned long addr_to_pfn (struct pt_regs *regs,unsigned long addr);extern void mce_common_process_ue (struct pt_regs *regs,struct mce_error_info *mce_err); +void machine_check_raise_dec_intr (void );int mce_register_notifier (struct notifier_block *nb);int mce_unregister_notifier (struct notifier_block *nb); +void mce_run_late_handlers (void );#ifdef CONFIG_PPC_BOOK3S_64 void flush_and_reload_slb (void );void flush_erat (void );arch/powerpc/include/asm/paca.h +1
-0@@ -280,6 +280,7 @@ struct paca_struct {#endif #ifdef CONFIG_PPC_BOOK3S_64 struct mce_info *mce_info; +u32 mces_to_process;#endif /* CONFIG_PPC_BOOK3S_64 */ }____cacheline_aligned ;arch/powerpc/kernel/mce.c +0
-10@@ -28,19 +28,9 @@#include "setup.h" -static void machine_check_process_queued_event (struct irq_work *work); -static void machine_check_ue_irq_work (struct irq_work *work);static void machine_check_ue_event (struct machine_check_event *evt);static void machine_process_ue_event (struct work_struct *work); -static struct irq_work mce_event_process_work = { - .func = machine_check_process_queued_event, -}; - -static struct irq_work mce_ue_event_irq_work = { - .func = machine_check_ue_irq_work, -}; - static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);static BLOCKING_NOTIFIER_HEAD (mce_notifier_list);@@ -89,6 +79,12 @@ static void mce_set_error_info(struct machine_check_event *mce, } }
Quoted 8 lines
arch/powerpc/kernel/mce.c static void mce_set_error_info(struct machine_check_event *mce, · line 86
Quoted 5 lines
arch/powerpc/kernel/mce.c +5
-0+/* Raise decrementer interrupt */ +void machine_check_raise_dec_intr (void ) +{ +arch_irq_work_raise (); +}It would be better if the name specifically related to irq work, which is more than just dec interrupt. It might be good to set
mces_to_processhere as well.
Sure
Quoted 3 lines
I would name it something like
mce_irq_work_queue, and the paca variable tomce_pending_irq_work...
Ok
Quoted 16 lines
arch/powerpc/kernel/mce.c static void machine_check_process_queued_event(struct irq_work *work) · line 359
Quoted 12 lines
arch/powerpc/kernel/mce.c +10
-0+void mce_run_late_handlers (void ) +{ +if (unlikely (local_paca->mces_to_process)) { +if (ppc_md.machine_check_log_err) + ppc_md.machine_check_log_err (); +machine_check_process_queued_event (); +machine_check_ue_work (); + local_paca->mces_to_process--; + } +}The problem with a counter is that you're clearing the irq work pending in the timer interrupt, so you'll never call in here again to clear that (until something else sets irq work).
Quoted 3 lines
But as far as I can see it does not need to be a counter, just a flag. The machine check calls will process multiple events, right? (and the current
irq_workqueue does not queue the same work multiple times).
You are right, It can just be a flag.
Quoted 117 lines
Oh. That's actually bad, isn't it? Our irq work should be per-CPU because the callbacks are mainly only operating on the local paca queued events, so we have a longstanding bug there AFAIKS. Your patch will solve it if everything is converted over.
Quoted 9 lines
arch/powerpc/kernel/mce.c +1
-0+void machine_check_print_event_info (struct machine_check_event *evt,bool user_mode,bool in_guest) {arch/powerpc/kernel/time.c +0
-0arch/powerpc/kernel/time.c DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) · line 599
Quoted 6 lines
arch/powerpc/kernel/time.c +3
-0@@ -594,6 +594,9 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)if (test_irq_work_pending ()) {clear_irq_work_pending (); +#ifdef CONFIG_PPC_BOOK3S_64 +mce_run_late_handlers (); +#endif Maybe create a no-op inline function for others and call unconditionally here. I wonder if the name could be better, we have lots of handlers, of varying earliness. real-mode, then virt mode NMI context, then IRQ context, then workqueue context.
mce_run_irq_context_handlers()might not be much better though.Quoted 89 lines
arch/powerpc/kernel/time.c +0
-0irq_work_run (); }arch/powerpc/platforms/pseries/pseries.h +1
-0@@ -21,6 +21,7 @@ struct pt_regs;extern int pSeries_system_reset_exception (struct pt_regs *regs);extern int pSeries_machine_check_exception (struct pt_regs *regs);extern long pseries_machine_check_realmode (struct pt_regs *regs); +void pSeries_machine_check_log_err (void );#ifdef CONFIG_SMP extern void smp_init_pseries (void );arch/powerpc/platforms/pseries/ras.c +1
-30@@ -23,11 +23,6 @@ static DEFINE_SPINLOCK(ras_log_buf_lock);static int ras_check_exception_token; -static void mce_process_errlog_event (struct irq_work *work); -static struct irq_work mce_errlog_process_work = { - .func = mce_process_errlog_event, -}; -#define EPOW_SENSOR_TOKEN 9#define EPOW_SENSOR_INDEX 0@@ -729,40 +724,16 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) error_type = mce_log->error_type; disposition =mce_handle_err_realmode (disposition, error_type); - -/* -* Enable translation as we will be accessing per-cpu variables -* in save_mce_event() which may fall outside RMO region, also -* leave it enabled because subsequently we will be queuing work -* to workqueues where again per-cpu variables accessed, besides -* fwnmi_release_errinfo() crashes when called in realmode on -* pseries. -* Note: All the realmode handling like flushing SLB entries for -* SLB multihit is done by now. -*/ out: - msr =mfmsr (); -mtmsr (msr |MSR_IR |MSR_DR ); - disposition =mce_handle_err_virtmode (regs, errp, mce_log, disposition); - -/* -* Queue irq work to log this rtas event later. -* irq_work_queue uses per-cpu variables, so do this in virt -* mode as well. -*/ -irq_work_queue (&mce_errlog_process_work); - -mtmsr (msr); -return disposition; }/* * Process MCE rtas errlog event. */ -static void mce_process_errlog_event (struct irq_work *work) +void pSeries_machine_check_log_err (void ) {struct rtas_error_log *err;arch/powerpc/platforms/pseries/setup.c +1
-0@@ -1084,6 +1084,7 @@ define_machine(pseries) { .system_reset_exception = pSeries_system_reset_exception, .machine_check_early = pseries_machine_check_realmode, .machine_check_exception = pSeries_machine_check_exception, + .machine_check_log_err = pSeries_machine_check_log_err, #ifdef CONFIG_KEXEC_CORE .machine_kexec = pSeries_machine_kexec, .kexec_cpu_down = pseries_kexec_cpu_down,Signature · 1 line
2.31.1