powerpc/64: fix irq replay missing preempt
Nicholas Piggin
reviewerPATCH 1/6 powerpc/64: fix irq replay missing preempt
Prior to commit 3282a3da25bd ("powerpc/64: Implement soft interrupt replay in C"), replayed interrupts returned by the regular interrupt exit code, which performs preemption in case an interrupt had set need_resched.
This logic was missed by the conversion. Adding preempt_disable/enable around the interrupt replay and final irq enable will reschedule if needed.
--- arch/powerpc/kernel/irq.c | 7 +++++++ 1 file changed, 7 insertions(+)
arch/powerpc/kernel/irq.c +7 -0
@@ -368,6 +368,12 @@ notrace void arch_local_irq_restore(unsigned long mask) } } +/* +* Disable preempt here, so that the below preempt_enable will +* perform resched if required (a replayed interrupt may set +* need_resched). +*/ +preempt_disable ();irq_soft_mask_set (IRQS_ALL_DISABLED );trace_hardirqs_off ();
@@ -377,6 +383,7 @@ notrace void arch_local_irq_restore(unsigned long mask)trace_hardirqs_on ();irq_soft_mask_set (IRQS_ENABLED );__hard_irq_enable (); +preempt_enable (); }EXPORT_SYMBOL (arch_local_irq_restore);
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 2/6 powerpc/64: fix irq replay pt_regs->softe value
Replayed interrupts get an "artificial" struct pt_regs constructed to pass to interrupt handler functions. This did not get the softe field set correctly, it's as though the interrupt has hit while irqs are disabled. It should be IRQS_ENABLED.
This is possibly harmless, asynchronous handlers should not be testing if irqs were disabled, but it might be possible for example some code is shared with synchronous or NMI handlers, and it makes more sense if debug output looks at this.
--- arch/powerpc/kernel/irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/powerpc/kernel/irq.c +1 -1
@@ -214,7 +214,7 @@ void replay_soft_interrupts(void)struct pt_regs regs;ppc_save_regs (®s); - regs.softe =IRQS_ALL_DISABLED ; + regs.softe =IRQS_ENABLED ; again:if (IS_ENABLED (CONFIG_PPC_IRQ_SOFT_MASK_DEBUG ))
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 3/6 powerpc/64e: remove PACA_IRQ_EE_EDGE
In reply to Nicholas PigginThis is not used anywhere.
--- arch/powerpc/include/asm/hw_irq.h | 5 ++--- arch/powerpc/kernel/exceptions-64e.S | 1 - arch/powerpc/kernel/irq.c | 23 ----------------------- 3 files changed, 2 insertions(+), 27 deletions(-)
arch/powerpc/include/asm/hw_irq.h +2 -3
@@ -25,9 +25,8 @@#define PACA_IRQ_DBELL 0x02#define PACA_IRQ_EE 0x04#define PACA_IRQ_DEC 0x08/* Or FIT */ -#define PACA_IRQ_EE_EDGE 0x10/* BookE only */ -#define PACA_IRQ_HMI 0x20 -#define PACA_IRQ_PMI 0x40 +#define PACA_IRQ_HMI 0x10 +#define PACA_IRQ_PMI 0x20/* * Some soft-masked interrupts must be hard masked until they are replayed
arch/powerpc/kernel/exceptions-64e.S +0 -1
@@ -988,7 +988,6 @@ kernel_dbg_exc: .endm masked_interrupt_book3e_0x500: - // XXX When adding support for EPR, use PACA_IRQ_EE_EDGE masked_interrupt_book3e PACA_IRQ_EE 1 masked_interrupt_book3e_0x900:
arch/powerpc/kernel/irq.c +0 -23
@@ -181,16 +181,6 @@ notrace unsigned int __check_irq_replay(void)return 0x500 ; } -/* -* Check if an EPR external interrupt happened this bit is typically -* set if we need to handle another "edge" interrupt from within the -* MPIC "EPR" handler. -*/ -if (happened &PACA_IRQ_EE_EDGE ) { - local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE ; -return 0x500 ; - } -if (happened &PACA_IRQ_DBELL ) { local_paca->irq_happened &= ~PACA_IRQ_DBELL ;return 0x280 ;
@@ -270,19 +260,6 @@ void replay_soft_interrupts(void)hard_irq_disable (); } -/* -* Check if an EPR external interrupt happened this bit is typically -* set if we need to handle another "edge" interrupt from within the -* MPIC "EPR" handler. -*/ -if (IS_ENABLED (CONFIG_PPC_BOOK3E ) && (happened &PACA_IRQ_EE_EDGE )) { - local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE ; - regs.trap =0x500 ; -do_IRQ (®s); -if (!(local_paca->irq_happened &PACA_IRQ_HARD_DIS )) -hard_irq_disable (); - } -if (IS_ENABLED (CONFIG_PPC_DOORBELL ) && (happened &PACA_IRQ_DBELL )) { local_paca->irq_happened &= ~PACA_IRQ_DBELL ;if (IS_ENABLED (CONFIG_PPC_BOOK3E ))
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 4/6 powerpc/64e: remove 64s specific interrupt soft-mask code
In reply to Nicholas PigginSince the assembly soft-masking code was moved to 64e specific, there are some 64s specific interrupt types still there. Remove them.
--- arch/powerpc/kernel/exceptions-64e.S | 10 ---------- arch/powerpc/kernel/irq.c | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-)
arch/powerpc/kernel/exceptions-64e.S +0 -10
@@ -1302,16 +1302,6 @@ fast_exception_return: addi r3,r1,STACK_FRAME_OVERHEAD; bl do_IRQ b ret_from_except -1: cmpwi cr0,r3,0xf00 - bne 1f - addi r3,r1,STACK_FRAME_OVERHEAD; - bl performance_monitor_exception - b ret_from_except -1: cmpwi cr0,r3,0xe60 - bne 1f - addi r3,r1,STACK_FRAME_OVERHEAD; - bl handle_hmi_exception - b ret_from_except 1: cmpwi cr0,r3,0x900 bne 1f addi r3,r1,STACK_FRAME_OVERHEAD;
arch/powerpc/kernel/irq.c +1 -1
@@ -113,7 +113,7 @@ static inline notrace int decrementer_check_overflow(void)#ifdef CONFIG_PPC_BOOK3E /* This is called whenever we are re-enabling interrupts -* and returns either 0 (nothing to do) or 500/900/280/a00/e80 if +* and returns either 0 (nothing to do) or 500/900/280 if * there's an EE, DEC or DBELL to generate. * * This is called in two contexts: From arch_local_irq_restore()
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 5/6 powerpc/64: make restore_interrupts 64e only
In reply to Nicholas PigginThis is not used by 64s.
--- arch/powerpc/kernel/irq.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-)
arch/powerpc/kernel/irq.c +19 -18
@@ -191,6 +191,25 @@ notrace unsigned int __check_irq_replay(void) return 0; } + +/* +* This is specifically called by assembly code to re-enable interrupts +* if they are currently disabled. This is typically called before +* schedule() or do_signal() when returning to userspace. We do it +* in C to avoid the burden of dealing with lockdep etc... +* +* NOTE: This is called with interrupts hard disabled but not marked +* as such in paca->irq_happened, so we need to resync this. +*/ +void notrace restore_interrupts(void) +{ +if (irqs_disabled ()) { + local_paca->irq_happened |=PACA_IRQ_HARD_DIS ; +local_irq_enable (); + }else +__hard_irq_enable (); +} +#endif /* CONFIG_PPC_BOOK3E */ void replay_soft_interrupts (void )
@@ -364,24 +383,6 @@ notrace void arch_local_irq_restore(unsigned long mask) }EXPORT_SYMBOL (arch_local_irq_restore); -/* -* This is specifically called by assembly code to re-enable interrupts -* if they are currently disabled. This is typically called before -* schedule() or do_signal() when returning to userspace. We do it -* in C to avoid the burden of dealing with lockdep etc... -* -* NOTE: This is called with interrupts hard disabled but not marked -* as such in paca->irq_happened, so we need to resync this. -*/ -void notrace restore_interrupts(void) -{ -if (irqs_disabled ()) { - local_paca->irq_happened |=PACA_IRQ_HARD_DIS ; -local_irq_enable (); - }else -__hard_irq_enable (); -} -/* * This is a helper to use when about to go into idle low-power * when the latter has the side effect of re-enabling interrupts
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 6/6 powerpc/64: irq replay remove decrementer overflow check
In reply to Nicholas PigginThis is an ad-hoc way to catch some cases of decrementer overflow. It won't catch cases where interrupts were hard disabled before any soft masked interrupts fired, for example. And it doesn't catch cases that have overflowed an even number of times.
It's not clear what exactly what problem s being solved here. A lost timer when we have an IRQ off latency of more than ~4.3 seconds could be avoided (so long as it's also less than ~8.6s) but this is already a hard lockup order of magnitude event, and the decrementer will wrap again and provide a timer interrupt within the same latency magnitdue.
So the test catches some cases of lost decrementers in very exceptional (buggy) latency event cases, reducing timer interrupt latency in that case by up to 4.3 seconds. And for large decrementer, it's useless. It is performed in potentially quite a hot path, reading the TB can be a noticable overhead.
Perhaps more importantly it allows the clunky MSR[EE] vs PACA_IRQ_HARD_DIS incoherency to be removed.
--- arch/powerpc/kernel/irq.c | 50 +-------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-)
arch/powerpc/kernel/irq.c +1 -49
@@ -102,14 +102,6 @@ static inline notrace unsigned long get_irq_happened(void)return happened; } -static inline notrace int decrementer_check_overflow(void) -{ -u64 now =get_tb_or_rtc (); -u64 *next_tb =this_cpu_ptr (&decrementers_next_tb); - -return now >= *next_tb; -} -#ifdef CONFIG_PPC_BOOK3E /* This is called whenever we are re-enabling interrupts
@@ -142,35 +134,6 @@ notrace unsigned int __check_irq_replay(void)trace_hardirqs_on ();trace_hardirqs_off (); -/* -* We are always hard disabled here, but PACA_IRQ_HARD_DIS may -* not be set, which means interrupts have only just been hard -* disabled as part of the local_irq_restore or interrupt return -* code. In that case, skip the decrementr check becaus it's -* expensive to read the TB. -* -* HARD_DIS then gets cleared here, but it's reconciled later. -* Either local_irq_disable will replay the interrupt and that -* will reconcile state like other hard interrupts. Or interrupt -* retur will replay the interrupt and in that case it sets -* PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S). -*/ -if (happened &PACA_IRQ_HARD_DIS ) { - local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS ; - -/* -* We may have missed a decrementer interrupt if hard disabled. -* Check the decrementer register in case we had a rollover -* while hard disabled. -*/ -if (!(happened &PACA_IRQ_DEC )) { -if (decrementer_check_overflow ()) { - local_paca->irq_happened |=PACA_IRQ_DEC ; - happened |=PACA_IRQ_DEC ; - } - } - } -if (happened &PACA_IRQ_DEC ) { local_paca->irq_happened &= ~PACA_IRQ_DEC ;return 0x900 ;
@@ -229,18 +192,6 @@ void replay_soft_interrupts(void)if (IS_ENABLED (CONFIG_PPC_IRQ_SOFT_MASK_DEBUG ))WARN_ON_ONCE (mfmsr () &MSR_EE ); -if (happened &PACA_IRQ_HARD_DIS ) { -/* -* We may have missed a decrementer interrupt if hard disabled. -* Check the decrementer register in case we had a rollover -* while hard disabled. -*/ -if (!(happened &PACA_IRQ_DEC )) { -if (decrementer_check_overflow ()) - happened |=PACA_IRQ_DEC ; - } - } -/* * Force the delivery of pending soft-disabled interrupts on PS3. * Any HV call will have this side effect.
@@ -345,6 +296,7 @@ notrace void arch_local_irq_restore(unsigned long mask)if (IS_ENABLED (CONFIG_PPC_IRQ_SOFT_MASK_DEBUG ))WARN_ON_ONCE (!(mfmsr () &MSR_EE ));__hard_irq_disable (); + local_paca->irq_happened |=PACA_IRQ_HARD_DIS ; }else {/* * We should already be hard disabled here. We had bugs
Signature · 1 line
2.23.0
Michael Ellerman
reviewerRe: [PATCH 6/6] powerpc/64: irq replay remove decrementer overflow check
Quoted 17 lines · Nicholas Piggin
This is an ad-hoc way to catch some cases of decrementer overflow. It won't catch cases where interrupts were hard disabled before any soft masked interrupts fired, for example. And it doesn't catch cases that have overflowed an even number of times.
It's not clear what exactly what problem s being solved here. A lost timer when we have an IRQ off latency of more than ~4.3 seconds could be avoided (so long as it's also less than ~8.6s) but this is already a hard lockup order of magnitude event, and the decrementer will wrap again and provide a timer interrupt within the same latency magnitdue.
So the test catches some cases of lost decrementers in very exceptional (buggy) latency event cases, reducing timer interrupt latency in that case by up to 4.3 seconds. And for large decrementer, it's useless. It is performed in potentially quite a hot path, reading the TB can be a noticable overhead.
Quoted 7 lines
Perhaps more importantly it allows the clunky MSR[EE] vs
PACA_IRQ_HARD_DISincoherency to be removed.Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/kernel/irq.c | 50 +-------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-)
Seems to be unhappy on qemu ppc64e:
kernel BUG at arch/powerpc/kernel/irq.c:153!
Which is:
notrace unsigned int __check_irq_replay(void)
{
...
/* There should be nothing left ! */ BUG_ON(local_paca->irq_happened != 0); return 0; }
Full log below.
cheers
spawn qemu-system-ppc64 -nographic -M ppce500 -cpu e5500 -m 2G -kernel /home/michael/build/adhoc/ci_output/build/corenet64_smp_defconfig@ppc64@[email protected]/uImage -initrd ppc64-novsx-rootfs.cpio.gz -append noreboot
MMU: Supported page sizes
4 KB as direct
4096 KB as direct
16384 KB as direct
65536 KB as direct
262144 KB as direct 1048576 KB as direct
MMU: Book3E HW tablewalk not supported
Linux version 5.9.0-rc2-00187-gf523995cc1ee (linuxppc@e054daee57c9) (powerpc64-linux-gnu-gcc (GCC) 10.1.0, GNU ld (GNU Binutils) 2.34) #1 SMP Fri Sep 18 11:52:25 Australia 2020 Found initrd at 0xc000000005000000:0xc0000000051e9a47
Using QEMU e500 machine descriptionioremap() called early from .find_legacy_serial_ports+0x6cc/0x7bc. Use early_ioremap() instead
printk: bootconsole [udbg0] enabled
CPU maps initialized for 1 thread per core
-----------------------------------------------------
phys_mem_size = 0x80000000 dcache_bsize = 0x40 icache_bsize = 0x40 cpu_features = 0x00000003008001b4 possible = 0x00000003009003b6 always = 0x00000003008003b4
cpu_user_features = 0xcc008000 0x08000000mmu_features = 0x000a0010firmware_features = 0x0000000000000000
-----------------------------------------------------qemu_e500_setup_arch()
barrier-nospec: using isync; sync as speculation barrier Zone ranges:
DMA [mem 0x0000000000000000-0x000000007fffffff]
Normal empty
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x0000000000000000-0x000000007fffffff]
Initmem setup node 0 [mem 0x0000000000000000-0x000000007fffffff]
MMU: Allocated 2112 bytes of context maps for 255 contexts
percpu: Embedded 28 pages/cpu s77400 r0 d37288 u1048576 Built 1 zonelists, mobility grouping on. Total pages: 517120
Kernel command line: noreboot Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear) Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear) mem auto-init: stack:off, heap alloc:off, heap free:off Memory: 1977432K/2097152K available (12048K kernel code, 2204K rwdata, 3788K rodata, 460K init, 321K bss, 119720K reserved, 0K cma-reserved) SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1 rcu: Hierarchical RCU implementation. rcu: RCU event tracing is enabled. rcu: RCU restricting CPUs from NR_CPUS=24 to nr_cpu_ids=1. rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies. rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1 NR_IRQS: 512, nr_irqs: 512, preallocated irqs: 16 mpic: Setting up MPIC " OpenPIC " version 1.2 at fe0040000, max 1 CPUs mpic: ISU size: 256, shift: 8, mask: ff mpic: Initializing for 256 sources random: get_random_u64 called from .start_kernel+0x498/0x70c with crng_init=0 clocksource: timebase: mask: 0xffffffffffffffff max_cycles: 0x5c4093a7d1, max_idle_ns: 440795210635 ns clocksource: timebase mult[2800000] shift[24] registered Console: colour dummy device 80x25 pid_max: default: 32768 minimum: 301 Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)
Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)
e500 family performance monitor hardware support registered
rcu: Hierarchical SRCU implementation.
smp: Bringing up secondary CPUs ...
smp: Brought up 1 node, 1 CPU
devtmpfs: initialized
clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns futex hash table entries: 256 (order: 2, 16384 bytes, linear)
------------[ cut here ]------------ kernel BUG at arch/powerpc/kernel/irq.c:153! Oops: Exception in kernel mode, sig: 5 [#1] BE PAGE_SIZE=4K SMP NR_CPUS=24 QEMU e500 Modules linked in: CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.9.0-rc2-00187-gf523995cc1ee #1 NIP: c0000000000039d4 LR: c00000000001a634 CTR: c000000000594d00 REGS: c00000007d0df5e0 TRAP: 0700 Not tainted (5.9.0-rc2-00187-gf523995cc1ee) MSR: 0000000080021000 <CE,ME> CR: 28000242 XER: 20000000
IRQMASK: 1
GPR00: 0000000000000001 c00000007d0df870 c0000000011e2200 0000000000000000
GPR04: 0000000000000800 0000000000000000 0000000000000001 c00000000116eec0
GPR08: 0000000000000000 0000000000000001 0000000000000000 0000000000000300
GPR12: 0000000028000242 c000000001263000 c000000000002434 0000000000000000
GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR24: c00000000110b248 c000000000fd3728 c000000000fc1ef8 c000000000f715f8
GPR28: c0000000010c8700 0000000000000000 0000000000000002 0000000000000001 NIP [c0000000000039d4] .__check_irq_replay+0x24/0x60
LR [c00000000001a634] fast_exception_return+0xe0/0x140
Call Trace:
[c00000007d0df870] [c00000000001a37c] storage_fault_common+0x40/0x44 (unreliable)
--- interrupt: 300 at .__se_sys_futex_time32+0x1fc/0x2c8
LR = .futex_init+0xbc/0x144
[c00000007d0dfb70] [c000000000f8af88] .futex_init+0x88/0x144 (unreliable)
[c00000007d0dfc10] [c000000000001ed8] .do_one_initcall+0x6c/0x28c
[c00000007d0dfcf0] [c000000000f79258] .kernel_init_freeable+0x298/0x314
[c00000007d0dfda0] [c000000000002450] .kernel_init+0x1c/0x138
[c00000007d0dfe20] [c00000000000082c] .ret_from_kernel_thread+0x58/0x60
Instruction dump:
4bffff64 48bb8a49 60000000 892d067b 71280008 40820028 71280004 40820040
71280002 40820028 3149ffff 7d2a4910 <0b090000> 38600000 4e800020 55290776
---[ end trace 276324e35afb2c5e ]---
Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000005Rebooting in 180 seconds..
System Halted, OK to tu
Quoted 8 lines · Nicholas Piggin
Prior to commit
3282a3da25bd("powerpc/64: Implement soft interrupt replay in C"), replayed interrupts returned by the regular interrupt exit code, which performs preemption in case an interrupt had setneed_resched.This logic was missed by the conversion. Adding preempt_disable/enable around the interrupt replay and final irq enable will reschedule if needed.
Patches 1-5 applied to powerpc/next.
[1/6] powerpc/64: fix irq replay missing preempt
https://git.kernel.org/powerpc/c/903fd31d3212ab72d564c68f6cfb5d04db68773e [2/6] powerpc/64: fix irq replay pt_regs->softe value https://git.kernel.org/powerpc/c/2b48e96be2f9f7151197fd25dc41487054bc6f5b [3/6] powerpc/64e: remove PACA_IRQ_EE_EDGE https://git.kernel.org/powerpc/c/012a9a97a8fd6c96d5ec64eb0583220490d95e73 [4/6] powerpc/64e: remove 64s specific interrupt soft-mask code https://git.kernel.org/powerpc/c/903dd1ff453e458fc7608ee4df42a6df16d3d1a0 [5/6] powerpc/64: make restore_interrupts 64e only https://git.kernel.org/powerpc/c/455575533c7aa294d3c0284d59a77ae9a60c0537
cheers