powerpc: interrupt wrappers
Subsystems (3)
Nicholas Piggin
reviewerv3 cover powerpc: interrupt wrappers
This adds interrupt handler wrapper functions, similar to the generic / x86 code, and moves several common operations into them from either asm or open coded in the individual handlers.
Since v1:
- Fixed a couple of compile issues
- Fixed perf weirdness (sometimes NMI, sometimes not)
- Also move irq_enter/exit into wrappers
Since v2:
- Rebased upstream
- Took code in patch 3 from Christophe
- Fixed some compile errors from 0day
Thanks,
Nick
Nicholas Piggin (19):
powerpc/64s: move the last of the page fault handling logic to C
powerpc: remove arguments from fault handler functions
powerpc: bad_page_fault, do_break get registers from regs powerpc/perf: move perf irq/nmi handling details into traps.c
powerpc: interrupt handler wrapper functions
powerpc: add interrupt wrapper entry / exit stub functions
powerpc: add interrupt_cond_local_irq_enable helper powerpc/64: context tracking remove _TIF_NOHZ powerpc/64: context tracking move to interrupt wrappers powerpc/64: add context tracking to asynchronous interrupts
powerpc: handle irq_enter/irq_exit in interrupt handler wrappers powerpc/64s: move context tracking exit to interrupt exit path powerpc/64s: reconcile interrupts in C
powerpc/64: move account_stolen_time into its own function powerpc/64: entry cpu time accounting in C
powerpc: move NMI entry/exit code into wrapper powerpc/64s: move NMI soft-mask handling to C powerpc/64s: runlatch interrupt handling in C powerpc/64s: power4 nap fixup in C
arch/Kconfig | 6 - arch/powerpc/Kconfig | 1 - arch/powerpc/include/asm/asm-prototypes.h | 29 -- arch/powerpc/include/asm/bug.h | 5 +- arch/powerpc/include/asm/cputime.h | 15 + arch/powerpc/include/asm/debug.h | 3 +- arch/powerpc/include/asm/hw_irq.h | 9 - arch/powerpc/include/asm/interrupt.h | 405 +++++++++++++++++++++ arch/powerpc/include/asm/ppc_asm.h | 24 -- arch/powerpc/include/asm/processor.h | 1 + arch/powerpc/include/asm/thread_info.h | 10 +- arch/powerpc/include/asm/time.h | 2 + arch/powerpc/kernel/dbell.c | 15 +- arch/powerpc/kernel/entry_32.S | 18 +- arch/powerpc/kernel/exceptions-64e.S | 6 +- arch/powerpc/kernel/exceptions-64s.S | 307 ++-------------- arch/powerpc/kernel/head_40x.S | 10 +- arch/powerpc/kernel/head_8xx.S | 11 +- arch/powerpc/kernel/head_book3s_32.S | 15 +- arch/powerpc/kernel/head_booke.h | 4 +- arch/powerpc/kernel/idle_book3s.S | 4 + arch/powerpc/kernel/irq.c | 7 +- arch/powerpc/kernel/mce.c | 16 +- arch/powerpc/kernel/process.c | 7 +- arch/powerpc/kernel/ptrace/ptrace.c | 4 - arch/powerpc/kernel/signal.c | 4 - arch/powerpc/kernel/syscall_64.c | 30 +- arch/powerpc/kernel/tau_6xx.c | 5 +- arch/powerpc/kernel/time.c | 7 +- arch/powerpc/kernel/traps.c | 231 ++++++------ arch/powerpc/kernel/watchdog.c | 15 +- arch/powerpc/kvm/book3s_hv.c | 1 + arch/powerpc/kvm/book3s_hv_builtin.c | 1 + arch/powerpc/kvm/booke.c | 1 + arch/powerpc/mm/book3s64/hash_utils.c | 82 +++-- arch/powerpc/mm/book3s64/slb.c | 12 +- arch/powerpc/mm/fault.c | 78 +++- arch/powerpc/perf/core-book3s.c | 35 +- arch/powerpc/perf/core-fsl-emb.c | 25 -- arch/powerpc/platforms/8xx/machine_check.c | 2 +- arch/powerpc/platforms/powernv/idle.c | 1 + 41 files changed, 772 insertions(+), 692 deletions(-) create mode 100644 arch/powerpc/include/asm/interrupt.h
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 1/19 powerpc/64s: move the last of the page fault handling logic to C
The page fault handling still has some complex logic particularly around hash table handling, in asm. Implement this in C instead.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 + arch/powerpc/include/asm/bug.h | 1 + arch/powerpc/kernel/exceptions-64s.S | 131 +++--------------- arch/powerpc/mm/book3s64/hash_utils.c | 77 ++++++---- arch/powerpc/mm/fault.c | 57 +++++++- 5 files changed, 127 insertions(+), 140 deletions(-)
arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -0
@@ -453,6 +453,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_LOCAL_UPDATE 0x1#define HPTE_NOHPTE_UPDATE 0x2 +int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);
arch/powerpc/include/asm/bug.h +1 -0
@@ -112,6 +112,7 @@struct pt_regs ;extern int do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); +int hash__do_page_fault (struct pt_regs *,unsigned long ,unsigned long );extern void bad_page_fault (struct pt_regs *,unsigned long ,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );
arch/powerpc/kernel/exceptions-64s.S +22 -109
161 lines not on this page · raw message
arch/powerpc/mm/book3s64/hash_utils.c +49 -28
105 lines not on this page · raw message
arch/powerpc/mm/fault.c +54 -3
79 lines not on this page · raw message
Signature · 1 line
2.23.0
Re: [PATCH v3 01/19] powerpc/64s: move the last of the page fault handling logic to C
Quoted 3 lines · Nicholas Piggin
The page fault handling still has some complex logic particularly around hash table handling, in asm. Implement this in C instead.
Reviewed-by Aneesh Kumar K.V <[email protected]>
Quoted 402 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 + arch/powerpc/include/asm/bug.h | 1 + arch/powerpc/kernel/exceptions-64s.S | 131 +++--------------- arch/powerpc/mm/book3s64/hash_utils.c | 77 ++++++---- arch/powerpc/mm/fault.c | 57 +++++++- 5 files changed, 127 insertions(+), 140 deletions(-)arch/powerpc/include/asm/book3s/64/mmu-hash.h +1
-0@@ -453,6 +453,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_LOCAL_UPDATE 0x1#define HPTE_NOHPTE_UPDATE 0x2 +int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);arch/powerpc/include/asm/bug.h +1
-0@@ -112,6 +112,7 @@struct pt_regs ;extern int do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); +int hash__do_page_fault (struct pt_regs *,unsigned long ,unsigned long );extern void bad_page_fault (struct pt_regs *,unsigned long ,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );arch/powerpc/kernel/exceptions-64s.S +22
-109@@ -1401,14 +1401,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) * * Handling: * - Hash MMU - * Go to do_hash_page first to see if the HPT can be filled from an entry in - * the Linux page table. Hash faults can hit in kernel mode in a fairly + * Go to do_hash_fault, which attempts to fill the HPT from an entry in the + * Linux page table. Hash faults can hit in kernel mode in a fairly * arbitrary state (e.g., interrupts disabled, locks held) when accessing * "non-bolted" regions, e.g., vmalloc space. However these should always be - * backed by Linux page tables. + * backed by Linux page table entries. * - * If none is found, do a Linux page fault. Linux page faults can happen in - * kernel mode due to user copy operations of course. + * If no entry is found the Linux page fault handler is invoked (by + * do_hash_fault). Linux page faults can happen in kernel mode due to user + * copy operations of course. * * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest * MMU context, which may cause a DSI in the host, which must go to the@@ -1439,13 +1440,17 @@ EXC_COMMON_BEGIN(data_access_common) GEN_COMMON data_access ld r4,_DAR(r1) ld r5,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x300 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + cmpdi r3,0 + beq+ interrupt_return + /* We need to restore NVGPRS */ + REST_NVGPRS(r1) + b interrupt_return GEN_KVM data_access@@ -1540,13 +1545,17 @@ EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access ld r4,_DAR(r1) ld r5,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x400 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + cmpdi r3,0 + beq+ interrupt_return + /* We need to restore NVGPRS */ + REST_NVGPRS(r1) + b interrupt_return GEN_KVM instruction_access@@ -3202,99 +3211,3 @@ disable_machine_check: RFI_TO_KERNEL 1: mtlr r0 blr - -/* - * Hash table stuff - */ - .balign IFETCH_ALIGN_BYTES -do_hash_page: -#ifdef CONFIG_PPC_BOOK3S_64 - lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h - ori r0,r0,DSISR_BAD_FAULT_64S@l - and. r0,r5,r0 /* weird error? */ - bne- handle_page_fault /* if not, try to insert a HPTE */ - - /* - * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then - * don't call hash_page, just fail the fault. This is required to - * prevent re-entrancy problems in the hash code, namely perf - * interrupts hitting while something holds H_PAGE_BUSY, and taking a - * hash fault. See the comment in hash_preload(). - */ - ld r11, PACA_THREAD_INFO(r13) - lwz r0,TI_PREEMPT(r11) - andis. r0,r0,NMI_MASK@h - bne 77f - - /* - * r3 contains the trap number - * r4 contains the faulting address - * r5 contains dsisr - * r6 msr - * - * at return r3 = 0 for success, 1 for page fault, negative for error - */ - bl __hash_page /* build HPTE if possible */ - cmpdi r3,0 /* see if __hash_page succeeded */ - - /* Success */ - beq interrupt_return /* Return from exception on success */ - - /* Error */ - blt- 13f - - /* Reload DAR/DSISR into r4/r5 for the DABR check below */ - ld r4,_DAR(r1) - ld r5,_DSISR(r1) -#endif /* CONFIG_PPC_BOOK3S_64 */ - -/* Here we have a page fault that hash_page can't handle. */ -handle_page_fault: -11: andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_page_fault - cmpdi r3,0 - beq+ interrupt_return - mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl bad_page_fault - b interrupt_return - -/* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - ld r4,_DAR(r1) - ld r5,_DSISR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_break - /* - * do_break() may have changed the NV GPRS while handling a breakpoint. - * If so, we need to restore them with their updated values. - */ - REST_NVGPRS(r1) - b interrupt_return - - -#ifdef CONFIG_PPC_BOOK3S_64 -/* We have a page fault that hash_page could handle but HV refused - * the PTE insertion - */ -13: mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl low_hash_fault - b interrupt_return -#endif - -/* - * We come here as a result of a DSI at a point where we don't want - * to call hash_page, such as when we are accessing memory (possibly - * user memory) inside a PMU interrupt that occurred while interrupts - * were soft-disabled. We want to invoke the exception handler for - * the access, or panic if there isn't a handler. - */ -77: addi r3,r1,STACK_FRAME_OVERHEAD - li r5,SIGSEGV - bl bad_page_fault - b interrupt_returnarch/powerpc/mm/book3s64/hash_utils.c +49
-28@@ -1510,16 +1510,40 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, }EXPORT_SYMBOL_GPL (hash_page); -int __hash_page (unsigned long trap,unsigned long ea,unsigned long dsisr, -unsigned long msr) +int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr) {unsigned long access = _PAGE_PRESENT | _PAGE_READ;unsigned long flags =0 ; -struct mm_struct *mm = current->mm; -unsigned int region_id =get_region_id (ea); +struct mm_struct *mm; +unsigned int region_id; +int err; + +if (unlikely (dsisr & (DSISR_BAD_FAULT_64S |DSISR_DABRMATCH |DSISR_KEYFAULT ))) + goto _do_page_fault; + +/* +* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then +* don't call hash_page, just fail the fault. This is required to +* prevent re-entrancy problems in the hash code, namely perf +* interrupts hitting while something holds H_PAGE_BUSY, and taking a +* hash fault. See the comment in hash_preload(). +* +* We come here as a result of a DSI at a point where we don't want +* to call hash_page, such as when we are accessing memory (possibly +* user memory) inside a PMU interrupt that occurred while interrupts +* were soft-disabled. We want to invoke the exception handler for +* the access, or panic if there isn't a handler. +*/ +if (unlikely (in_nmi ())) { +bad_page_fault (regs, ea,SIGSEGV ); +return 0 ; + } + region_id =get_region_id (ea);if ((region_id ==VMALLOC_REGION_ID ) || (region_id ==IO_REGION_ID )) mm = &init_mm; +else + mm = current->mm;if (dsisr &DSISR_NOHPTE ) flags |=HPTE_NOHPTE_UPDATE ;@@ -1535,13 +1559,31 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,* 2) user space access kernel space. */ access |= _PAGE_PRIVILEGED; -if ((msr &MSR_PR ) || (region_id ==USER_REGION_ID )) +if (user_mode (regs) || (region_id ==USER_REGION_ID )) access &= ~_PAGE_PRIVILEGED; -if (trap ==0x400 ) +if (regs->trap ==0x400 ) access |= _PAGE_EXEC; -return hash_page_mm (mm, ea, access, trap, flags); + err =hash_page_mm (mm, ea, access, regs->trap, flags); +if (unlikely (err <0 )) { +// failed to instert a hash PTE due to an hypervisor error +if (user_mode (regs)) { +if (IS_ENABLED (CONFIG_PPC_SUBPAGE_PROT ) && err ==-2 ) +_exception (SIGSEGV , regs,SEGV_ACCERR , ea); +else +_exception (SIGBUS , regs,BUS_ADRERR , ea); + }else { +bad_page_fault (regs, ea,SIGBUS ); + } + err =0 ; + + }else if (err) { +_do_page_fault: + err =hash__do_page_fault (regs, ea, dsisr); + } + +return err; }#ifdef CONFIG_PPC_MM_SLICES @@ -1841,27 +1883,6 @@ void flush_hash_range(unsigned long number, int local) } } -/* -* low_hash_fault is called when we the low level hash code failed -* to instert a PTE due to an hypervisor error -*/ -void low_hash_fault (struct pt_regs *regs,unsigned long address,int rc) -{ -enum ctx_state prev_state =exception_enter (); - -if (user_mode (regs)) { -#ifdef CONFIG_PPC_SUBPAGE_PROT -if (rc ==-2 ) -_exception (SIGSEGV , regs,SEGV_ACCERR , address); - else -#endif -_exception (SIGBUS , regs,BUS_ADRERR , address); - }else -bad_page_fault (regs, address,SIGBUS ); - -exception_exit (prev_state); -} -long hpte_insert_repeating (unsigned long hash,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int ssize)arch/powerpc/mm/fault.c +54
-3@@ -370,7 +370,9 @@ static void sanity_check_fault(bool is_write, bool is_user,#define page_fault_is_write (__err) ((__err) & DSISR_ISSTORE)#if defined(CONFIG_PPC_8xx)#define page_fault_is_bad (__err) ((__err) & DSISR_NOEXEC_OR_G) -#elif defined(CONFIG_PPC64 ) +#elif defined(CONFIG_PPC_BOOK3S_64 ) +#define page_fault_is_bad (__err) ((__err) & (DSISR_BAD_FAULT_64S | DSISR_DABRMATCH)) +#elif defined(CONFIG_PPC_BOOK3E_64 )#define page_fault_is_bad (__err) ((__err) & DSISR_BAD_FAULT_64S)#else #define page_fault_is_bad (__err) ((__err) & DSISR_BAD_FAULT_32S)@@ -406,6 +408,9 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,return 0 ;if (unlikely (page_fault_is_bad (error_code))) { +if (IS_ENABLED (CONFIG_PPC_BOOK3S_64 ) && (error_code &DSISR_DABRMATCH )) +return -1 ; +if (is_user) {_exception (SIGBUS , regs,BUS_OBJERR , address);return 0 ;@@ -548,12 +553,58 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, unsigned long error_code) {enum ctx_state prev_state =exception_enter (); -int rc =__do_page_fault (regs, address, error_code); +int err; + + err =__do_page_fault (regs, address, error_code); + +#ifdef CONFIG_PPC_BOOK3S_64 +/* 32 and 64e handle errors in their asm code */ +if (unlikely (err)) { +if (err >0 ) { +bad_page_fault (regs, address, err); + err =0 ; + }else { +/* +* do_break() may change NV GPRS while handling the +* breakpoint. Return -ve to caller to do that. +*/ +do_break (regs, address, error_code); + } + } +#endif +exception_exit (prev_state); -return rc; + +return err; }NOKPROBE_SYMBOL (do_page_fault); +#ifdef CONFIG_PPC_BOOK3S_64 +/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ +int hash__do_page_fault (struct pt_regs *regs,unsigned long address, +unsigned long error_code) +{ +int err; + + err =__do_page_fault (regs, address, error_code); +if (unlikely (err)) { +if (err >0 ) { +bad_page_fault (regs, address, err); + err =0 ; + }else { +/* +* do_break() may change NV GPRS while handling the +* breakpoint. Return -ve to caller to do that. +*/ +do_break (regs, address, error_code); + } + } + +return err; +} +NOKPROBE_SYMBOL (hash__do_page_fault); +#endif +/* * bad_page_fault is called when we have a bad access from the kernel. * It is called from the DSI and ISI handlers in head.S and from some Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 2/19 powerpc: remove arguments from fault handler functions
In reply to Nicholas PigginMake mm fault handlers all just take the pt_regs * argument and load DAR/DSISR from that. Make those that return a value return long.
This is done to make the function signatures match other handlers, which will help with a future patch to add wrappers. Explicit arguments could be added for performance but that would require more wrapper macro variants.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/asm-prototypes.h | 4 ++-- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 +- arch/powerpc/include/asm/bug.h | 4 ++-- arch/powerpc/kernel/exceptions-64e.S | 2 -- arch/powerpc/kernel/exceptions-64s.S | 14 ++------------ arch/powerpc/kernel/head_40x.S | 10 +++++----- arch/powerpc/kernel/head_8xx.S | 6 +++--- arch/powerpc/kernel/head_book3s_32.S | 6 ++---- arch/powerpc/kernel/head_booke.h | 4 +--- arch/powerpc/mm/book3s64/hash_utils.c | 8 +++++--- arch/powerpc/mm/book3s64/slb.c | 11 +++++++---- arch/powerpc/mm/fault.c | 16 +++++++++------- 12 files changed, 39 insertions(+), 48 deletions(-)
arch/powerpc/include/asm/asm-prototypes.h +2 -2
@@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);void system_reset_exception (struct pt_regs *regs);void machine_check_exception (struct pt_regs *regs);void emulation_assist_interrupt (struct pt_regs *regs); -long do_slb_fault (struct pt_regs *regs,unsigned long ea); -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err); +long do_slb_fault (struct pt_regs *regs); +void do_bad_slb_fault (struct pt_regs *regs);/* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx,
arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -1
@@ -453,7 +453,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_LOCAL_UPDATE 0x1#define HPTE_NOHPTE_UPDATE 0x2 -int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr); +long do_hash_fault (struct pt_regs *regs);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);
arch/powerpc/include/asm/bug.h +2 -2
@@ -111,8 +111,8 @@#ifndef __ASSEMBLY__struct pt_regs ; -extern int do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); -int hash__do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); +long do_page_fault (struct pt_regs *); +long hash__do_page_fault (struct pt_regs *);extern void bad_page_fault (struct pt_regs *,unsigned long ,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );
arch/powerpc/kernel/exceptions-64e.S +0 -2
@@ -1011,8 +1011,6 @@ storage_fault_common: std r14,_DAR(r1) std r15,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 - mr r5,r15 ld r14,PACA_EXGEN+EX_R14(r13) ld r15,PACA_EXGEN+EX_R15(r13) bl do_page_fault
arch/powerpc/kernel/exceptions-64s.S +2 -12
@@ -1438,8 +1438,6 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) EXC_VIRT_END(data_access, 0x4300, 0x80) EXC_COMMON_BEGIN(data_access_common) GEN_COMMON data_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION bl do_hash_fault
@@ -1492,10 +1490,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) EXC_VIRT_END(data_access_slb, 0x4380, 0x80) EXC_COMMON_BEGIN(data_access_slb_common) GEN_COMMON data_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f
@@ -1507,8 +1504,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return
@@ -1543,8 +1538,6 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) EXC_VIRT_END(instruction_access, 0x4400, 0x80) EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION bl do_hash_fault
@@ -1588,10 +1581,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) EXC_COMMON_BEGIN(instruction_access_slb_common) GEN_COMMON instruction_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f
@@ -1603,8 +1595,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return
arch/powerpc/kernel/head_40x.S +5 -5
@@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5, SPRN_ESR /* Grab the ESR, save it */ stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault)
@@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG - mr r4,r12 /* Pass SRR0 as arg2 */ - stw r4, _DEAR(r11) - li r5,0 /* Pass zero as arg3 */ + li r5,0 + stw r5, _ESR(r11) /* Zero ESR */ + stw r12, _DEAR(r11) /* SRR0 as DEAR */ EXC_XFER_LITE(0x400, handle_page_fault) /* 0x0500 - External Interrupt Exception */
arch/powerpc/kernel/head_8xx.S +3 -3
@@ -324,14 +324,14 @@ DataStoreTLBMiss: . = 0x1300 InstructionTLBError: EXCEPTION_PROLOG - mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ andis. r10,r9,SRR1_ISI_NOPT@h beq+ .Litlbie - tlbie r4 + tlbie r12 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due to
arch/powerpc/kernel/head_book3s_32.S +2 -4
@@ -370,9 +370,9 @@ BEGIN_MMU_FTR_SECTION bl hash_page END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r5, _DSISR(r11) + stw r12, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */
@@ -687,8 +687,6 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) - lwz r5, _DSISR(r11) /* fall through */ handle_page_fault_tramp_2: EXC_XFER_LITE(0x300, handle_page_fault)
arch/powerpc/kernel/head_booke.h +1 -3
@@ -477,9 +477,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)NORMAL_EXCEPTION_PROLOG (INST_STORAGE ); \mfspr r5,SPRN_ESR ;/* Grab the ESR and save it */ \stw r5,_ESR (r11 ); \ -mr r4,r12;/* Pass SRR0 as arg2 */ \ -stw r4,_DEAR (r11 ); \ - li r5,0; /* Pass zero as arg3 */ \ +stw r12,_DEAR (r11 );/* Pass SRR0 as arg2 */ \ EXC_XFER_LITE(0x0400, handle_page_fault)#define ALIGNMENT_EXCEPTION \
arch/powerpc/mm/book3s64/hash_utils.c +5 -3
@@ -1510,13 +1510,15 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, }EXPORT_SYMBOL_GPL (hash_page); -int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr) +long do_hash_fault (struct pt_regs *regs) { +unsigned long ea = regs->dar; +unsigned long dsisr = regs->dsisr;unsigned long access = _PAGE_PRESENT | _PAGE_READ;unsigned long flags =0 ;struct mm_struct *mm;unsigned int region_id; -int err; +long err;if (unlikely (dsisr & (DSISR_BAD_FAULT_64S |DSISR_DABRMATCH |DSISR_KEYFAULT ))) goto _do_page_fault;
@@ -1580,7 +1582,7 @@ int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr) }else if (err) { _do_page_fault: - err =hash__do_page_fault (regs, ea, dsisr); + err =hash__do_page_fault (regs); }return err;
arch/powerpc/mm/book3s64/slb.c +7 -4
@@ -837,8 +837,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)return slb_insert_entry (ea, context, flags, ssize, false); } -long do_slb_fault (struct pt_regs *regs,unsigned long ea) +long do_slb_fault (struct pt_regs *regs) { +unsigned long ea = regs->dar;unsigned long id =get_region_id (ea);/* IRQs are not reconciled here, so can't check irqs_disabled */
@@ -889,13 +890,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) } } -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err) +void do_bad_slb_fault (struct pt_regs *regs) { +int err = regs->result; +if (err == -EFAULT ) {if (user_mode (regs)) -_exception (SIGSEGV , regs,SEGV_BNDERR , ea); +_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs, regs->dar,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {
arch/powerpc/mm/fault.c +9 -7
@@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, }NOKPROBE_SYMBOL (__do_page_fault); -int do_page_fault (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +long do_page_fault (struct pt_regs *regs) {enum ctx_state prev_state =exception_enter (); -int err; +unsigned long address = regs->dar; +unsigned long error_code = regs->dsisr; +long err; err =__do_page_fault (regs, address, error_code);
@@ -580,11 +581,12 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,NOKPROBE_SYMBOL (do_page_fault);#ifdef CONFIG_PPC_BOOK3S_64 -/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ -int hash__do_page_fault (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +/* Same as do_page_fault but no interrupt entry */ +long hash__do_page_fault (struct pt_regs *regs) { -int err; +unsigned long address = regs->dar; +unsigned long error_code = regs->dsisr; +long err; err =__do_page_fault (regs, address, error_code);if (unlikely (err)) {
Signature · 1 line
2.23.0
Re: [PATCH v3 02/19] powerpc: remove arguments from fault handler functions
Quoted 7 lines · Nicholas Piggin
Make mm fault handlers all just take the
pt_regs* argument and load DAR/DSISR from that. Make those that return a value return long.This is done to make the function signatures match other handlers, which will help with a future patch to add wrappers. Explicit arguments could be added for performance but that would require more wrapper macro variants.
Reviewed-by Aneesh Kumar K.V <[email protected]>
Quoted 332 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/asm-prototypes.h | 4 ++-- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 +- arch/powerpc/include/asm/bug.h | 4 ++-- arch/powerpc/kernel/exceptions-64e.S | 2 -- arch/powerpc/kernel/exceptions-64s.S | 14 ++------------ arch/powerpc/kernel/head_40x.S | 10 +++++----- arch/powerpc/kernel/head_8xx.S | 6 +++--- arch/powerpc/kernel/head_book3s_32.S | 6 ++---- arch/powerpc/kernel/head_booke.h | 4 +--- arch/powerpc/mm/book3s64/hash_utils.c | 8 +++++--- arch/powerpc/mm/book3s64/slb.c | 11 +++++++---- arch/powerpc/mm/fault.c | 16 +++++++++------- 12 files changed, 39 insertions(+), 48 deletions(-)arch/powerpc/include/asm/asm-prototypes.h +2
-2@@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);void system_reset_exception (struct pt_regs *regs);void machine_check_exception (struct pt_regs *regs);void emulation_assist_interrupt (struct pt_regs *regs); -long do_slb_fault (struct pt_regs *regs,unsigned long ea); -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err); +long do_slb_fault (struct pt_regs *regs); +void do_bad_slb_fault (struct pt_regs *regs);/* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx,arch/powerpc/include/asm/book3s/64/mmu-hash.h +1
-1@@ -453,7 +453,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_LOCAL_UPDATE 0x1#define HPTE_NOHPTE_UPDATE 0x2 -int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr); +long do_hash_fault (struct pt_regs *regs);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);arch/powerpc/include/asm/bug.h +2
-2@@ -111,8 +111,8 @@#ifndef __ASSEMBLY__struct pt_regs ; -extern int do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); -int hash__do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); +long do_page_fault (struct pt_regs *); +long hash__do_page_fault (struct pt_regs *);extern void bad_page_fault (struct pt_regs *,unsigned long ,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );arch/powerpc/kernel/exceptions-64e.S +0
-2@@ -1011,8 +1011,6 @@ storage_fault_common: std r14,_DAR(r1) std r15,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 - mr r5,r15 ld r14,PACA_EXGEN+EX_R14(r13) ld r15,PACA_EXGEN+EX_R15(r13) bl do_page_faultarch/powerpc/kernel/exceptions-64s.S +2
-12@@ -1438,8 +1438,6 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) EXC_VIRT_END(data_access, 0x4300, 0x80) EXC_COMMON_BEGIN(data_access_common) GEN_COMMON data_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION bl do_hash_fault@@ -1492,10 +1490,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) EXC_VIRT_END(data_access_slb, 0x4380, 0x80) EXC_COMMON_BEGIN(data_access_slb_common) GEN_COMMON data_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f@@ -1507,8 +1504,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return@@ -1543,8 +1538,6 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) EXC_VIRT_END(instruction_access, 0x4400, 0x80) EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION bl do_hash_fault@@ -1588,10 +1581,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) EXC_COMMON_BEGIN(instruction_access_slb_common) GEN_COMMON instruction_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f@@ -1603,8 +1595,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_returnarch/powerpc/kernel/head_40x.S +5
-5@@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5, SPRN_ESR /* Grab the ESR, save it */ stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault)@@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG - mr r4,r12 /* Pass SRR0 as arg2 */ - stw r4, _DEAR(r11) - li r5,0 /* Pass zero as arg3 */ + li r5,0 + stw r5, _ESR(r11) /* Zero ESR */ + stw r12, _DEAR(r11) /* SRR0 as DEAR */ EXC_XFER_LITE(0x400, handle_page_fault) /* 0x0500 - External Interrupt Exception */arch/powerpc/kernel/head_8xx.S +3
-3@@ -324,14 +324,14 @@ DataStoreTLBMiss: . = 0x1300 InstructionTLBError: EXCEPTION_PROLOG - mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ andis. r10,r9,SRR1_ISI_NOPT@h beq+ .Litlbie - tlbie r4 + tlbie r12 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due toarch/powerpc/kernel/head_book3s_32.S +2
-4@@ -370,9 +370,9 @@ BEGIN_MMU_FTR_SECTION bl hash_page END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r5, _DSISR(r11) + stw r12, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */@@ -687,8 +687,6 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) - lwz r5, _DSISR(r11) /* fall through */ handle_page_fault_tramp_2: EXC_XFER_LITE(0x300, handle_page_fault)arch/powerpc/kernel/head_booke.h +1
-3@@ -477,9 +477,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)NORMAL_EXCEPTION_PROLOG (INST_STORAGE ); \mfspr r5,SPRN_ESR ;/* Grab the ESR and save it */ \stw r5,_ESR (r11 ); \ -mr r4,r12;/* Pass SRR0 as arg2 */ \ -stw r4,_DEAR (r11 ); \ - li r5,0; /* Pass zero as arg3 */ \ +stw r12,_DEAR (r11 );/* Pass SRR0 as arg2 */ \ EXC_XFER_LITE(0x0400, handle_page_fault)#define ALIGNMENT_EXCEPTION \arch/powerpc/mm/book3s64/hash_utils.c +5
-3@@ -1510,13 +1510,15 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, }EXPORT_SYMBOL_GPL (hash_page); -int do_hash_fault (struct pt_regs *regs,unsigned long ea,unsigned long dsisr) +long do_hash_fault (struct pt_regs *regs) { +unsigned long ea = regs->dar; +unsigned long dsisr = regs->dsisr;unsigned long access = _PAGE_PRESENT | _PAGE_READ;unsigned long flags =0 ;struct mm_struct *mm;unsigned int region_id; -int err; +long err;if (unlikely (dsisr & (DSISR_BAD_FAULT_64S |DSISR_DABRMATCH |DSISR_KEYFAULT ))) goto _do_page_fault;@@ -1580,7 +1582,7 @@ int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr) }else if (err) { _do_page_fault: - err =hash__do_page_fault (regs, ea, dsisr); + err =hash__do_page_fault (regs); }return err;arch/powerpc/mm/book3s64/slb.c +7
-4@@ -837,8 +837,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)return slb_insert_entry (ea, context, flags, ssize, false); } -long do_slb_fault (struct pt_regs *regs,unsigned long ea) +long do_slb_fault (struct pt_regs *regs) { +unsigned long ea = regs->dar;unsigned long id =get_region_id (ea);/* IRQs are not reconciled here, so can't check irqs_disabled */ @@ -889,13 +890,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) } } -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err) +void do_bad_slb_fault (struct pt_regs *regs) { +int err = regs->result; +if (err == -EFAULT ) {if (user_mode (regs)) -_exception (SIGSEGV , regs,SEGV_BNDERR , ea); +_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs, regs->dar,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {arch/powerpc/mm/fault.c +9
-7@@ -549,11 +549,12 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, }NOKPROBE_SYMBOL (__do_page_fault); -int do_page_fault (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +long do_page_fault (struct pt_regs *regs) {enum ctx_state prev_state =exception_enter (); -int err; +unsigned long address = regs->dar; +unsigned long error_code = regs->dsisr; +long err; err =__do_page_fault (regs, address, error_code);@@ -580,11 +581,12 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,NOKPROBE_SYMBOL (do_page_fault);#ifdef CONFIG_PPC_BOOK3S_64 -/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ -int hash__do_page_fault (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +/* Same as do_page_fault but no interrupt entry */ +long hash__do_page_fault (struct pt_regs *regs) { -int err; +unsigned long address = regs->dar; +unsigned long error_code = regs->dsisr; +long err; err =__do_page_fault (regs, address, error_code);if (unlikely (err)) {Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 3/19 powerpc: bad_page_fault, do_break get registers from regs
In reply to Nicholas PigginSimilar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
bad_page_fault and do_break are not performance critical.
[32s DABR code from Christophe Leroy <[email protected]>]
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/bug.h | 2 +- arch/powerpc/include/asm/debug.h | 3 +-- arch/powerpc/kernel/entry_32.S | 18 +----------------- arch/powerpc/kernel/exceptions-64e.S | 3 +-- arch/powerpc/kernel/exceptions-64s.S | 3 +-- arch/powerpc/kernel/head_8xx.S | 5 ++--- arch/powerpc/kernel/head_book3s_32.S | 3 +++ arch/powerpc/kernel/process.c | 7 +++---- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/mm/book3s64/hash_utils.c | 4 ++-- arch/powerpc/mm/book3s64/slb.c | 2 +- arch/powerpc/mm/fault.c | 10 +++++----- arch/powerpc/platforms/8xx/machine_check.c | 2 +- 13 files changed, 23 insertions(+), 41 deletions(-)
arch/powerpc/include/asm/bug.h +1 -1
@@ -113,7 +113,7 @@struct pt_regs ;long do_page_fault (struct pt_regs *);long hash__do_page_fault (struct pt_regs *); -extern void bad_page_fault (struct pt_regs *,unsigned long ,int ); +void bad_page_fault (struct pt_regs *,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );extern void die (const char *,struct pt_regs *,long );
arch/powerpc/include/asm/debug.h +1 -2
@@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt);#else -extern void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code); +void do_break (struct pt_regs *regs);#endif #endif /* _ASM_POWERPC_DEBUG_H */
arch/powerpc/kernel/entry_32.S +1 -17
@@ -657,10 +657,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except
@@ -668,23 +664,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the state
arch/powerpc/kernel/exceptions-64e.S +1 -2
@@ -1018,9 +1018,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl bad_page_fault b ret_from_except
arch/powerpc/kernel/exceptions-64s.S +1 -2
@@ -2136,8 +2136,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r4,_DAR(r1) - li r5,SIGSEGV + li r4,SIGSEGV bl bad_page_fault MMU_FTR_SECTION_ELSE bl unknown_exception
arch/powerpc/kernel/head_8xx.S +2 -3
@@ -376,10 +376,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break)
arch/powerpc/kernel/head_book3s_32.S +3 -0
@@ -689,7 +689,10 @@ handle_page_fault_tramp_1: #endif /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK .macro save_regs_thread thread
arch/powerpc/kernel/process.c +3 -4
@@ -660,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +void do_break (struct pt_regs *regs) { current->thread.trap_nr =TRAP_HWBKPT ; -if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, error_code, +if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, regs->dsisr,11 ,SIGSEGV ) ==NOTIFY_STOP )return ;
@@ -682,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address,do_break_handler (regs);/* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); }#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
arch/powerpc/kernel/traps.c +1 -1
@@ -1641,7 +1641,7 @@ void alignment_exception(struct pt_regs *regs)if (user_mode (regs))_exception (sig, regs, code, regs->dar);else -bad_page_fault (regs, regs->dar, sig); +bad_page_fault (regs, sig); bail:exception_exit (prev_state);
arch/powerpc/mm/book3s64/hash_utils.c +2 -2
@@ -1537,7 +1537,7 @@ long do_hash_fault(struct pt_regs *regs)* the access, or panic if there isn't a handler. */ if (unlikely (in_nmi ())) { -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs,SIGSEGV );return 0 ; }
@@ -1576,7 +1576,7 @@ long do_hash_fault(struct pt_regs *regs)else _exception (SIGBUS , regs,BUS_ADRERR , ea); }else { -bad_page_fault (regs, ea,SIGBUS ); +bad_page_fault (regs,SIGBUS ); } err =0 ;
arch/powerpc/mm/book3s64/slb.c +1 -1
@@ -898,7 +898,7 @@ void do_bad_slb_fault(struct pt_regs *regs)if (user_mode (regs))_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, regs->dar,SIGSEGV ); +bad_page_fault (regs,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {
arch/powerpc/mm/fault.c +5 -5
@@ -562,14 +562,14 @@ long do_page_fault(struct pt_regs *regs)/* 32 and 64e handle errors in their asm code */ if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }#endif
@@ -591,14 +591,14 @@ long hash__do_page_fault(struct pt_regs *regs) err =__do_page_fault (regs, address, error_code);if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }
@@ -612,7 +612,7 @@ NOKPROBE_SYMBOL(hash__do_page_fault);* It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void bad_page_fault (struct pt_regs *regs,unsigned long address,int sig) +void bad_page_fault (struct pt_regs *regs,int sig) {const struct exception_table_entry *entry;int is_write =page_fault_is_write (regs->dsisr);
arch/powerpc/platforms/8xx/machine_check.c +1 -1
@@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)* to deal with that than having a wart in the mcheck handler. * -- BenH */ -bad_page_fault (regs, regs->dar,SIGBUS ); +bad_page_fault (regs,SIGBUS );return 1 ;#else return 0 ;
Signature · 1 line
2.23.0
Re: [PATCH v3 03/19] powerpc: bad_page_fault, do_break get registers from regs
Quoted 5 lines · Nicholas Piggin
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
bad_page_faultanddo_breakare not performance critical.
Reviewed-by Aneesh Kumar K.V <[email protected]>
Quoted 283 lines
[32s DABR code from Christophe Leroy <[email protected]>]Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/bug.h | 2 +- arch/powerpc/include/asm/debug.h | 3 +-- arch/powerpc/kernel/entry_32.S | 18 +----------------- arch/powerpc/kernel/exceptions-64e.S | 3 +-- arch/powerpc/kernel/exceptions-64s.S | 3 +-- arch/powerpc/kernel/head_8xx.S | 5 ++--- arch/powerpc/kernel/head_book3s_32.S | 3 +++ arch/powerpc/kernel/process.c | 7 +++---- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/mm/book3s64/hash_utils.c | 4 ++-- arch/powerpc/mm/book3s64/slb.c | 2 +- arch/powerpc/mm/fault.c | 10 +++++----- arch/powerpc/platforms/8xx/machine_check.c | 2 +- 13 files changed, 23 insertions(+), 41 deletions(-)arch/powerpc/include/asm/bug.h +1
-1@@ -113,7 +113,7 @@struct pt_regs ;long do_page_fault (struct pt_regs *);long hash__do_page_fault (struct pt_regs *); -extern void bad_page_fault (struct pt_regs *,unsigned long ,int ); +void bad_page_fault (struct pt_regs *,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );extern void die (const char *,struct pt_regs *,long );arch/powerpc/include/asm/debug.h +1
-2@@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt);#else -extern void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code); +void do_break (struct pt_regs *regs);#endif #endif /* _ASM_POWERPC_DEBUG_H */ arch/powerpc/kernel/entry_32.S +1
-17@@ -657,10 +657,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except@@ -668,23 +664,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the statearch/powerpc/kernel/exceptions-64e.S +1
-2@@ -1018,9 +1018,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl bad_page_fault b ret_from_exceptarch/powerpc/kernel/exceptions-64s.S +1
-2@@ -2136,8 +2136,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r4,_DAR(r1) - li r5,SIGSEGV + li r4,SIGSEGV bl bad_page_fault MMU_FTR_SECTION_ELSE bl unknown_exceptionarch/powerpc/kernel/head_8xx.S +2
-3@@ -376,10 +376,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break)arch/powerpc/kernel/head_book3s_32.S +3
-0@@ -689,7 +689,10 @@ handle_page_fault_tramp_1: #endif /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK .macro save_regs_thread threadarch/powerpc/kernel/process.c +3
-4@@ -660,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +void do_break (struct pt_regs *regs) { current->thread.trap_nr =TRAP_HWBKPT ; -if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, error_code, +if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, regs->dsisr,11 ,SIGSEGV ) ==NOTIFY_STOP )return ;@@ -682,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address,do_break_handler (regs);/* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); }#endif /* CONFIG_PPC_ADV_DEBUG_REGS */ arch/powerpc/kernel/traps.c +1
-1@@ -1641,7 +1641,7 @@ void alignment_exception(struct pt_regs *regs)if (user_mode (regs))_exception (sig, regs, code, regs->dar);else -bad_page_fault (regs, regs->dar, sig); +bad_page_fault (regs, sig); bail:exception_exit (prev_state);arch/powerpc/mm/book3s64/hash_utils.c +2
-2@@ -1537,7 +1537,7 @@ long do_hash_fault(struct pt_regs *regs)* the access, or panic if there isn't a handler. */ if (unlikely (in_nmi ())) { -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs,SIGSEGV );return 0 ; }@@ -1576,7 +1576,7 @@ long do_hash_fault(struct pt_regs *regs)else _exception (SIGBUS , regs,BUS_ADRERR , ea); }else { -bad_page_fault (regs, ea,SIGBUS ); +bad_page_fault (regs,SIGBUS ); } err =0 ;arch/powerpc/mm/book3s64/slb.c +1
-1@@ -898,7 +898,7 @@ void do_bad_slb_fault(struct pt_regs *regs)if (user_mode (regs))_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, regs->dar,SIGSEGV ); +bad_page_fault (regs,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {arch/powerpc/mm/fault.c +5
-5@@ -562,14 +562,14 @@ long do_page_fault(struct pt_regs *regs)/* 32 and 64e handle errors in their asm code */ if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }#endif @@ -591,14 +591,14 @@ long hash__do_page_fault(struct pt_regs *regs) err =__do_page_fault (regs, address, error_code);if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }@@ -612,7 +612,7 @@ NOKPROBE_SYMBOL(hash__do_page_fault);* It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void bad_page_fault (struct pt_regs *regs,unsigned long address,int sig) +void bad_page_fault (struct pt_regs *regs,int sig) {const struct exception_table_entry *entry;int is_write =page_fault_is_write (regs->dsisr);arch/powerpc/platforms/8xx/machine_check.c +1
-1@@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)* to deal with that than having a wart in the mcheck handler. * -- BenH */ -bad_page_fault (regs, regs->dar,SIGBUS ); +bad_page_fault (regs,SIGBUS );return 1 ;#else return 0 ;Signature · 1 line
2.23.0
Re: [PATCH v3 03/19] powerpc: bad_page_fault, do_break get registers from regs
In reply to Nicholas PigginLe 28/11/2020 à 15:40, Nicholas Piggin a écrit :
Quoted 4 lines
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
bad_page_faultanddo_breakare not performance critical.
I partly took your changes into one of my series, in different order though.
Please have a look at https://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=221656 patches 4 to 7
I think some of the changes are missing in your series, especially the changes in entry_32.S from patch 7.
Will see how our two series make their way into mainline, yours needs rebase anyway.
Christophe
Quoted 283 lines
[32s DABR code from Christophe Leroy <[email protected]>]Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/bug.h | 2 +- arch/powerpc/include/asm/debug.h | 3 +-- arch/powerpc/kernel/entry_32.S | 18 +----------------- arch/powerpc/kernel/exceptions-64e.S | 3 +-- arch/powerpc/kernel/exceptions-64s.S | 3 +-- arch/powerpc/kernel/head_8xx.S | 5 ++--- arch/powerpc/kernel/head_book3s_32.S | 3 +++ arch/powerpc/kernel/process.c | 7 +++---- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/mm/book3s64/hash_utils.c | 4 ++-- arch/powerpc/mm/book3s64/slb.c | 2 +- arch/powerpc/mm/fault.c | 10 +++++----- arch/powerpc/platforms/8xx/machine_check.c | 2 +- 13 files changed, 23 insertions(+), 41 deletions(-)arch/powerpc/include/asm/bug.h +1
-1@@ -113,7 +113,7 @@struct pt_regs ;long do_page_fault (struct pt_regs *);long hash__do_page_fault (struct pt_regs *); -extern void bad_page_fault (struct pt_regs *,unsigned long ,int ); +void bad_page_fault (struct pt_regs *,int );extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );extern void die (const char *,struct pt_regs *,long );arch/powerpc/include/asm/debug.h +1
-2@@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt);#else -extern void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code); +void do_break (struct pt_regs *regs);#endif #endif /* _ASM_POWERPC_DEBUG_H */ arch/powerpc/kernel/entry_32.S +1
-17@@ -657,10 +657,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except@@ -668,23 +664,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the statearch/powerpc/kernel/exceptions-64e.S +1
-2@@ -1018,9 +1018,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl bad_page_fault b ret_from_exceptarch/powerpc/kernel/exceptions-64s.S +1
-2@@ -2136,8 +2136,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r4,_DAR(r1) - li r5,SIGSEGV + li r4,SIGSEGV bl bad_page_fault MMU_FTR_SECTION_ELSE bl unknown_exceptionarch/powerpc/kernel/head_8xx.S +2
-3@@ -376,10 +376,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break)arch/powerpc/kernel/head_book3s_32.S +3
-0@@ -689,7 +689,10 @@ handle_page_fault_tramp_1: #endif /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK .macro save_regs_thread threadarch/powerpc/kernel/process.c +3
-4@@ -660,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +void do_break (struct pt_regs *regs) { current->thread.trap_nr =TRAP_HWBKPT ; -if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, error_code, +if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, regs->dsisr,11 ,SIGSEGV ) ==NOTIFY_STOP )return ;@@ -682,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address,do_break_handler (regs);/* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); }#endif /* CONFIG_PPC_ADV_DEBUG_REGS */ arch/powerpc/kernel/traps.c +1
-1@@ -1641,7 +1641,7 @@ void alignment_exception(struct pt_regs *regs)if (user_mode (regs))_exception (sig, regs, code, regs->dar);else -bad_page_fault (regs, regs->dar, sig); +bad_page_fault (regs, sig); bail:exception_exit (prev_state);arch/powerpc/mm/book3s64/hash_utils.c +2
-2@@ -1537,7 +1537,7 @@ long do_hash_fault(struct pt_regs *regs)* the access, or panic if there isn't a handler. */ if (unlikely (in_nmi ())) { -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs,SIGSEGV );return 0 ; }@@ -1576,7 +1576,7 @@ long do_hash_fault(struct pt_regs *regs)else _exception (SIGBUS , regs,BUS_ADRERR , ea); }else { -bad_page_fault (regs, ea,SIGBUS ); +bad_page_fault (regs,SIGBUS ); } err =0 ;arch/powerpc/mm/book3s64/slb.c +1
-1@@ -898,7 +898,7 @@ void do_bad_slb_fault(struct pt_regs *regs)if (user_mode (regs))_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, regs->dar,SIGSEGV ); +bad_page_fault (regs,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {arch/powerpc/mm/fault.c +5
-5@@ -562,14 +562,14 @@ long do_page_fault(struct pt_regs *regs)/* 32 and 64e handle errors in their asm code */ if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }#endif @@ -591,14 +591,14 @@ long hash__do_page_fault(struct pt_regs *regs) err =__do_page_fault (regs, address, error_code);if (unlikely (err)) {if (err >0 ) { -bad_page_fault (regs, address, err); +bad_page_fault (regs, err); err =0 ; }else {/* * do_break() may change NV GPRS while handling the * breakpoint. Return -ve to caller to do that. */ -do_break (regs, address, error_code); +do_break (regs); } }@@ -612,7 +612,7 @@ NOKPROBE_SYMBOL(hash__do_page_fault);* It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void bad_page_fault (struct pt_regs *regs,unsigned long address,int sig) +void bad_page_fault (struct pt_regs *regs,int sig) {const struct exception_table_entry *entry;int is_write =page_fault_is_write (regs->dsisr);arch/powerpc/platforms/8xx/machine_check.c +1
-1@@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)* to deal with that than having a wart in the mcheck handler. * -- BenH */ -bad_page_fault (regs, regs->dar,SIGBUS ); +bad_page_fault (regs,SIGBUS );return 1 ;#else return 0 ;
Nicholas Piggin
reviewerRe: [PATCH v3 03/19] powerpc: bad_page_fault, do_break get registers from regs
Excerpts from Christophe Leroy's message of December 23, 2020 12:42 am:
Quoted 6 lines
Le 28/11/2020 à 15:40, Nicholas Piggin a écrit :
Quoted 3 lines
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
Quoted 6 lines
Quoted 1 line
bad_page_faultanddo_breakare not performance critical.I partly took your changes into one of my series, in different order though.
Please have a look at https://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=221656 patches 4 to 7
Thanks, I had a look. Seems like the result is basically the same as my series, so that's good if you like the end result now :)
Quoted 2 lines
I think some of the changes are missing in your series, especially the changes in
entry_32.Sfrom patch 7.
Okay I could take them in. In your patch 7/15, why do you leave this load of DSISR?
arch/powerpc/kernel/head_book3s_32.S +2 -3
@@ -369,9 +369,9 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) #endif #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */
@@ -693,7 +693,6 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) lwz r5, _DSISR(r11)
^^^^^^^^^^^^^^^^^^^^^^
/* fall through */
handle_page_fault_tramp_2:
?
Quoted 1 line
Will see how our two series make their way into mainline, yours needs rebase anyway.
I have it rebased, just waiting for a bit after merge window to repost. Would be good if mine can go first so I don't have to redo the 64s page fault to C conversion again. AFAIKS after that you can just drop 4-7, no conflicts? (after bugs are fixed)
Thanks,
Nick
Nicholas Piggin
reviewerRe: [PATCH v3 03/19] powerpc: bad_page_fault, do_break get registers from regs
Excerpts from Nicholas Piggin's message of December 26, 2020 6:19 pm:
Quoted 49 lines
Excerpts from Christophe Leroy's message of December 23, 2020 12:42 am:
Quoted 6 lines
Le 28/11/2020 à 15:40, Nicholas Piggin a écrit :
Quoted 3 lines
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
Quoted 6 lines
Quoted 1 line
bad_page_faultanddo_breakare not performance critical.I partly took your changes into one of my series, in different order though.
Please have a look at https://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=221656 patches 4 to 7
Thanks, I had a look. Seems like the result is basically the same as my series, so that's good if you like the end result now :)
Quoted 2 lines
I think some of the changes are missing in your series, especially the changes in
entry_32.Sfrom patch 7.Okay I could take them in. In your patch 7/15, why do you leave this load of DSISR?
arch/powerpc/kernel/head_book3s_32.S +2
-3@@ -369,9 +369,9 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) #endif #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */@@ -693,7 +693,6 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) lwz r5, _DSISR(r11) ^^^^^^^^^^^^^^^^^^^^^^ /* fall through */
handle_page_fault_tramp_2:
Quoted 1 line
?
Ah never mind, this needs to come back after your DABR match move patch, which you have earlier in the series. I confused myself.
I'll rebase my series on your patch 4 rather than have it squashed in with other do_break stuff.
Thanks,
Nick
Nicholas Piggin
reviewerPATCH v3 4/19 powerpc/perf: move perf irq/nmi handling details into traps.c
In reply to Nicholas PigginThis is required in order to allow more significant differences between NMI type interrupt handlers and regular asynchronous handlers.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/kernel/traps.c | 31 +++++++++++++++++++++++++++- arch/powerpc/perf/core-book3s.c | 35 ++------------------------------ arch/powerpc/perf/core-fsl-emb.c | 25 ----------------------- 3 files changed, 32 insertions(+), 59 deletions(-)
arch/powerpc/kernel/traps.c +30 -1
@@ -1919,11 +1919,40 @@ void vsx_unavailable_tm(struct pt_regs *regs) }#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ -void performance_monitor_exception (struct pt_regs *regs) +static void performance_monitor_exception_nmi (struct pt_regs *regs) +{ +nmi_enter (); + +__this_cpu_inc (irq_stat.pmu_irqs); + +perf_irq (regs); + +nmi_exit (); +} + +static void performance_monitor_exception_async (struct pt_regs *regs) { +irq_enter (); +__this_cpu_inc (irq_stat.pmu_irqs);perf_irq (regs); + +irq_exit (); +} + +void performance_monitor_exception (struct pt_regs *regs) +{ +/* +* On 64-bit, if perf interrupts hit in a local_irq_disable +* (soft-masked) region, we consider them as NMIs. This is required to +* prevent hash faults on user addresses when reading callchains (and +* looks better from an irq tracing perspective). +*/ +if (IS_ENABLED (CONFIG_PPC64 ) &&unlikely (arch_irq_disabled_regs (regs))) +performance_monitor_exception_nmi (regs); +else +performance_monitor_exception_async (regs); }#ifdef CONFIG_PPC_ADV_DEBUG_REGS
arch/powerpc/perf/core-book3s.c +2 -33
@@ -109,10 +109,6 @@ static inline void perf_read_regs(struct pt_regs *regs) { regs->result =0 ; } -static inline int perf_intr_is_nmi (struct pt_regs *regs) -{ -return 0 ; -}static inline int siar_valid (struct pt_regs *regs) {
@@ -328,15 +324,6 @@ static inline void perf_read_regs(struct pt_regs *regs) regs->result = use_siar; } -/* -* If interrupts were soft-disabled when a PMU interrupt occurs, treat -* it as an NMI. -*/ -static inline int perf_intr_is_nmi (struct pt_regs *regs) -{ -return (regs->softe &IRQS_DISABLED ); -} -/* * On processors like P7+ that have the SIAR-Valid bit, marked instructions * must be sampled only if the SIAR-valid bit is set.
@@ -2224,7 +2211,6 @@ static void __perf_event_interrupt(struct pt_regs *regs)struct perf_event *event;unsigned long val[8 ];int found, active; -int nmi;if (cpuhw->n_limited) freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
@@ -2232,18 +2218,6 @@ static void __perf_event_interrupt(struct pt_regs *regs)perf_read_regs (regs); -/* -* If perf interrupts hit in a local_irq_disable (soft-masked) region, -* we consider them as NMIs. This is required to prevent hash faults on -* user addresses when reading callchains. See the NMI test in -* do_hash_page. -*/ - nmi =perf_intr_is_nmi (regs); -if (nmi) -nmi_enter (); -else -irq_enter (); -/* Read all the PMCs since we'll need them a bunch of times */ for (i =0 ; i < ppmu->n_counter; ++i) val[i] =read_pmc (i +1 );
@@ -2289,8 +2263,8 @@ static void __perf_event_interrupt(struct pt_regs *regs) } } } -if (!found && !nmi &&printk_ratelimit ()) -printk (KERN_WARNING "Can't find PMC that caused IRQ\n" ); +if (unlikely (!found) && !arch_irq_disabled_regs (regs)) +printk_ratelimited (KERN_WARNING "Can't find PMC that caused IRQ\n" );/* * Reset MMCR0 to its normal value. This will set PMXE and
@@ -2300,11 +2274,6 @@ static void __perf_event_interrupt(struct pt_regs *regs)* we get back out of this interrupt. */ write_mmcr0 (cpuhw, cpuhw->mmcr.mmcr0); - -if (nmi) -nmi_exit (); -else -irq_exit (); }static void perf_event_interrupt (struct pt_regs *regs)
arch/powerpc/perf/core-fsl-emb.c +0 -25
@@ -31,19 +31,6 @@ static atomic_t num_events;/* Used to avoid races in calling reserve/release_pmc_hardware */ static DEFINE_MUTEX (pmc_reserve_mutex); -/* -* If interrupts were soft-disabled when a PMU interrupt occurs, treat -* it as an NMI. -*/ -static inline int perf_intr_is_nmi (struct pt_regs *regs) -{ -#ifdef __powerpc64__ -return (regs->softe &IRQS_DISABLED ); -#else -return 0 ; -#endif -} -static void perf_event_interrupt (struct pt_regs *regs);/*
@@ -659,13 +646,6 @@ static void perf_event_interrupt(struct pt_regs *regs)struct perf_event *event;unsigned long val;int found =0 ; -int nmi; - - nmi =perf_intr_is_nmi (regs); -if (nmi) -nmi_enter (); -else -irq_enter ();for (i =0 ; i < ppmu->n_counter; ++i) { event = cpuhw->event[i];
@@ -690,11 +670,6 @@ static void perf_event_interrupt(struct pt_regs *regs)mtmsr (mfmsr () |MSR_PMM );mtpmr (PMRN_PMGC0 ,PMGC0_PMIE |PMGC0_FCECE );isync (); - -if (nmi) -nmi_exit (); -else -irq_exit (); }void hw_perf_event_setup (int cpu)
Signature · 1 line
2.23.0
Re: [PATCH v3 04/19] powerpc/perf: move perf irq/nmi handling details into traps.c
On 28-Nov-2020, at 8:10 PM, Nicholas Piggin <[email protected]> wrote:
This is required in order to allow more significant differences between NMI type interrupt handlers and regular asynchronous handlers.
Signed-off-by Nicholas Piggin <[email protected]>
Reviewed this patch and the changes looks good to me.
Reviewed-by Athira Rajeev <[email protected]>
Thanks
Athira
---arch/powerpc/kernel/traps.c | 31 +++++++++++++++++++++++++++- arch/powerpc/perf/core-book3s.c | 35 ++------------------------------ arch/powerpc/perf/core-fsl-emb.c | 25 -----------------------
3 files changed, 32 insertions(+), 59 deletions(-)
arch/powerpc/kernel/traps.c +0 -0
@@ -1919,11 +1919,40 @@ void vsx_unavailable_tm(struct pt_regs *regs)
}
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
-void performance_monitor_exception(struct pt_regs *regs) +static void performance_monitor_exception_nmi(struct pt_regs *regs) +{
+nmi_enter();
+
+__this_cpu_inc(irq_stat.pmu_irqs);
+
+perf_irq(regs);
+
+nmi_exit();
+}
+
+static void performance_monitor_exception_async(struct pt_regs *regs) {
+irq_enter();
+__this_cpu_inc(irq_stat.pmu_irqs);
perf_irq(regs);
+
+irq_exit();
+}
+
+void performance_monitor_exception(struct pt_regs *regs)
+{
+ /*
+* On 64-bit, if perf interrupts hit in a local_irq_disable +* (soft-masked) region, we consider them as NMIs. This is required to +* prevent hash faults on user addresses when reading callchains (and +* looks better from an irq tracing perspective).
+ */
+if (IS_ENABLED(CONFIG_PPC64) && unlikely(arch_irq_disabled_regs(regs))) +performance_monitor_exception_nmi(regs);
+ else
+performance_monitor_exception_async(regs);
}
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
arch/powerpc/perf/core-book3s.c +0 -0
@@ -109,10 +109,6 @@ static inline void perf_read_regs(struct pt_regs *regs)
{regs->result = 0;
}
-static inline int perf_intr_is_nmi(struct pt_regs *regs) -{
-return 0;
-}
static inline int siar_valid(struct pt_regs *regs)
{
@@ -328,15 +324,6 @@ static inline void perf_read_regs(struct pt_regs *regs) regs->result = use_siar;
}
-/*
- * If interrupts were soft-disabled when a PMU interrupt occurs, treat
- * it as an NMI.
- */
-static inline int perf_intr_is_nmi(struct pt_regs *regs)
-{
-return (regs->softe & IRQS_DISABLED);
-}
-
/*
* On processors like P7+ that have the SIAR-Valid bit, marked instructions
* must be sampled only if the SIAR-valid bit is set.
@@ -2224,7 +2211,6 @@ static void __perf_event_interrupt(struct pt_regs *regs) struct perf_event *event;
unsigned long val[8];
int found, active;
-int nmi;
if (cpuhw->n_limited)
freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
@@ -2232,18 +2218,6 @@ static void __perf_event_interrupt(struct pt_regs *regs)
perf_read_regs(regs);
- /*
-* If perf interrupts hit in a local_irq_disable (soft-masked) region, -* we consider them as NMIs. This is required to prevent hash faults on -* user addresses when reading callchains. See the NMI test in
-* do_hash_page.
- */
-nmi = perf_intr_is_nmi(regs);
-if (nmi)
-nmi_enter();
- else
-irq_enter();
-
/* Read all the PMCs since we'll need them a bunch of times */ for (i = 0; i < ppmu->n_counter; ++i)
val[i] = read_pmc(i + 1);
@@ -2289,8 +2263,8 @@ static void __perf_event_interrupt(struct pt_regs *regs)
}
}
}
-if (!found && !nmi && printk_ratelimit())
-printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
+if (unlikely(!found) && !arch_irq_disabled_regs(regs)) +printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n");
/*
* Reset MMCR0 to its normal value. This will set PMXE and
@@ -2300,11 +2274,6 @@ static void __perf_event_interrupt(struct pt_regs *regs)
* we get back out of this interrupt.
*/write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0);
-
-if (nmi)
-nmi_exit();
- else
-irq_exit();
}
static void perf_event_interrupt(struct pt_regs *regs)
arch/powerpc/perf/core-fsl-emb.c +0 -0
@@ -31,19 +31,6 @@ static atomic_t num_events;
/* Used to avoid races in calling reserve/release_pmc_hardware */ static DEFINE_MUTEX(pmc_reserve_mutex);
-/*
- * If interrupts were soft-disabled when a PMU interrupt occurs, treat
- * it as an NMI.
- */
-static inline int perf_intr_is_nmi(struct pt_regs *regs)
-{
-#ifdef __powerpc64__
-return (regs->softe & IRQS_DISABLED);
-#else
-return 0;
-#endif
-}
-
static void perf_event_interrupt(struct pt_regs *regs);
/*
@@ -659,13 +646,6 @@ static void perf_event_interrupt(struct pt_regs *regs) struct perf_event *event;
unsigned long val;
int found = 0;
-int nmi;
-
-nmi = perf_intr_is_nmi(regs);
-if (nmi)
-nmi_enter();
- else
-irq_enter();
for (i = 0; i < ppmu->n_counter; ++i) {
event = cpuhw->event[i];
@@ -690,11 +670,6 @@ static void perf_event_interrupt(struct pt_regs *regs) mtmsr(mfmsr() | MSR_PMM);mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);isync();
-
-if (nmi)
-nmi_exit();
- else
-irq_exit();
}
void hw_perf_event_setup(int cpu)
--
2.23.0
Nicholas Piggin
reviewerPATCH v3 5/19 powerpc: interrupt handler wrapper functions
In reply to Nicholas PigginAdd wrapper functions (derived from x86 macros) for interrupt handler functions. This allows interrupt entry code to be written in C.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/asm-prototypes.h | 29 --- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 - arch/powerpc/include/asm/hw_irq.h | 9 - arch/powerpc/include/asm/interrupt.h | 205 ++++++++++++++++++ arch/powerpc/include/asm/time.h | 2 + arch/powerpc/kernel/dbell.c | 12 +- arch/powerpc/kernel/exceptions-64s.S | 7 +- arch/powerpc/kernel/head_book3s_32.S | 6 +- arch/powerpc/kernel/irq.c | 3 +- arch/powerpc/kernel/mce.c | 5 +- arch/powerpc/kernel/syscall_64.c | 1 + arch/powerpc/kernel/tau_6xx.c | 2 +- arch/powerpc/kernel/time.c | 3 +- arch/powerpc/kernel/traps.c | 90 +++++--- arch/powerpc/kernel/watchdog.c | 7 +- arch/powerpc/kvm/book3s_hv.c | 1 + arch/powerpc/kvm/book3s_hv_builtin.c | 1 + arch/powerpc/kvm/booke.c | 1 + arch/powerpc/mm/book3s64/hash_utils.c | 3 +- arch/powerpc/mm/book3s64/slb.c | 5 +- arch/powerpc/mm/fault.c | 10 +- arch/powerpc/platforms/powernv/idle.c | 1 + 22 files changed, 311 insertions(+), 93 deletions(-) create mode 100644 arch/powerpc/include/asm/interrupt.h
arch/powerpc/include/asm/asm-prototypes.h +0 -29
@@ -56,35 +56,6 @@ int exit_vmx_usercopy(void);int enter_vmx_ops (void );void *exit_vmx_ops (void *dest); -/* Traps */ -long machine_check_early (struct pt_regs *regs); -long hmi_exception_realmode (struct pt_regs *regs); -void SMIException (struct pt_regs *regs); -void handle_hmi_exception (struct pt_regs *regs); -void instruction_breakpoint_exception (struct pt_regs *regs); -void RunModeException (struct pt_regs *regs); -void single_step_exception (struct pt_regs *regs); -void program_check_exception (struct pt_regs *regs); -void alignment_exception (struct pt_regs *regs); -void StackOverflow (struct pt_regs *regs); -void stack_overflow_exception (struct pt_regs *regs); -void kernel_fp_unavailable_exception (struct pt_regs *regs); -void altivec_unavailable_exception (struct pt_regs *regs); -void vsx_unavailable_exception (struct pt_regs *regs); -void fp_unavailable_tm (struct pt_regs *regs); -void altivec_unavailable_tm (struct pt_regs *regs); -void vsx_unavailable_tm (struct pt_regs *regs); -void facility_unavailable_exception (struct pt_regs *regs); -void TAUException (struct pt_regs *regs); -void altivec_assist_exception (struct pt_regs *regs); -void unrecoverable_exception (struct pt_regs *regs); -void kernel_bad_stack (struct pt_regs *regs); -void system_reset_exception (struct pt_regs *regs); -void machine_check_exception (struct pt_regs *regs); -void emulation_assist_interrupt (struct pt_regs *regs); -long do_slb_fault (struct pt_regs *regs); -void do_bad_slb_fault (struct pt_regs *regs); -/* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx, struct ucontext __user *new_ctx,
arch/powerpc/include/asm/book3s/64/mmu-hash.h +0 -1
@@ -453,7 +453,6 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_LOCAL_UPDATE 0x1#define HPTE_NOHPTE_UPDATE 0x2 -long do_hash_fault (struct pt_regs *regs);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);
arch/powerpc/include/asm/hw_irq.h +0 -9
@@ -50,15 +50,6 @@#ifndef __ASSEMBLY__ -extern void replay_system_reset (void ); -extern void replay_soft_interrupts (void ); - -extern void timer_interrupt (struct pt_regs *); -extern void timer_broadcast_interrupt (void ); -extern void performance_monitor_exception (struct pt_regs *regs); -extern void WatchdogException (struct pt_regs *regs); -extern void unknown_exception (struct pt_regs *regs); -#ifdef CONFIG_PPC64 #include <asm/paca.h>
arch/powerpc/include/asm/interrupt.h new file +205 -0
205 lines not on this page · raw message
arch/powerpc/include/asm/time.h +2 -0
@@ -131,6 +131,8 @@ DECLARE_PER_CPU(u64, decrementers_next_tb);/* Convert timebase ticks to nanoseconds */ unsigned long long tb_to_ns (unsigned long long tb_ticks); +void timer_broadcast_interrupt (void ); +/* SPLPAR */ void accumulate_stolen_time (void );
arch/powerpc/kernel/dbell.c +4 -8
@@ -12,14 +12,14 @@#include <linux/hardirq.h> #include <asm/dbell.h> +#include <asm/interrupt.h> #include <asm/irq_regs.h> #include <asm/kvm_ppc.h> #include <asm/trace.h> -#ifdef CONFIG_SMP - -void doorbell_exception (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC (doorbell_exception) { +#ifdef CONFIG_SMP struct pt_regs *old_regs =set_irq_regs (regs);irq_enter ();
@@ -37,11 +37,7 @@ void doorbell_exception(struct pt_regs *regs)trace_doorbell_exit (regs);irq_exit ();set_irq_regs (old_regs); -}#else /* CONFIG_SMP */ -void doorbell_exception (struct pt_regs *regs) -{printk (KERN_WARNING "Received doorbell on non-smp system\n" ); -}#endif /* CONFIG_SMP */ - +}
arch/powerpc/kernel/exceptions-64s.S +3 -4
@@ -1923,7 +1923,7 @@ EXC_COMMON_BEGIN(doorbell_super_common) #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return
@@ -2136,8 +2136,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - li r4,SIGSEGV - bl bad_page_fault + bl do_bad_page_fault MMU_FTR_SECTION_ELSE bl unknown_exception ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
@@ -2310,7 +2309,7 @@ EXC_COMMON_BEGIN(h_doorbell_common) #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return
arch/powerpc/kernel/head_book3s_32.S +3 -3
@@ -240,8 +240,8 @@ __secondary_hold_acknowledge: /* System reset */ /* core99 pmac starts the seconary here by changing the vector, and - putting it back to what it was (unknown_exception) when done. */ - EXCEPTION(0x100, Reset, unknown_exception, EXC_XFER_STD) + putting it back to what it was (unknown_async_exception) when done. */ + EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD) /* Machine check */ /*
@@ -642,7 +642,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) #endif #ifndef CONFIG_TAU_INT -#define TAUException unknown_exception +#define TAUException unknown_async_exception #endif EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
arch/powerpc/kernel/irq.c +2 -1
@@ -54,6 +54,7 @@#include <linux/pgtable.h> #include <linux/uaccess.h> +#include <asm/interrupt.h> #include <asm/io.h> #include <asm/irq.h> #include <asm/cache.h>
@@ -665,7 +666,7 @@ void __do_irq(struct pt_regs *regs)irq_exit (); } -void do_IRQ (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC (do_IRQ) {struct pt_regs *old_regs =set_irq_regs (regs);void *cursp, *irqsp, *sirqsp;
arch/powerpc/kernel/mce.c +3 -2
@@ -18,6 +18,7 @@#include <linux/extable.h> #include <linux/ftrace.h> +#include <asm/interrupt.h> #include <asm/machdep.h> #include <asm/mce.h> #include <asm/nmi.h>
@@ -588,7 +589,7 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info);* * regs->nip and regs->msr contains srr0 and ssr1. */ -long notrace machine_check_early (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI (machine_check_early) {long handled =0 ;u8 ftrace_enabled =this_cpu_get_ftrace_enabled ();
@@ -722,7 +723,7 @@ long hmi_handle_debugtrig(struct pt_regs *regs)/* * Return values: */ -long hmi_exception_realmode (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI (hmi_exception_realmode) {int ret;
arch/powerpc/kernel/syscall_64.c +1 -0
@@ -5,6 +5,7 @@#include <asm/kup.h> #include <asm/cputime.h> #include <asm/hw_irq.h> +#include <asm/interrupt.h> #include <asm/kprobes.h> #include <asm/paca.h> #include <asm/ptrace.h>
arch/powerpc/kernel/tau_6xx.c +1 -1
@@ -100,7 +100,7 @@ static void TAUupdate(int cpu)* with interrupts disabled */ -void TAUException (struct pt_regs * regs) +DEFINE_INTERRUPT_HANDLER_ASYNC (TAUException) {int cpu =smp_processor_id ();
arch/powerpc/kernel/time.c +2 -1
@@ -56,6 +56,7 @@#include <linux/processor.h> #include <asm/trace.h> +#include <asm/interrupt.h> #include <asm/io.h> #include <asm/nvram.h> #include <asm/cache.h>
@@ -545,7 +546,7 @@ void arch_irq_work_raise(void)* timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. */ -void timer_interrupt (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC (timer_interrupt) {struct clock_event_device *evt =this_cpu_ptr (&decrementers);u64 *next_tb =this_cpu_ptr (&decrementers_next_tb);
arch/powerpc/kernel/traps.c +62 -28
277 lines not on this page · raw message
arch/powerpc/kernel/watchdog.c +5 -2
@@ -26,6 +26,7 @@#include <linux/delay.h> #include <linux/smp.h> +#include <asm/interrupt.h> #include <asm/paca.h> /*
@@ -247,14 +248,14 @@ static void watchdog_timer_interrupt(int cpu)watchdog_smp_panic (cpu, tb); } -void soft_nmi_interrupt (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI (soft_nmi_interrupt) {unsigned long flags;int cpu =raw_smp_processor_id ();u64 tb;if (!cpumask_test_cpu (cpu, &wd_cpus_enabled)) -return ; +return 0 ;nmi_enter ();
@@ -291,6 +292,8 @@ void soft_nmi_interrupt(struct pt_regs *regs) out:nmi_exit (); + +return 0 ; }static enum hrtimer_restart watchdog_timer_fn (struct hrtimer *hrtimer)
arch/powerpc/kvm/book3s_hv.c +1 -0
@@ -53,6 +53,7 @@#include <asm/cputable.h> #include <asm/cacheflush.h> #include <linux/uaccess.h> +#include <asm/interrupt.h> #include <asm/io.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h>
arch/powerpc/kvm/book3s_hv_builtin.c +1 -0
@@ -17,6 +17,7 @@#include <asm/asm-prototypes.h> #include <asm/cputable.h> +#include <asm/interrupt.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> #include <asm/archrandom.h>
arch/powerpc/kvm/booke.c +1 -0
@@ -20,6 +20,7 @@#include <asm/cputable.h> #include <linux/uaccess.h> +#include <asm/interrupt.h> #include <asm/kvm_ppc.h> #include <asm/cacheflush.h> #include <asm/dbell.h>
arch/powerpc/mm/book3s64/hash_utils.c +2 -1
@@ -38,6 +38,7 @@#include <linux/pgtable.h> #include <asm/debugfs.h> +#include <asm/interrupt.h> #include <asm/processor.h> #include <asm/mmu.h> #include <asm/mmu_context.h>
@@ -1510,7 +1511,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, }EXPORT_SYMBOL_GPL (hash_page); -long do_hash_fault (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_RET (do_hash_fault) {unsigned long ea = regs->dar;unsigned long dsisr = regs->dsisr;
arch/powerpc/mm/book3s64/slb.c +3 -2
@@ -10,6 +10,7 @@*/ #include <asm/asm-prototypes.h> +#include <asm/interrupt.h> #include <asm/mmu.h> #include <asm/mmu_context.h> #include <asm/paca.h>
@@ -837,7 +838,7 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)return slb_insert_entry (ea, context, flags, ssize, false); } -long do_slb_fault (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_RAW (do_slb_fault) {unsigned long ea = regs->dar;unsigned long id =get_region_id (ea);
@@ -890,7 +891,7 @@ long do_slb_fault(struct pt_regs *regs) } } -void do_bad_slb_fault (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER (do_bad_slb_fault) {int err = regs->result;
arch/powerpc/mm/fault.c +9 -1
@@ -34,6 +34,7 @@#include <linux/uaccess.h> #include <asm/firmware.h> +#include <asm/interrupt.h> #include <asm/page.h> #include <asm/mmu.h> #include <asm/mmu_context.h>
@@ -549,7 +550,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, }NOKPROBE_SYMBOL (__do_page_fault); -long do_page_fault (struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_RET (do_page_fault) {enum ctx_state prev_state =exception_enter ();unsigned long address = regs->dar;
@@ -656,3 +657,10 @@ void bad_page_fault(struct pt_regs *regs, int sig)die ("Kernel access of bad area" , regs, sig); } + +#ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER (do_bad_page_fault) +{ +bad_page_fault (regs,SIGSEGV ); +} +#endif
arch/powerpc/platforms/powernv/idle.c +1 -0
@@ -14,6 +14,7 @@#include <asm/asm-prototypes.h> #include <asm/firmware.h> +#include <asm/interrupt.h> #include <asm/machdep.h> #include <asm/opal.h> #include <asm/cputhreads.h>
Signature · 1 line
2.23.0
Re: [PATCH v3 05/19] powerpc: interrupt handler wrapper functions
Nicholas Piggin <[email protected]> writes:
.....
+#endif
arch/powerpc/include/asm/interrupt.h line 188
Quoted 2 lines
arch/powerpc/include/asm/interrupt.h new file +2
-0+DECLARE_INTERRUPT_HANDLER (emulation_assist_interrupt); +DECLARE_INTERRUPT_HANDLER_RAW (do_slb_fault);
Can we add comments here explaining why some of these handlers need to remain RAW()?
arch/powerpc/include/asm/interrupt.h line 197
Quoted 9 lines
arch/powerpc/include/asm/interrupt.h new file +9
-0+DECLARE_INTERRUPT_HANDLER (do_bad_slb_fault); +DECLARE_INTERRUPT_HANDLER_RET (do_hash_fault); +DECLARE_INTERRUPT_HANDLER_RET (do_page_fault); +DECLARE_INTERRUPT_HANDLER (do_bad_page_fault); + +DECLARE_INTERRUPT_HANDLER_ASYNC (timer_interrupt); +DECLARE_INTERRUPT_HANDLER_NMI (performance_monitor_exception_nmi); +DECLARE_INTERRUPT_HANDLER_ASYNC (performance_monitor_exception_async); +DECLARE_INTERRUPT_HANDLER_RAW (performance_monitor_exception);
Same for this.
Quoted 14 lines
arch/powerpc/include/asm/interrupt.h new file +8
-0+DECLARE_INTERRUPT_HANDLER (WatchdogException); +DECLARE_INTERRUPT_HANDLER (unknown_exception); +DECLARE_INTERRUPT_HANDLER_ASYNC (unknown_async_exception); + +void replay_system_reset (void ); +void replay_soft_interrupts (void ); + +#endif /* _ASM_POWERPC_INTERRUPT_H */ arch/powerpc/include/asm/time.h +0
-0
arch/powerpc/include/asm/time.h DECLARE_PER_CPU(u64, decrementers_next_tb); · line 138
Quoted 8 lines
arch/powerpc/include/asm/time.h +2
-0@@ -131,6 +131,8 @@ DECLARE_PER_CPU(u64, decrementers_next_tb);/* Convert timebase ticks to nanoseconds */ unsigned long long tb_to_ns (unsigned long long tb_ticks); +void timer_broadcast_interrupt (void ); +/* SPLPAR */ void accumulate_stolen_time (void );
-aneesh
Nicholas Piggin
reviewerRe: [PATCH v3 05/19] powerpc: interrupt handler wrapper functions
Excerpts from Aneesh Kumar K.V's message of November 30, 2020 5:37 pm:
Quoted 8 lines
Nicholas Piggin <[email protected]> writes:
.....
+#endifarch/powerpc/include/asm/interrupt.h line 188
Quoted 2 lines
arch/powerpc/include/asm/interrupt.h new file +2
-0+DECLARE_INTERRUPT_HANDLER (emulation_assist_interrupt); +DECLARE_INTERRUPT_HANDLER_RAW (do_slb_fault);Can we add comments here explaining why some of these handlers need to remain
RAW()?
I possibly could. My patch doesn't change the reason, of course, they already have these issues.
It wants to avoid reconciling interrupts and probably context tracking because you can take SLB faults within those subsystems, which are not expecting re-entrant calls into them. It's okay to avoid these things because the interrupts don't enable interrupts, go to process context, add any timers, etc.
Now that I look at it, possibly the primary do_hash_fault handler needs to be RAW as well for the same reason.
Quoted 11 lines
arch/powerpc/include/asm/interrupt.h line 197
Quoted 9 lines
arch/powerpc/include/asm/interrupt.h new file +9
-0+DECLARE_INTERRUPT_HANDLER (do_bad_slb_fault); +DECLARE_INTERRUPT_HANDLER_RET (do_hash_fault); +DECLARE_INTERRUPT_HANDLER_RET (do_page_fault); +DECLARE_INTERRUPT_HANDLER (do_bad_page_fault); + +DECLARE_INTERRUPT_HANDLER_ASYNC (timer_interrupt); +DECLARE_INTERRUPT_HANDLER_NMI (performance_monitor_exception_nmi); +DECLARE_INTERRUPT_HANDLER_ASYNC (performance_monitor_exception_async); +DECLARE_INTERRUPT_HANDLER_RAW (performance_monitor_exception);Same for this.
That's just because nmi vs async is decided at runtime for PMIs. I can add a comment, although it's more obvious when looking at the body.
Thanks,
Nick
Nicholas Piggin
reviewerPATCH v3 6/19 powerpc: add interrupt wrapper entry / exit stub functions
In reply to Nicholas PigginThese will be used by subsequent patches.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+)
arch/powerpc/include/asm/interrupt.h +66 -0
@@ -5,6 +5,50 @@#include <linux/context_tracking.h> #include <asm/ftrace.h> +struct interrupt_state { +}; + +static inline void interrupt_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) +{ +} + +/* +* Care should be taken to note that interrupt_exit_prepare and +* interrupt_async_exit_prepare do not necessarily return immediately to +* regs context (e.g., if regs is usermode, we don't necessarily return to +* user mode). Other interrupts might be taken between here and return, +* context switch / preemption may occur in the exit path after this, or a +* signal may be delivered, etc. +* +* The real interrupt exit code is platform specific, e.g., +* interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. +* +* However interrupt_nmi_exit_prepare does return directly to regs, because +* NMIs do not do "exit work" or replay soft-masked interrupts. +*/ +static inline void interrupt_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) +{ +} + +static inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) +{ +} + +static inline void interrupt_async_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) +{ +} + +struct interrupt_nmi_state { +}; + +static inline void interrupt_nmi_enter_prepare (struct pt_regs *regs,struct interrupt_nmi_state *state) +{ +} + +static inline void interrupt_nmi_exit_prepare (struct pt_regs *regs,struct interrupt_nmi_state *state) +{ +} +/** * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function * @func: Function name of the entry point
@@ -59,7 +103,13 @@ static __always_inline void ___##func(struct pt_regs *regs); \ \ __visible noinstr void func(struct pt_regs *regs) \ { \ +struct interrupt_state state; \ + \ +interrupt_enter_prepare (regs, &state); \ + \ ___##func (regs); \ + \ + interrupt_exit_prepare(regs, &state); \ } \ \ static __always_inline void ___##func(struct pt_regs *regs)
@@ -87,10 +137,15 @@ static __always_inline long ___##func(struct pt_regs *regs); \ \ __visible noinstr long func(struct pt_regs *regs) \ { \ +struct interrupt_state state; \long ret; \ \ +interrupt_enter_prepare (regs, &state); \ + \ ret = ___##func (regs); \ \ +interrupt_exit_prepare (regs, &state); \ + \return ret; \ } \ \
@@ -117,7 +172,13 @@ static __always_inline void ___##func(struct pt_regs *regs); \ \ __visible noinstr void func(struct pt_regs *regs) \ { \ +struct interrupt_state state; \ + \ +interrupt_async_enter_prepare (regs, &state); \ + \ ___##func (regs); \ + \ + interrupt_async_exit_prepare(regs, &state); \ } \ \ static __always_inline void ___##func(struct pt_regs *regs)
@@ -145,10 +206,15 @@ static __always_inline long ___##func(struct pt_regs *regs); \ \ __visible noinstr long func(struct pt_regs *regs) \ { \ +struct interrupt_nmi_state state; \long ret; \ \ +interrupt_nmi_enter_prepare (regs, &state); \ + \ ret = ___##func (regs); \ \ +interrupt_nmi_exit_prepare (regs, &state); \ + \return ret; \ } \ \
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 7/19 powerpc: add interrupt_cond_local_irq_enable helper
In reply to Nicholas PigginSimple helper for synchronous interrupt handlers to use to enable interrupts if they were taken in interrupt-enabled context.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 7 +++++++ arch/powerpc/kernel/traps.c | 24 +++++++----------------- arch/powerpc/mm/fault.c | 4 +--- 3 files changed, 15 insertions(+), 20 deletions(-)
arch/powerpc/include/asm/interrupt.h +7 -0
@@ -3,6 +3,7 @@#define _ASM_POWERPC_INTERRUPT_H#include <linux/context_tracking.h> +#include <linux/hardirq.h> #include <asm/ftrace.h> struct interrupt_state {
@@ -268,4 +269,10 @@ DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);void replay_system_reset (void );void replay_soft_interrupts (void ); +static inline void interrupt_cond_local_irq_enable (struct pt_regs *regs) +{ +if (!arch_irq_disabled_regs (regs)) +local_irq_enable (); +} +#endif /* _ASM_POWERPC_INTERRUPT_H */
arch/powerpc/kernel/traps.c +7 -17
@@ -343,8 +343,8 @@ static bool exception_common(int signr, struct pt_regs *regs, int code,show_signal_msg (signr, regs, code, addr); -if (arch_irqs_disabled () && !arch_irq_disabled_regs (regs)) -local_irq_enable (); +if (arch_irqs_disabled ()) +interrupt_cond_local_irq_enable (regs); current->thread.trap_nr = code;
@@ -1575,9 +1575,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)if (!user_mode (regs)) goto sigill; -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs);/* (reason & REASON_ILLEGAL) would be the obvious thing here, * but there seems to be a hardware bug on the 405GP (RevD)
@@ -1631,9 +1629,7 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)int sig, code, fixed =0 ;unsigned long reason; -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs); reason =get_reason (regs);
@@ -1794,9 +1790,7 @@ DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception)die ("Unexpected facility unavailable exception" , regs,SIGABRT ); } -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs);if (status ==FSCR_DSCR_LG ) {/*
@@ -2176,9 +2170,7 @@ void SPEFloatingPointException(struct pt_regs *regs)int code =FPE_FLTUNK ;int err; -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs);flush_spe_to_thread (current);
@@ -2225,9 +2217,7 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)extern int speround_handler (struct pt_regs *regs);int err; -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs);preempt_disable ();if (regs->msr &MSR_SPE )
arch/powerpc/mm/fault.c +1 -3
@@ -443,9 +443,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,return bad_area_nosemaphore (regs, address); } -/* We restore the interrupt state now */ -if (!arch_irq_disabled_regs (regs)) -local_irq_enable (); +interrupt_cond_local_irq_enable (regs);perf_sw_event (PERF_COUNT_SW_PAGE_FAULTS ,1 , regs, address);
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 8/19 powerpc/64: context tracking remove _TIF_NOHZ
In reply to Nicholas PigginAdd context tracking to the system call handler explicitly, and remove _TIF_NOHZ.
This saves 35 cycles on gettid system call cost on POWER9 with a CONFIG_NOHZ_FULL kernel.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/Kconfig | 6 ------ arch/powerpc/Kconfig | 1 - arch/powerpc/include/asm/thread_info.h | 4 +--- arch/powerpc/kernel/ptrace/ptrace.c | 4 ---- arch/powerpc/kernel/signal.c | 4 ---- arch/powerpc/kernel/syscall_64.c | 10 ++++++++++ 6 files changed, 11 insertions(+), 18 deletions(-)
arch/Kconfig +0 -6
@@ -618,12 +618,6 @@ config HAVE_CONTEXT_TRACKING protected inside rcu_irq_enter/rcu_irq_exit() but preemption or signal handling on irq exit still need to be protected. -config HAVE_TIF_NOHZ - bool - help - Arch relies on TIF_NOHZ and syscall slow path to implement context - tracking calls to user_enter()/user_exit(). - config HAVE_VIRT_CPU_ACCOUNTING bool
arch/powerpc/Kconfig +0 -1
@@ -193,7 +193,6 @@ config PPC select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_CONTEXT_TRACKING if PPC64 - select HAVE_TIF_NOHZ if PPC64 select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW select HAVE_DYNAMIC_FTRACE
arch/powerpc/include/asm/thread_info.h +1 -3
@@ -95,7 +95,6 @@ void arch_setup_new_exec(void);#define TIF_PATCH_PENDING 6/* pending live patching update */ #define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SINGLESTEP 8/* singlestepping active */ -#define TIF_NOHZ 9/* in adaptive nohz mode */ #define TIF_SECCOMP 10/* secure computing */ #define TIF_RESTOREALL 11/* Restore all regs (implies NOERROR) */ #define TIF_NOERROR 12/* Force successful syscall return */
@@ -128,11 +127,10 @@ void arch_setup_new_exec(void);#define _TIF_UPROBE (1<<TIF_UPROBE)#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)#define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE) -#define _TIF_NOHZ (1<<TIF_NOHZ)#define _TIF_SYSCALL_EMU (1<<TIF_SYSCALL_EMU)#define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \ - _TIF_NOHZ | _TIF_SYSCALL_EMU) + _TIF_SYSCALL_EMU)#define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
arch/powerpc/kernel/ptrace/ptrace.c +0 -4
@@ -290,8 +290,6 @@ long do_syscall_trace_enter(struct pt_regs *regs) {u32 flags; -user_exit (); - flags =READ_ONCE (current_thread_info ()->flags) & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE);
@@ -368,8 +366,6 @@ void do_syscall_trace_leave(struct pt_regs *regs) step =test_thread_flag (TIF_SINGLESTEP );if (step ||test_thread_flag (TIF_SYSCALL_TRACE ))tracehook_report_syscall_exit (regs, step); - -user_enter (); }void __init pt_regs_check (void );
arch/powerpc/kernel/signal.c +0 -4
@@ -310,8 +310,6 @@ static void do_signal(struct task_struct *tsk)void do_notify_resume (struct pt_regs *regs,unsigned long thread_info_flags) { -user_exit (); -if (thread_info_flags & _TIF_UPROBE)uprobe_notify_resume (regs);
@@ -327,8 +325,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); } - -user_enter (); }unsigned long get_tm_stackpointer (struct task_struct *tsk)
arch/powerpc/kernel/syscall_64.c +10 -0
@@ -1,9 +1,11 @@// SPDX-License-Identifier: GPL-2.0-or-later +#include <linux/context_tracking.h> #include <linux/err.h> #include <asm/asm-prototypes.h> #include <asm/kup.h> #include <asm/cputime.h> +#include <asm/interrupt.h> #include <asm/hw_irq.h> #include <asm/interrupt.h> #include <asm/kprobes.h>
@@ -28,6 +30,9 @@ notrace long system_call_exception(long r3, long r4, long r5,if (IS_ENABLED (CONFIG_PPC_IRQ_SOFT_MASK_DEBUG ))BUG_ON (irq_soft_mask_return () !=IRQS_ALL_DISABLED ); +CT_WARN_ON (ct_state () ==CONTEXT_KERNEL ); +user_exit_irqoff (); +trace_hardirqs_off ();/* finish reconciling */ if (IS_ENABLED (CONFIG_PPC_BOOK3S ))
@@ -158,6 +163,8 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,unsigned long ti_flags;unsigned long ret =0 ; +CT_WARN_ON (ct_state () ==CONTEXT_USER ); +kuap_check_amr (); regs->result = r3;
@@ -234,8 +241,11 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, } } +user_enter_irqoff (); +/* scv need not set RI=0 because SRRs are not used */ if (unlikely (!prep_irq_for_enabled_exit (!scv))) { +user_exit_irqoff ();local_irq_enable (); goto again; }
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 9/19 powerpc/64: context tracking move to interrupt wrappers
In reply to Nicholas PigginThis moves exception_enter/exit calls to wrapper functions for synchronous interrupts. More interrupt handlers are covered by this than previously.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 9 ++++ arch/powerpc/kernel/traps.c | 74 ++++++--------------------- arch/powerpc/mm/book3s64/hash_utils.c | 2 - arch/powerpc/mm/fault.c | 3 -- 4 files changed, 26 insertions(+), 62 deletions(-)
arch/powerpc/include/asm/interrupt.h +9 -0
@@ -7,10 +7,16 @@#include <asm/ftrace.h> struct interrupt_state { +#ifdef CONFIG_PPC64 +enum ctx_state ctx_state; +#endif };static inline void interrupt_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) { +#ifdef CONFIG_PPC64 + state->ctx_state =exception_enter (); +#endif }/*
@@ -29,6 +35,9 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup*/ static inline void interrupt_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) { +#ifdef CONFIG_PPC64 +exception_exit (state->ctx_state); +#endif }static inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state)
arch/powerpc/kernel/traps.c +17 -57
230 lines not on this page · raw message
arch/powerpc/mm/book3s64/hash_utils.c +0 -2
@@ -1289,7 +1289,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long flags) {bool is_thp; -enum ctx_state prev_state =exception_enter ();pgd_t *pgdir;unsigned long vsid;pte_t *ptep;
@@ -1489,7 +1488,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,DBG_LOW (" -> rc=%d\n" , rc); bail: -exception_exit (prev_state);return rc; }EXPORT_SYMBOL_GPL (hash_page_mm);
arch/powerpc/mm/fault.c +0 -3
@@ -550,7 +550,6 @@ NOKPROBE_SYMBOL(__do_page_fault);DEFINE_INTERRUPT_HANDLER_RET (do_page_fault) { -enum ctx_state prev_state =exception_enter ();unsigned long address = regs->dar;unsigned long error_code = regs->dsisr;long err;
@@ -573,8 +572,6 @@ DEFINE_INTERRUPT_HANDLER_RET(do_page_fault) }#endif -exception_exit (prev_state); -return err; }NOKPROBE_SYMBOL (do_page_fault);
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 10/19 powerpc/64: add context tracking to asynchronous interrupts
In reply to Nicholas PigginPreviously context tracking was not done for asynchronous interrupts, (those that run in interrupt context), and if those would cause a reschedule when they exit, then scheduling functions (schedule_user, preempt_schedule_irq) call exception_enter/exit to fix this up and exit user context.
This is a hack we would like to get away from, so do context tracking for asynchronous interrupts too.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 2 ++ 1 file changed, 2 insertions(+)
arch/powerpc/include/asm/interrupt.h +2 -0
@@ -42,10 +42,12 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interruptstatic inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) { +interrupt_enter_prepare (regs, state); }static inline void interrupt_async_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) { +interrupt_exit_prepare (regs, state); }struct interrupt_nmi_state {
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 11/19 powerpc: handle irq_enter/irq_exit in interrupt handler wrappers
In reply to Nicholas PigginMove irq_enter/irq_exit into asynchronous interrupt handler wrappers.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 2 ++ arch/powerpc/kernel/dbell.c | 3 +-- arch/powerpc/kernel/irq.c | 4 ---- arch/powerpc/kernel/tau_6xx.c | 3 --- arch/powerpc/kernel/time.c | 4 ++-- arch/powerpc/kernel/traps.c | 6 ------ 6 files changed, 5 insertions(+), 17 deletions(-)
arch/powerpc/include/asm/interrupt.h +2 -0
@@ -43,10 +43,12 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interruptstatic inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) {interrupt_enter_prepare (regs, state); +irq_enter (); }static inline void interrupt_async_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) { +irq_exit ();interrupt_exit_prepare (regs, state); }
arch/powerpc/kernel/dbell.c +1 -2
@@ -22,7 +22,6 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception)#ifdef CONFIG_SMP struct pt_regs *old_regs =set_irq_regs (regs); -irq_enter ();trace_doorbell_entry (regs);ppc_msgsync ();
@@ -35,7 +34,7 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception)smp_ipi_demux_relaxed ();/* already performed the barrier */ trace_doorbell_exit (regs); -irq_exit (); +set_irq_regs (old_regs);#else /* CONFIG_SMP */ printk (KERN_WARNING "Received doorbell on non-smp system\n" );
arch/powerpc/kernel/irq.c +0 -4
@@ -641,8 +641,6 @@ void __do_irq(struct pt_regs *regs) {unsigned int irq; -irq_enter (); -trace_irq_entry (regs);/*
@@ -662,8 +660,6 @@ void __do_irq(struct pt_regs *regs)generic_handle_irq (irq);trace_irq_exit (regs); - -irq_exit (); }DEFINE_INTERRUPT_HANDLER_ASYNC (do_IRQ)
arch/powerpc/kernel/tau_6xx.c +0 -3
@@ -104,12 +104,9 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(TAUException) {int cpu =smp_processor_id (); -irq_enter (); tau[cpu].interrupts++;TAUupdate (cpu); - -irq_exit (); }#endif /* CONFIG_TAU_INT */
arch/powerpc/kernel/time.c +2 -2
@@ -585,7 +585,7 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)#endif old_regs =set_irq_regs (regs); -irq_enter (); +trace_timer_interrupt_entry (regs);if (test_irq_work_pending ()) {
@@ -610,7 +610,7 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) }trace_timer_interrupt_exit (regs); -irq_exit (); +set_irq_regs (old_regs); }EXPORT_SYMBOL (timer_interrupt);
arch/powerpc/kernel/traps.c +0 -6
@@ -1082,7 +1082,6 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception)struct pt_regs *old_regs; old_regs =set_irq_regs (regs); -irq_enter ();#ifdef CONFIG_VSX /* Real mode flagged P9 special emu is needed */
@@ -1102,7 +1101,6 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception)if (ppc_md.handle_hmi_exception) ppc_md.handle_hmi_exception (regs); -irq_exit ();set_irq_regs (old_regs); }
@@ -1918,13 +1916,9 @@ DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi)DEFINE_INTERRUPT_HANDLER_ASYNC (performance_monitor_exception_async) { -irq_enter (); -__this_cpu_inc (irq_stat.pmu_irqs);perf_irq (regs); - -irq_exit (); }DEFINE_INTERRUPT_HANDLER_RAW (performance_monitor_exception)
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 12/19 powerpc/64s: move context tracking exit to interrupt exit path
In reply to Nicholas PigginThe interrupt handler wrapper functions are not the ideal place to maintain context tracking because after they return, the low level exit code must then determine if there are interrupts to replay, or if the task should be preempted, etc. Those paths (e.g., schedule_user) include their own exception_enter/exit pairs to fix this up but it's a bit hacky (see schedule_user() comments).
Ideally context tracking will go to user mode only when there are no more interrupts or context switches or other exit processing work to handle.
64e can not do this because it does not use the C interrupt exit code.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 34 +++++++++++++++++++++++++--- arch/powerpc/kernel/syscall_64.c | 9 ++++++++ 2 files changed, 40 insertions(+), 3 deletions(-)
arch/powerpc/include/asm/interrupt.h +31 -3
@@ -7,16 +7,30 @@#include <asm/ftrace.h> struct interrupt_state { -#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3E_64 enum ctx_state ctx_state;#endif };static inline void interrupt_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) { -#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3E_64 state->ctx_state =exception_enter ();#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +if (user_mode (regs)) { +CT_WARN_ON (ct_state () !=CONTEXT_USER ); +user_exit_irqoff (); + }else { +/* +* CT_WARN_ON comes here via program_check_exception, +* so avoid recursion. +*/ +if (TRAP (regs) !=0x700 ) +CT_WARN_ON (ct_state () !=CONTEXT_KERNEL ); + } +#endif }/*
@@ -35,9 +49,23 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup*/ static inline void interrupt_exit_prepare (struct pt_regs *regs,struct interrupt_state *state) { -#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3E_64 exception_exit (state->ctx_state);#endif + +/* +* Book3S exits to user via interrupt_exit_user_prepare(), which does +* context tracking, which is a cleaner way to handle PREEMPT=y +* and avoid context entry/exit in e.g., preempt_schedule_irq()), +* which is likely to be where the core code wants to end up. +* +* The above comment explains why we can't do the +* +* if (user_mode(regs)) +* user_exit_irqoff(); +* +* sequence here. +*/ }static inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state)
arch/powerpc/kernel/syscall_64.c +9 -0
@@ -275,6 +275,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsignedBUG_ON (!(regs->msr &MSR_PR ));BUG_ON (!FULL_REGS (regs));BUG_ON (regs->softe !=IRQS_ENABLED ); +CT_WARN_ON (ct_state () ==CONTEXT_USER );/* * We don't need to restore AMR on the way back to userspace for KUAP.
@@ -317,7 +318,9 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned } } +user_enter_irqoff ();if (unlikely (!prep_irq_for_enabled_exit (true))) { +user_exit_irqoff ();local_irq_enable ();local_irq_disable (); goto again;
@@ -358,6 +361,12 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsignunrecoverable_exception (regs);BUG_ON (regs->msr &MSR_PR );BUG_ON (!FULL_REGS (regs)); +/* +* CT_WARN_ON comes here via program_check_exception, +* so avoid recursion. +*/ +if (TRAP (regs) !=0x700 ) +CT_WARN_ON (ct_state () ==CONTEXT_USER ); amr =kuap_get_and_check_amr ();
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 13/19 powerpc/64s: reconcile interrupts in C
In reply to Nicholas PigginThere is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 15 +++++++++++---- arch/powerpc/kernel/exceptions-64s.S | 26 -------------------------- 2 files changed, 11 insertions(+), 30 deletions(-)
arch/powerpc/include/asm/interrupt.h +11 -4
@@ -14,11 +14,14 @@ struct interrupt_state {static inline void interrupt_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) { -#ifdef CONFIG_PPC_BOOK3E_64 - state->ctx_state =exception_enter (); -#endif - +/* +* Book3E reconciles irq soft mask in asm +*/ #ifdef CONFIG_PPC_BOOK3S_64 +if (irq_soft_mask_set_return (IRQS_ALL_DISABLED ) ==IRQS_ENABLED ) +trace_hardirqs_off (); + local_paca->irq_happened |=PACA_IRQ_HARD_DIS ; +if (user_mode (regs)) {CT_WARN_ON (ct_state () !=CONTEXT_USER );user_exit_irqoff ();
@@ -31,6 +34,10 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup CT_WARN_ON(ct_state() != CONTEXT_KERNEL); }#endif + +#ifdef CONFIG_PPC_BOOK3E_64 + state->ctx_state =exception_enter (); +#endif }/*
arch/powerpc/kernel/exceptions-64s.S +0 -26
@@ -139,7 +139,6 @@ name: #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ #define __ISTACK(name) .L_ISTACK_ ## name -#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */ #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ #define INT_DEFINE_BEGIN(n) \
@@ -203,9 +202,6 @@ do_define_int n .ifndef ISTACK ISTACK=1 .endif - .ifndef IRECONCILE - IRECONCILE=1 - .endif .ifndef IKUAP IKUAP=1 .endif
@@ -653,10 +649,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .if ISTACK ACCOUNT_STOLEN_TIME .endif - - .if IRECONCILE - RECONCILE_IRQ_STATE(r10, r11) - .endif .endm /*
@@ -935,7 +927,6 @@ INT_DEFINE_BEGIN(system_reset) */ ISET_RI=0 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 INT_DEFINE_END(system_reset)
@@ -1123,7 +1114,6 @@ INT_DEFINE_BEGIN(machine_check_early) ISTACK=0 IDAR=1 IDSISR=1 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ INT_DEFINE_END(machine_check_early)
@@ -1476,7 +1466,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) INT_DEFINE_BEGIN(data_access_slb) IVEC=0x380 IAREA=PACA_EXSLB - IRECONCILE=0 IDAR=1 IKVM_SKIP=1 IKVM_REAL=1
@@ -1503,7 +1492,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return
@@ -1565,7 +1553,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) INT_DEFINE_BEGIN(instruction_access_slb) IVEC=0x480 IAREA=PACA_EXSLB - IRECONCILE=0 IISIDE=1 IDAR=1 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
@@ -1594,7 +1581,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return
@@ -1754,7 +1740,6 @@ EXC_COMMON_BEGIN(program_check_common) */ INT_DEFINE_BEGIN(fp_unavailable) IVEC=0x800 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif
@@ -1769,7 +1754,6 @@ EXC_VIRT_END(fp_unavailable, 0x4800, 0x100) EXC_COMMON_BEGIN(fp_unavailable_common) GEN_COMMON fp_unavailable bne 1f /* if from user, just load it up */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl kernel_fp_unavailable_exception 0: trap
@@ -1788,7 +1772,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl fp_unavailable_tm b interrupt_return
@@ -1853,7 +1836,6 @@ INT_DEFINE_BEGIN(hdecrementer) IVEC=0x980 IHSRR=1 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 IKVM_VIRT=1 INT_DEFINE_END(hdecrementer)
@@ -2227,7 +2209,6 @@ INT_DEFINE_BEGIN(hmi_exception_early) IHSRR=1 IREALMODE_COMMON=1 ISTACK=0 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ IKVM_REAL=1 INT_DEFINE_END(hmi_exception_early)
@@ -2401,7 +2382,6 @@ EXC_COMMON_BEGIN(performance_monitor_common) */ INT_DEFINE_BEGIN(altivec_unavailable) IVEC=0xf20 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif
@@ -2431,7 +2411,6 @@ BEGIN_FTR_SECTION b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_tm b interrupt_return
@@ -2439,7 +2418,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_exception b interrupt_return
@@ -2455,7 +2433,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) */ INT_DEFINE_BEGIN(vsx_unavailable) IVEC=0xf40 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif
@@ -2484,7 +2461,6 @@ BEGIN_FTR_SECTION b load_up_vsx #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_tm b interrupt_return
@@ -2492,7 +2468,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_VSX) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_exception b interrupt_return
@@ -2827,7 +2802,6 @@ EXC_VIRT_NONE(0x5800, 0x100) INT_DEFINE_BEGIN(soft_nmi) IVEC=0x900 ISTACK=0 - IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */ INT_DEFINE_END(soft_nmi) /*
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 14/19 powerpc/64: move account_stolen_time into its own function
In reply to Nicholas PigginThis will be used by interrupt entry as well.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/cputime.h | 15 +++++++++++++++ arch/powerpc/kernel/syscall_64.c | 10 +--------- 2 files changed, 16 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/cputime.h +15 -0
@@ -87,6 +87,18 @@ static notrace inline void account_cpu_user_exit(void) acct->starttime_user = tb; } +static notrace inline void account_stolen_time(void) +{ +#ifdef CONFIG_PPC_SPLPAR +if (IS_ENABLED (CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) && +firmware_has_feature (FW_FEATURE_SPLPAR )) { +struct lppaca *lp = local_paca->lppaca_ptr; + +if (unlikely (local_paca->dtl_ridx !=be64_to_cpu (lp->dtl_idx))) +accumulate_stolen_time (); + } +#endif +}#endif /* __KERNEL__ */ #else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
@@ -96,5 +108,8 @@ static inline void account_cpu_user_entry(void)static inline void account_cpu_user_exit (void ) { } +static notrace inline void account_stolen_time(void) +{ +}#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __POWERPC_CPUTIME_H */
arch/powerpc/kernel/syscall_64.c +1 -9
@@ -45,15 +45,7 @@ notrace long system_call_exception(long r3, long r4, long r5,account_cpu_user_entry (); -#ifdef CONFIG_PPC_SPLPAR -if (IS_ENABLED (CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) && -firmware_has_feature (FW_FEATURE_SPLPAR )) { -struct lppaca *lp = local_paca->lppaca_ptr; - -if (unlikely (local_paca->dtl_ridx !=be64_to_cpu (lp->dtl_idx))) -accumulate_stolen_time (); - } -#endif +account_stolen_time ();/* * This is not required for the syscall exit path, but makes the
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 15/19 powerpc/64: entry cpu time accounting in C
In reply to Nicholas PigginThere is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 7 +++++++ arch/powerpc/include/asm/ppc_asm.h | 24 ------------------------ arch/powerpc/kernel/exceptions-64e.S | 1 - arch/powerpc/kernel/exceptions-64s.S | 5 ----- 4 files changed, 7 insertions(+), 30 deletions(-)
arch/powerpc/include/asm/interrupt.h +7 -0
@@ -4,6 +4,7 @@#include <linux/context_tracking.h> #include <linux/hardirq.h> +#include <asm/cputime.h> #include <asm/ftrace.h> struct interrupt_state {
@@ -25,6 +26,9 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupif (user_mode (regs)) {CT_WARN_ON (ct_state () !=CONTEXT_USER );user_exit_irqoff (); + +account_cpu_user_entry (); +account_stolen_time (); }else {/* * CT_WARN_ON comes here via program_check_exception,
@@ -38,6 +42,9 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup#ifdef CONFIG_PPC_BOOK3E_64 state->ctx_state =exception_enter ();#endif + +if (!IS_ENABLED (CONFIG_PPC_BOOK3S_64 ) &&user_mode (regs)) +account_cpu_user_entry (); }/*
arch/powerpc/include/asm/ppc_asm.h +0 -24
@@ -25,7 +25,6 @@#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #define ACCOUNT_CPU_USER_ENTRY (ptr, ra, rb)#define ACCOUNT_CPU_USER_EXIT (ptr, ra, rb) -#define ACCOUNT_STOLEN_TIME #else #define ACCOUNT_CPU_USER_ENTRY (ptr, ra, rb) \ MFTB(ra);/* get timebase */ \
@@ -44,29 +43,6 @@PPC_LL ra,ACCOUNT_SYSTEM_TIME (ptr ); \add ra,ra,rb;/* add on to system time */ \ PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr) - -#ifdef CONFIG_PPC_SPLPAR -#define ACCOUNT_STOLEN_TIME \ -BEGIN_FW_FTR_SECTION; \ - beq 33f; \ -/* from user - see if there are any DTL entries to process */ \ -ld r10,PACALPPACAPTR (r13 );/* get ptr to VPA */ \ -ld r11,PACA_DTL_RIDX (r13 );/* get log read index */ \ -addi r10,r10,LPPACA_DTLIDX ; \ - LDX_BE r10,0,r10; /* get log write index */ \ -cmpd cr1,r11,r10; \ - beq+ cr1,33f ; \ -bl accumulate_stolen_time; \ -ld r12,_MSR (r1 ); \ - andi. r10,r12,MSR_PR ;/* Restore cr0 (coming from user) */ \ -33: \ -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) - -#else /* CONFIG_PPC_SPLPAR */ -#define ACCOUNT_STOLEN_TIME - -#endif /* CONFIG_PPC_SPLPAR */ -#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ /*
arch/powerpc/kernel/exceptions-64e.S +0 -1
@@ -398,7 +398,6 @@ exc_##n##_common: \ std r10,_NIP(r1); /* save SRR0 to stackframe */ \ std r11,_MSR(r1); /* save SRR1 to stackframe */ \ beq 2f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r13,r10,r11);/* accounting (uses cr0+eq) */ \ 2: ld r3,excf+EX_R10(r13); /* get back r10 */ \ ld r4,excf+EX_R11(r13); /* get back r11 */ \ mfspr r5,scratch; /* get back r13 */ \
arch/powerpc/kernel/exceptions-64s.S +0 -5
@@ -577,7 +577,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) kuap_save_amr_and_lock r9, r10, cr1, cr0 .endif beq 101f /* if from kernel mode */ - ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) BEGIN_FTR_SECTION ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ std r9,_PPR(r1)
@@ -645,10 +644,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r11,exception_marker@toc(r2) std r10,RESULT(r1) /* clear regs->result */ std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ - - .if ISTACK - ACCOUNT_STOLEN_TIME - .endif .endm /*
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 16/19 powerpc: move NMI entry/exit code into wrapper
In reply to Nicholas PigginThis moves the common NMI entry and exit code into the interrupt handler wrappers.
This changes the behaviour of soft-NMI (watchdog) and HMI interrupts, and also MCE interrupts on 64e, by adding missing parts of the NMI entry to them. It fixes a bug with sreset on pseries HPT guests which shouldn't call nmi_enter().
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 24 ++++++++++++++++ arch/powerpc/kernel/mce.c | 11 -------- arch/powerpc/kernel/traps.c | 42 +++++----------------------- arch/powerpc/kernel/watchdog.c | 10 +++---- 4 files changed, 35 insertions(+), 52 deletions(-)
arch/powerpc/include/asm/interrupt.h +24 -0
@@ -95,14 +95,38 @@ static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct int }struct interrupt_nmi_state { +#ifdef CONFIG_PPC64 +u8 ftrace_enabled; +#endif };static inline void interrupt_nmi_enter_prepare (struct pt_regs *regs,struct interrupt_nmi_state *state) { +#ifdef CONFIG_PPC64 + state->ftrace_enabled =this_cpu_get_ftrace_enabled (); +this_cpu_set_ftrace_enabled (0 ); +#endif + +/* +* Do not use nmi_enter() for pseries hash guest taking a real-mode +* NMI because not everything it touches is within the RMA limit. +*/ +if (!IS_ENABLED (CONFIG_PPC_BOOK3S_64 ) || + !firmware_has_feature (FW_FEATURE_LPAR ) || +radix_enabled () || (mfmsr () &MSR_DR )) +nmi_enter (); }static inline void interrupt_nmi_exit_prepare (struct pt_regs *regs,struct interrupt_nmi_state *state) { +if (!IS_ENABLED (CONFIG_PPC_BOOK3S_64 ) || + !firmware_has_feature (FW_FEATURE_LPAR ) || +radix_enabled () || (mfmsr () &MSR_DR )) +nmi_exit (); + +#ifdef CONFIG_PPC64 +this_cpu_set_ftrace_enabled (state->ftrace_enabled); +#endif }/**
arch/powerpc/kernel/mce.c +0 -11
@@ -592,12 +592,6 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info);DEFINE_INTERRUPT_HANDLER_NMI (machine_check_early) {long handled =0 ; -u8 ftrace_enabled =this_cpu_get_ftrace_enabled (); - -this_cpu_set_ftrace_enabled (0 ); -/* Do not use nmi_enter/exit for pseries hpte guest */ -if (radix_enabled () || !firmware_has_feature (FW_FEATURE_LPAR )) -nmi_enter ();hv_nmi_check_nonrecoverable (regs);
@@ -607,11 +601,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early)if (ppc_md.machine_check_early) handled = ppc_md.machine_check_early (regs); -if (radix_enabled () || !firmware_has_feature (FW_FEATURE_LPAR )) -nmi_exit (); - -this_cpu_set_ftrace_enabled (ftrace_enabled); -return handled; }
arch/powerpc/kernel/traps.c +7 -35
92 lines not on this page · raw message
arch/powerpc/kernel/watchdog.c +4 -6
@@ -254,11 +254,12 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)int cpu =raw_smp_processor_id ();u64 tb; +/* should only arrive from kernel, with irqs disabled */ +WARN_ON_ONCE (!arch_irq_disabled_regs (regs)); +if (!cpumask_test_cpu (cpu, &wd_cpus_enabled))return 0 ; -nmi_enter (); -__this_cpu_inc (irq_stat.soft_nmi_irqs); tb =get_tb ();
@@ -266,7 +267,7 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)wd_smp_lock (&flags);if (cpumask_test_cpu (cpu, &wd_smp_cpus_stuck)) {wd_smp_unlock (&flags); - goto out; +return 0 ; }set_cpu_stuck (cpu, tb);
@@ -290,9 +291,6 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)if (wd_panic_timeout_tb <0x7fffffff )mtspr (SPRN_DEC , wd_panic_timeout_tb); -out: -nmi_exit (); -return 0 ; }
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 17/19 powerpc/64s: move NMI soft-mask handling to C
In reply to Nicholas PigginSaving and restoring soft-mask state can now be done in C using the interrupt handler wrapper functions.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 26 ++++++++++++ arch/powerpc/kernel/exceptions-64s.S | 60 ---------------------------- 2 files changed, 26 insertions(+), 60 deletions(-)
arch/powerpc/include/asm/interrupt.h +26 -0
@@ -96,6 +96,10 @@ static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct intstruct interrupt_nmi_state {#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 +u8 irq_soft_mask; +u8 irq_happened; +#endif u8 ftrace_enabled;#endif };
@@ -103,6 +107,21 @@ struct interrupt_nmi_state {static inline void interrupt_nmi_enter_prepare (struct pt_regs *regs,struct interrupt_nmi_state *state) {#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 + state->irq_soft_mask = local_paca->irq_soft_mask; + state->irq_happened = local_paca->irq_happened; + +/* +* Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does +* the right thing, and set IRQ_HARD_DIS. We do not want to reconcile +* because that goes through irq tracing which we don't want in NMI. +*/ + local_paca->irq_soft_mask =IRQS_ALL_DISABLED ; + local_paca->irq_happened |=PACA_IRQ_HARD_DIS ; + +/* Don't do any per-CPU operations until interrupt state is fixed */ + state->ftrace_enabled =this_cpu_get_ftrace_enabled (); +#endif state->ftrace_enabled =this_cpu_get_ftrace_enabled ();this_cpu_set_ftrace_enabled (0 );#endif
@@ -126,6 +145,13 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter#ifdef CONFIG_PPC64 this_cpu_set_ftrace_enabled (state->ftrace_enabled); + +#ifdef CONFIG_PPC_BOOK3S_64 +/* Check we didn't change the pending interrupt mask. */ +WARN_ON_ONCE ((state->irq_happened |PACA_IRQ_HARD_DIS ) != local_paca->irq_happened); + local_paca->irq_happened = state->irq_happened; + local_paca->irq_soft_mask = state->irq_soft_mask; +#endif #endif }
arch/powerpc/kernel/exceptions-64s.S +0 -60
95 lines not on this page · raw message
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 18/19 powerpc/64s: runlatch interrupt handling in C
In reply to Nicholas PigginThere is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 7 +++++++ arch/powerpc/kernel/exceptions-64s.S | 18 ------------------ 2 files changed, 7 insertions(+), 18 deletions(-)
arch/powerpc/include/asm/interrupt.h +7 -0
@@ -6,6 +6,7 @@#include <linux/hardirq.h> #include <asm/cputime.h> #include <asm/ftrace.h> +#include <asm/runlatch.h> struct interrupt_state {#ifdef CONFIG_PPC_BOOK3E_64
@@ -84,6 +85,12 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interruptstatic inline void interrupt_async_enter_prepare (struct pt_regs *regs,struct interrupt_state *state) { +#ifdef CONFIG_PPC_BOOK3S_64 +if (cpu_has_feature (CPU_FTR_CTRL ) && + !test_thread_local_flags (_TLF_RUNLATCH)) +__ppc64_runlatch_on (); +#endif +interrupt_enter_prepare (regs, state);irq_enter (); }
arch/powerpc/kernel/exceptions-64s.S +0 -18
@@ -692,14 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r1,GPR1(r1) .endm -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - ld r3, PACA_THREAD_INFO(r13); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - /* * When the idle code in power4_idle puts the CPU into NAP mode, * it has to do so in a loop, and relies on the external interrupt
@@ -1582,7 +1574,6 @@ EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) EXC_COMMON_BEGIN(hardware_interrupt_common) GEN_COMMON hardware_interrupt FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return
@@ -1768,7 +1759,6 @@ EXC_VIRT_END(decrementer, 0x4900, 0x80) EXC_COMMON_BEGIN(decrementer_common) GEN_COMMON decrementer FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl timer_interrupt b interrupt_return
@@ -1854,7 +1844,6 @@ EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) EXC_COMMON_BEGIN(doorbell_super_common) GEN_COMMON doorbell_super FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception
@@ -2209,7 +2198,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) EXC_COMMON_BEGIN(hmi_exception_common) GEN_COMMON hmi_exception FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl handle_hmi_exception b interrupt_return
@@ -2239,7 +2227,6 @@ EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) EXC_COMMON_BEGIN(h_doorbell_common) GEN_COMMON h_doorbell FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception
@@ -2273,7 +2260,6 @@ EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) EXC_COMMON_BEGIN(h_virt_irq_common) GEN_COMMON h_virt_irq FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return
@@ -2320,7 +2306,6 @@ EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) EXC_COMMON_BEGIN(performance_monitor_common) GEN_COMMON performance_monitor FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl performance_monitor_exception b interrupt_return
@@ -3035,9 +3020,6 @@ kvmppc_skip_Hinterrupt: * come here. */ -EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) - b __ppc64_runlatch_on - USE_FIXED_SECTION(virt_trampolines) /* * All code below __end_interrupts is treated as soft-masked. If
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v3 19/19 powerpc/64s: power4 nap fixup in C
In reply to Nicholas PigginThere is no need for this to be in asm, use the new intrrupt entry wrapper.
Signed-off-by Nicholas Piggin <[email protected]>
--- arch/powerpc/include/asm/interrupt.h | 15 +++++++++ arch/powerpc/include/asm/processor.h | 1 + arch/powerpc/include/asm/thread_info.h | 6 ++++ arch/powerpc/kernel/exceptions-64s.S | 45 -------------------------- arch/powerpc/kernel/idle_book3s.S | 4 +++ 5 files changed, 26 insertions(+), 45 deletions(-)
arch/powerpc/include/asm/interrupt.h +15 -0
@@ -8,6 +8,16 @@#include <asm/ftrace.h> #include <asm/runlatch.h> +static inline void nap_adjust_return (struct pt_regs *regs) +{ +#ifdef CONFIG_PPC_970_NAP +if (unlikely (test_thread_local_flags (_TLF_NAPPING))) { +clear_thread_local_flags (_TLF_NAPPING); + regs->nip = (unsigned long )power4_idle_nap_return; + } +#endif +} +struct interrupt_state {#ifdef CONFIG_PPC_BOOK3E_64 enum ctx_state ctx_state;
@@ -99,6 +109,9 @@ static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct int {irq_exit ();interrupt_exit_prepare (regs, state); + +/* Adjust at exit so the main handler sees the true NIA */ +nap_adjust_return (regs); }struct interrupt_nmi_state {
@@ -150,6 +163,8 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter radix_enabled() || (mfmsr() & MSR_DR))nmi_exit (); +nap_adjust_return (regs); +#ifdef CONFIG_PPC64 this_cpu_set_ftrace_enabled (state->ftrace_enabled);
arch/powerpc/include/asm/processor.h +1 -0
@@ -418,6 +418,7 @@ extern unsigned long isa300_idle_stop_mayloss(unsigned long psscr_val);extern unsigned long isa206_idle_insn_mayloss (unsigned long type);#ifdef CONFIG_PPC_970_NAP extern void power4_idle_nap (void ); +void power4_idle_nap_return (void );#endif extern unsigned long cpuidle_disable;
arch/powerpc/include/asm/thread_info.h +6 -0
@@ -151,6 +151,12 @@ void arch_setup_new_exec(void);#ifndef __ASSEMBLY__ +static inline void clear_thread_local_flags (unsigned int flags) +{ +struct thread_info *ti =current_thread_info (); + ti->local_flags &= ~flags; +} +static inline bool test_thread_local_flags (unsigned int flags) {struct thread_info *ti =current_thread_info ();
arch/powerpc/kernel/exceptions-64s.S +0 -45
105 lines not on this page · raw message
arch/powerpc/kernel/idle_book3s.S +4 -0
@@ -201,4 +201,8 @@ _GLOBAL(power4_idle_nap) mtmsrd r7 isync b 1b + + .globl power4_idle_nap_return +power4_idle_nap_return: + blr #endif
Signature · 1 line
2.23.0