powerpc/64s: reimplement book3s idle code in C
Subsystems (2)
Nicholas Piggin
reviewerPATCH v4 1/1 powerpc/64s: reimplement book3s idle code in C
Reimplement Book3S idle code in C, moving POWER7/8/9 implementation speific HV idle code to the powernv platform code.
Book3S assembly stubs are kept in common code and used only to save the stack frame and non-volatile GPRs before executing architected idle instructions, and restoring the stack and reloading GPRs then returning to C after waking from idle.
The complex logic dealing with threads and subcores, locking, SPRs, HMIs, timebase resync, etc., is all done in C which makes it more maintainable.
This is not a strict translation to C code, there are some significant differences:
- Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs, but saves and restores them itself.
- The optimisation where EC=ESL=0 idle modes did not have to save GPRs or change MSR is restored, because it's now simple to do. ESL=1 sleeps that do not lose GPRs can use this optimization too.
- KVM secondary entry and cede is now more of a call/return style rather than branchy. nap_state_lost is not required because KVM always returns via NVGPR restoring path.
- P9 restores some of the PMU SPRs, but not others, and P8 only zeroes them. There are improvmets to be made to SPR save restore policies and documentation, but this first pass tries to keep things as they were.
Left to do:
- Test actual POWER7 hardware.
- KVM could use more review, it's pretty tricky. Not sure if what I'm doing with the emergency stack is kosher. But it runs pretty fine here with a POWER8 SMP+SMT guest. Possible to streamline KVM cede code now that idle function saves nv gprs for us?
Since RFC v1:
- Now tested and working with POWER9 hash and radix.
- KVM support added. This took a bit of work to untangle and might still have some issues, but POWER9 seems to work including hash on radix with dependent threads mode.
- This snowballed a bit because of KVM and other details making it not feasible to leave POWER7/8 code alone. That's only half done at the moment.
- So far this trades about 800 lines of asm for 500 of C. With POWER7/8 support done it might be another hundred or so lines of C.
Since RFC v2:
- Fixed deep state SLB reloading
- Now tested and working with POWER8.
- Accounted for most feedback.
Since RFC v3:
- Rebased to powerpc merge + idle state bugfix
- Split SLB flush/restore code out and shared with MCE code (pseries MCE patches can also use).
- More testing on POWER8 including KVM with secondaries.
- Performance testing looks good. EC=ESL=0 is about 5% faster, other stop states look a little faster too.
- Adjusted SPR saving to handler POWER7, haven't tested it.
Since submission v1:
- More review comments from Gautham.
- Rename isa3_ to isa300_ prefix.
- Tinkered with some comments, copyright notice, changelog.
- Cede and regular idle do not go via KVM secondary wakeup code path, so hwthread_state stores and barriers can be simplified, and some KVM code paths simplified a little.
Since v2:
- Rebase, SLB reload patch has been merged.
- More testing. Tested machine check idle wakeup path with mambo stepping through instructions.
Since v3:
- Build fixes caught by CI
--- arch/powerpc/include/asm/cpuidle.h | 19 +- arch/powerpc/include/asm/paca.h | 38 +- arch/powerpc/include/asm/processor.h | 9 +- arch/powerpc/include/asm/reg.h | 7 +- arch/powerpc/kernel/asm-offsets.c | 18 - arch/powerpc/kernel/dt_cpu_ftrs.c | 21 +- arch/powerpc/kernel/exceptions-64s.S | 17 +- arch/powerpc/kernel/idle_book3s.S | 1012 +++------------------- arch/powerpc/kernel/setup-common.c | 4 +- arch/powerpc/kvm/book3s_hv_rmhandlers.S | 86 +- arch/powerpc/platforms/powernv/idle.c | 828 ++++++++++++++---- arch/powerpc/platforms/powernv/subcore.c | 2 +- arch/powerpc/xmon/xmon.c | 24 +- 13 files changed, 880 insertions(+), 1205 deletions(-)
arch/powerpc/include/asm/cpuidle.h +3 -16
@@ -27,10 +27,11 @@* the THREAD_WINKLE_BITS are set, which indicate which threads have not * yet woken from the winkle state. */ -#define PNV_CORE_IDLE_LOCK_BIT 0x10000000 +#define NR_PNV_CORE_IDLE_LOCK_BIT 28 +#define PNV_CORE_IDLE_LOCK_BIT (1ULL << NR_PNV_CORE_IDLE_LOCK_BIT) +#define PNV_CORE_IDLE_WINKLE_COUNT_SHIFT 16#define PNV_CORE_IDLE_WINKLE_COUNT 0x00010000 -#define PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT 0x00080000#define PNV_CORE_IDLE_WINKLE_COUNT_BITS 0x000F0000#define PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT 8#define PNV_CORE_IDLE_THREAD_WINKLE_BITS 0x0000FF00
@@ -68,16 +69,6 @@#define ERR_DEEP_STATE_ESL_MISMATCH -2#ifndef __ASSEMBLY__ -/* Additional SPRs that need to be saved/restored during stop */ -struct stop_sprs { -u64 pid; -u64 ldbar; -u64 fscr; -u64 hfscr; -u64 mmcr1; -u64 mmcr2; -u64 mmcra; -};#define PNV_IDLE_NAME_LEN 16 struct pnv_idle_states_t {
@@ -92,10 +83,6 @@ struct pnv_idle_states_t {extern struct pnv_idle_states_t *pnv_idle_states;extern int nr_pnv_idle_states; -extern u32 pnv_fastsleep_workaround_at_entry[]; -extern u32 pnv_fastsleep_workaround_at_exit[]; - -extern u64 pnv_first_deep_stop_state;unsigned long pnv_cpu_offline (unsigned int cpu);int validate_psscr_val_mask (u64 *psscr_val,u64 *psscr_mask,u32 flags);
arch/powerpc/include/asm/paca.h +21 -17
@@ -168,7 +168,6 @@ struct paca_struct {u8 irq_happened;/* irq happened while soft-disabled */ u8 io_sync;/* writel() needs spin_unlock sync */ u8 irq_work_pending;/* IRQ_WORK interrupt while soft-disable */ -u8 nap_state_lost;/* NV GPR values lost in power7_idle */ #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE u8 pmcregs_in_use;/* pseries puts this in lppaca */ #endif
@@ -178,23 +177,28 @@ struct paca_struct {#endif #ifdef CONFIG_PPC_POWERNV -/* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */ -u32 *core_idle_state_ptr; -u8 thread_idle_state;/* PNV_THREAD_RUNNING/NAP/SLEEP */ -/* Mask to indicate thread id in core */ -u8 thread_mask; -/* Mask to denote subcore sibling threads */ -u8 subcore_sibling_mask; -/* Flag to request this thread not to stop */ -atomic_t dont_stop; -/* The PSSCR value that the kernel requested before going to stop */ -u64 requested_psscr; +/* PowerNV idle fields */ +/* PNV_CORE_IDLE_* bits, all siblings work on thread 0 paca */ +unsigned long idle_state; +union { +/* P7/P8 specific fields */ +struct { +/* PNV_THREAD_RUNNING/NAP/SLEEP */ +u8 thread_idle_state; +/* Mask to denote subcore sibling threads */ +u8 subcore_sibling_mask; + }; -/* -* Save area for additional SPRs that need to be -* saved/restored during cpuidle stop. -*/ -struct stop_sprs stop_sprs; +/* P9 specific fields */ +struct { +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +/* The PSSCR value that the kernel requested before going to stop */ +u64 requested_psscr; +/* Flag to request this thread not to stop */ +atomic_t dont_stop; +#endif + }; + };#endif #ifdef CONFIG_PPC_BOOK3S_64
arch/powerpc/include/asm/processor.h +6 -3
@@ -506,14 +506,17 @@ static inline unsigned long get_clean_sp(unsigned long sp, int is_32) }#endif +/* asm stubs */ +extern unsigned long isa300_idle_stop_noloss (unsigned long psscr_val); +extern unsigned long isa300_idle_stop_mayloss (unsigned long psscr_val); +extern unsigned long isa206_idle_insn_mayloss (unsigned long type); +extern unsigned long cpuidle_disable;enum idle_boot_override {IDLE_NO_OVERRIDE =0 ,IDLE_POWERSAVE_OFF };extern int powersave_nap;/* set if nap mode can be used in idle loop */ -extern unsigned long power7_idle_insn (unsigned long type);/* PNV_THREAD_NAP/etc*/ +extern void power7_idle_type (unsigned long type); -extern unsigned long power9_idle_stop (unsigned long psscr_val); -extern unsigned long power9_offline_stop (unsigned long psscr_val);extern void power9_idle_type (unsigned long stop_psscr_val,unsigned long stop_psscr_mask);
arch/powerpc/include/asm/reg.h +3 -4
@@ -752,10 +752,9 @@#define SRR1_WAKERESET 0x00100000/* System reset */ #define SRR1_WAKEHDBELL 0x000c0000/* Hypervisor doorbell on P8 */ #define SRR1_WAKESTATE 0x00030000/* Powersave exit mask [46:47] */ -#define SRR1_WS_DEEPEST 0x00030000/* Some resources not maintained, -* may not be recoverable */ -#define SRR1_WS_DEEPER 0x00020000/* Some resources not maintained */ -#define SRR1_WS_DEEP 0x00010000/* All resources maintained */ +#define SRR1_WS_HVLOSS 0x00030000/* HV resources not maintained */ +#define SRR1_WS_GPRLOSS 0x00020000/* GPRs not maintained */ +#define SRR1_WS_NOLOSS 0x00010000/* All resources maintained */ #define SRR1_PROGTM 0x00200000/* TM Bad Thing */ #define SRR1_PROGFPE 0x00100000/* Floating Point Enabled */ #define SRR1_PROGILL 0x00080000/* Illegal instruction */
arch/powerpc/kernel/asm-offsets.c +0 -18
@@ -256,7 +256,6 @@ int main(void)OFFSET (ACCOUNT_USER_TIME , paca_struct, accounting.utime);OFFSET (ACCOUNT_SYSTEM_TIME , paca_struct, accounting.stime);OFFSET (PACA_TRAP_SAVE , paca_struct, trap_save); -OFFSET (PACA_NAPSTATELOST , paca_struct, nap_state_lost);OFFSET (PACA_SPRG_VDSO , paca_struct, sprg_vdso);#else /* CONFIG_PPC64 */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
@@ -761,23 +760,6 @@ int main(void)OFFSET (VCPU_TIMING_LAST_ENTER_TBL , kvm_vcpu, arch.timing_last_enter.tv32.tbl);#endif -#ifdef CONFIG_PPC_POWERNV -OFFSET (PACA_CORE_IDLE_STATE_PTR , paca_struct, core_idle_state_ptr); -OFFSET (PACA_THREAD_IDLE_STATE , paca_struct, thread_idle_state); -OFFSET (PACA_THREAD_MASK , paca_struct, thread_mask); -OFFSET (PACA_SUBCORE_SIBLING_MASK , paca_struct, subcore_sibling_mask); -OFFSET (PACA_REQ_PSSCR , paca_struct, requested_psscr); -OFFSET (PACA_DONT_STOP , paca_struct, dont_stop); -#define STOP_SPR (x, f) OFFSET(x, paca_struct, stop_sprs.f) -STOP_SPR (STOP_PID , pid); -STOP_SPR (STOP_LDBAR , ldbar); -STOP_SPR (STOP_FSCR , fscr); -STOP_SPR (STOP_HFSCR , hfscr); -STOP_SPR (STOP_MMCR1 , mmcr1); -STOP_SPR (STOP_MMCR2 , mmcr2); -STOP_SPR (STOP_MMCRA , mmcra); -#endif -DEFINE (PPC_DBELL_SERVER ,PPC_DBELL_SERVER );DEFINE (PPC_DBELL_MSGTYPE ,PPC_DBELL_MSGTYPE );
arch/powerpc/kernel/dt_cpu_ftrs.c +1 -20
@@ -71,7 +71,6 @@ static int hv_mode;static struct {u64 lpcr; -u64 lpcr_clear;u64 hfscr;u64 fscr; } system_registers;
@@ -80,24 +79,7 @@ static void (*init_pmu_registers)(void);static void __restore_cpu_cpufeatures (void ) { -u64 lpcr; - -/* -* LPCR is restored by the power on engine already. It can be changed -* after early init e.g., by radix enable, and we have no unified API -* for saving and restoring such SPRs. -* -* This ->restore hook should really be removed from idle and register -* restore moved directly into the idle restore code, because this code -* doesn't know how idle is implemented or what it needs restored here. -* -* The best we can do to accommodate secondary boot and idle restore -* for now is "or" LPCR with existing. -*/ - lpcr =mfspr (SPRN_LPCR ); - lpcr |= system_registers.lpcr; - lpcr &= ~system_registers.lpcr_clear; -mtspr (SPRN_LPCR , lpcr); +mtspr (SPRN_LPCR , system_registers.lpcr);if (hv_mode) {mtspr (SPRN_LPID ,0 );mtspr (SPRN_HFSCR , system_registers.hfscr);
@@ -318,7 +300,6 @@ static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) {u64 lpcr; - system_registers.lpcr_clear |= (LPCR_ISL |LPCR_UPRT |LPCR_HR ); lpcr =mfspr (SPRN_LPCR ); lpcr &= ~(LPCR_ISL |LPCR_UPRT |LPCR_HR );mtspr (SPRN_LPCR , lpcr);
arch/powerpc/kernel/exceptions-64s.S +9 -8
@@ -136,8 +136,9 @@ TRAMP_KVM(PACA_EXNMI, 0x100) #ifdef CONFIG_PPC_P7_NAP EXC_COMMON_BEGIN(system_reset_idle_common) - mfspr r12,SPRN_SRR1 - b pnv_powersave_wakeup + mfspr r3,SPRN_SRR1 + bltlr cr3 /* no state loss, return to idle caller */ + b idle_return_gpr_loss #endif /*
@@ -416,17 +417,17 @@ EXC_COMMON_BEGIN(machine_check_idle_common) * Then decrement MCE nesting after finishing with the stack. */ ld r3,_MSR(r1) + ld r4,_LINK(r1) lhz r11,PACA_IN_MCE(r13) subi r11,r11,1 sth r11,PACA_IN_MCE(r13) - /* Turn off the RI bit because SRR1 is used by idle wakeup code. */ - /* Recoverability could be improved by reducing the use of SRR1. */ - li r11,0 - mtmsrd r11,1 - - b pnv_powersave_wakeup_mce + mtlr r4 + rlwinm r10,r3,47-31,30,31 + cmpwi cr3,r10,2 + bltlr cr3 /* no state loss, return to idle caller */ + b idle_return_gpr_loss #endif /* * Handle machine check early in real mode. We come here with
arch/powerpc/kernel/idle_book3s.S +103 -909
1,053 lines not on this page · raw message
arch/powerpc/kernel/setup-common.c +2 -2
@@ -401,8 +401,8 @@ void __init check_for_initrd(void)#ifdef CONFIG_SMP -int threads_per_core, threads_per_subcore, threads_shift; -cpumask_t threads_core_mask; +int threads_per_core, threads_per_subcore, threads_shift__read_mostly ; +cpumask_t threads_core_mask__read_mostly ;EXPORT_SYMBOL_GPL (threads_per_core);EXPORT_SYMBOL_GPL (threads_per_subcore);EXPORT_SYMBOL_GPL (threads_shift);
arch/powerpc/kvm/book3s_hv_rmhandlers.S +48 -38
136 lines not on this page · raw message
arch/powerpc/platforms/powernv/idle.c +673 -153
980 lines not on this page · raw message
arch/powerpc/platforms/powernv/subcore.c +1 -1
@@ -183,7 +183,7 @@ static void unsplit_core(void) cpu =smp_processor_id ();if (cpu_thread_in_core (cpu) !=0 ) {while (mfspr (SPRN_HID0 ) & mask) -power7_idle_insn (PNV_THREAD_NAP ); +power7_idle_type (PNV_THREAD_NAP );per_cpu (split_state, cpu).step =SYNC_STEP_UNSPLIT ;return ;
arch/powerpc/xmon/xmon.c +10 -14
@@ -2419,7 +2419,6 @@ static void dump_one_paca(int cpu)DUMP (p, irq_happened,"%#-*x" );DUMP (p, io_sync,"%#-*x" );DUMP (p, irq_work_pending,"%#-*x" ); -DUMP (p, nap_state_lost,"%#-*x" );DUMP (p, sprg_vdso,"%#-*llx" );#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -2427,19 +2426,16 @@ static void dump_one_paca(int cpu)#endif #ifdef CONFIG_PPC_POWERNV -DUMP (p, core_idle_state_ptr,"%-*px" ); -DUMP (p, thread_idle_state,"%#-*x" ); -DUMP (p, thread_mask,"%#-*x" ); -DUMP (p, subcore_sibling_mask,"%#-*x" ); -DUMP (p, requested_psscr,"%#-*llx" ); -DUMP (p, stop_sprs.pid,"%#-*llx" ); -DUMP (p, stop_sprs.ldbar,"%#-*llx" ); -DUMP (p, stop_sprs.fscr,"%#-*llx" ); -DUMP (p, stop_sprs.hfscr,"%#-*llx" ); -DUMP (p, stop_sprs.mmcr1,"%#-*llx" ); -DUMP (p, stop_sprs.mmcr2,"%#-*llx" ); -DUMP (p, stop_sprs.mmcra,"%#-*llx" ); -DUMP (p, dont_stop.counter,"%#-*x" ); +DUMP (p, idle_state,"%#-*lx" ); +if (!early_cpu_has_feature (CPU_FTR_ARCH_300 )) { +DUMP (p, thread_idle_state,"%#-*x" ); +DUMP (p, subcore_sibling_mask,"%#-*x" ); + }else { +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +DUMP (p, requested_psscr,"%#-*llx" ); +DUMP (p, dont_stop.counter,"%#-*x" ); +#endif + }#endif DUMP (p, accounting.utime,"%#-*lx" );
Signature · 1 line
2.18.0
arch/powerpc/platforms/powernv/idle.c void power7_idle(void) · line 711
Quoted 7 lines · Nicholas Piggin
arch/powerpc/platforms/powernv/idle.c +7
-0+ +/* +* On POWER9, SRR1 bits do not match exactly as expected. +* SRR1_WS_GPRLOSS (10b) can also result in SPR loss, so +* always test PSSCR if there is any state loss. +*/ +if (likely ((psscr &PSSCR_RL_MASK ) < pnv_first_hv_loss_level)) {
Shouldn't we check PLS field to see if the cpu/core woke up from hv loss ?
Currently, a cpu requested stop4 (RL=4) and exited from a shallower state (PLS=2), SPR's are unecessarily restored.
We can do something like :
#define PSSCR_PLS_SHIFT 60
if (likely((psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT) < pnv_first_hv_loss_level)
Quoted 25 lines
arch/powerpc/platforms/powernv/idle.c +25
-0+if (sprs_saved) +atomic_stop_thread_idle (); + goto out; + } + +/* HV state loss */ +BUG_ON (!sprs_saved); + +atomic_lock_thread_idle (); + +if ((*state & ((1 << threads_per_core) -1 )) !=0 ) + goto core_woken; + +/* Per-core SPRs */ +mtspr (SPRN_PTCR , sprs.ptcr); +mtspr (SPRN_RPR , sprs.rpr); +mtspr (SPRN_TSCR , sprs.tscr); +mtspr (SPRN_LDBAR , sprs.ldbar); +mtspr (SPRN_AMOR , sprs.amor); + +if ((psscr &PSSCR_RL_MASK ) >= pnv_first_tb_loss_level) { +/* TB loss */ +if (opal_resync_timebase () !=OPAL_SUCCESS ) +BUG (); + }
Nicholas Piggin
reviewerQuoted 11 lines · On Wed, 26 Sep 2018 19:39:14 +0530 Akshay Adiga…
arch/powerpc/platforms/powernv/idle.c void power7_idle(void) · line 711
Quoted 7 lines · Nicholas Piggin
arch/powerpc/platforms/powernv/idle.c +7
-0+ +/* +* On POWER9, SRR1 bits do not match exactly as expected. +* SRR1_WS_GPRLOSS (10b) can also result in SPR loss, so +* always test PSSCR if there is any state loss. +*/ +if (likely ((psscr &PSSCR_RL_MASK ) < pnv_first_hv_loss_level)) {Shouldn't we check PLS field to see if the cpu/core woke up from hv loss ?
Quoted 7 lines
Currently, a cpu requested stop4 (RL=4) and exited from a shallower state (PLS=2), SPR's are unecessarily restored.
We can do something like :
#define
PSSCR_PLS_SHIFT60
if (likely((psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT)<pnv_first_hv_loss_level)
Ah, that corresponds with the following existing code?
/*
* POWER ISA 3. Use PSSCR to determine if we
* are waking up from deep idle state */
LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
/*
* 0-3 bits correspond to Power-Saving Level Status
* which indicates the idle state we are waking up from */
mfspr r5, SPRN_PSSCR
rldicl r5,r5,4,60
li r0, 0 /* clear requested_psscr to say we're awake */
std r0, PACA_REQ_PSSCR(r13)
cmpd cr4,r5,r4
bge cr4,pnv_wakeup_tb_loss /* returns to caller */
Yes I didn't get that right, good catch.
Thanks,
Nick