powerpc/64s: security fallback improvement
Subsystems (2)
Nicholas Piggin
reviewerRFC PATCH 1/1 powerpc/64s: security fallback improvement
The fallback sequences for L1D flushing and store forwarding barriers require contortions with saving and reloading registers from a PACA save area and SPRGs because they are called at nasty places when registers are not available. The intention was to make them cover as much kernel data as possible.
This is quite painful and non-obvious, and has caused a few difficult bugs (recently the scv interrupt re-entrancy), and things would get even more hairy with planned interrupt exit optimizations that can return without disabling interrupts.
So I propose to move the fallbacks in a bit to the point where we have the kernel stack set up and registers available, reentrancy PACA scratch and SPRs become non-issues. This does potenitally expose slightly more attack surface, but not a huge amount (mainly some stack frame and paca), and firmware to implement the stateless security ops has been available for several years now.
This patch isn't quite complete, it's missing a bit of logic about where to enable / disable the call to the fallback functions, but that's not too hard (if any workaround requires fallback -> make the call, and possibly patch the workaround function to suit). But the tricky bits seem to be working.
This goes on top of the previous series, I thought I would add some C calls with static unlikely branches, but it turns out we still have a few cases (like the FPU register loading) that is not amenable to it and return directly with fast_interrupt_return, so it ends up easier to make it all in asm for now. Longer term I might look at C wrappers for those guys and move this to C, but for now it wasn't to be so.
Thanks,
Nick
--- arch/powerpc/include/asm/exception-64s.h | 52 +++--- arch/powerpc/include/asm/feature-fixups.h | 18 ++ arch/powerpc/include/asm/paca.h | 6 +- arch/powerpc/kernel/asm-offsets.c | 2 +- arch/powerpc/kernel/entry_64.S | 24 ++- arch/powerpc/kernel/exceptions-64s.S | 177 +++++--------------- arch/powerpc/kernel/vmlinux.lds.S | 14 ++ arch/powerpc/kvm/book3s_hv_rmhandlers.S | 4 + arch/powerpc/lib/feature-fixups.c | 192 ++++++++++------------ 9 files changed, 212 insertions(+), 277 deletions(-)
arch/powerpc/include/asm/exception-64s.h +24 -28
@@ -44,30 +44,21 @@#define STF_ENTRY_BARRIER_SLOT \ STF_ENTRY_BARRIER_FIXUP_SECTION; \ - nop; \ - nop; \ nop#define STF_EXIT_BARRIER_SLOT \ STF_EXIT_BARRIER_FIXUP_SECTION; \ - nop; \ - nop; \ - nop; \ - nop; \ - nop; \ nop#define ENTRY_FLUSH_SLOT \ ENTRY_FLUSH_FIXUP_SECTION; \ nop; \ - nop; \ - nop; + nop#define SCV_ENTRY_FLUSH_SLOT \ SCV_ENTRY_FLUSH_FIXUP_SECTION; \ nop; \ - nop; \ - nop; + nop/* * r10 must be free to use, r13 must be paca
@@ -100,7 +91,6 @@#define RFI_FLUSH_SLOT \ RFI_FLUSH_FIXUP_SECTION; \ nop; \ - nop; \ nop#define RFI_TO_KERNEL \
@@ -109,20 +99,17 @@#define RFI_TO_USER \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - rfid; \ - b rfi_flush_fallback + rfid#define RFI_TO_USER_OR_KERNEL \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - rfid; \ - b rfi_flush_fallback + rfid#define RFI_TO_GUEST \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - rfid; \ - b rfi_flush_fallback + rfid#define HRFI_TO_KERNEL \ hrfid
@@ -130,35 +117,44 @@#define HRFI_TO_USER \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - hrfid; \ - b hrfi_flush_fallback + hrfid#define HRFI_TO_USER_OR_KERNEL \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - hrfid; \ - b hrfi_flush_fallback + hrfid#define HRFI_TO_GUEST \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - hrfid; \ - b hrfi_flush_fallback + hrfid#define HRFI_TO_UNKNOWN \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - hrfid; \ - b hrfi_flush_fallback + hrfid#define RFSCV_TO_USER \ STF_EXIT_BARRIER_SLOT; \ RFI_FLUSH_SLOT; \ - RFSCV; \ - b rfscv_flush_fallback + RFSCV + +#define ENTER_KERNEL_SECURITY_FALLBACK \ + ENTER_SECURITY_FALLBACK_SECTION; \ + nop + +#define EXIT_KERNEL_SECURITY_FALLBACK \ + EXIT_SECURITY_FALLBACK_SECTION; \ + nop + +#define ENTER_GUEST_SECURITY_FALLBACK \ + EXIT_SECURITY_FALLBACK_SECTION; \ + nop#else /* __ASSEMBLY__ */ /* Prototype for function defined in exceptions-64s.S */ +void exit_security_fallback (void ); +void enter_security_fallback (void );void do_uaccess_flush (void );#endif /* __ASSEMBLY__ */
arch/powerpc/include/asm/feature-fixups.h +18 -0
@@ -256,6 +256,22 @@ label##3: \ FTR_ENTRY_OFFSET 951b-952b; \ .popsection; +#define ENTER_SECURITY_FALLBACK_SECTION \ +958: \ + .pushsection __enter_security_fallback_fixup,"a"; \ + .align 2; \ +959: \ + FTR_ENTRY_OFFSET 958b-959b; \ + .popsection; + +#define EXIT_SECURITY_FALLBACK_SECTION \ +960: \ + .pushsection __exit_security_fallback_fixup,"a"; \ + .align 2; \ +961: \ + FTR_ENTRY_OFFSET 960b-961b; \ + .popsection; +#define NOSPEC_BARRIER_FIXUP_SECTION \ 953: \ .pushsection __barrier_nospec_fixup,"a"; \
@@ -288,6 +304,8 @@ extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup;extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;extern long __start___scv_entry_flush_fixup, __stop___scv_entry_flush_fixup;extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; +extern long __start___enter_security_fallback_fixup, __stop___enter_security_fallback_fixup; +extern long __start___exit_security_fallback_fixup, __stop___exit_security_fallback_fixup;extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
arch/powerpc/include/asm/paca.h +1 -5
@@ -249,11 +249,7 @@ struct paca_struct {#endif #endif #ifdef CONFIG_PPC_BOOK3S_64 -/* -* rfi fallback flush must be in its own cacheline to prevent -* other paca data leaking into the L1d -*/ -u64 exrfi[EX_SIZE ]__aligned(0x80) ; +u64 stf_fallback_scratch[2 ];void *rfi_flush_fallback_area;u64 l1d_flush_size;#endif
arch/powerpc/kernel/asm-offsets.c +1 -1
@@ -279,7 +279,7 @@ int main(void)OFFSET (PACA_IN_MCE , paca_struct, in_mce);OFFSET (PACA_IN_NMI , paca_struct, in_nmi);OFFSET (PACA_RFI_FLUSH_FALLBACK_AREA , paca_struct, rfi_flush_fallback_area); -OFFSET (PACA_EXRFI , paca_struct, exrfi); +OFFSET (PACA_STF_FALLBACK_SCRATCH , paca_struct, stf_fallback_scratch);OFFSET (PACA_L1D_FLUSH_SIZE , paca_struct, l1d_flush_size);#endif
arch/powerpc/kernel/entry_64.S +18 -6
@@ -117,6 +117,8 @@ BEGIN_FTR_SECTION HMT_MEDIUM END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) + ENTER_KERNEL_SECURITY_FALLBACK + /* * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which * would clobber syscall parameters. Also we always enter with IRQs
@@ -136,6 +138,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) li r5,1 /* scv */ bl syscall_exit_prepare + EXIT_KERNEL_SECURITY_FALLBACK + ld r2,_CCR(r1) ld r4,_NIP(r1) ld r5,_MSR(r1)
@@ -294,6 +298,8 @@ END_BTB_FLUSH_SECTION stb r11,PACAIRQSOFTMASK(r13) stb r12,PACAIRQHAPPENED(r13) + ENTER_KERNEL_SECURITY_FALLBACK + /* Calling convention has r9 = orig r0, r10 = regs */ mr r9,r0 bl system_call_exception
@@ -303,6 +309,8 @@ END_BTB_FLUSH_SECTION li r5,0 /* !scv */ bl syscall_exit_prepare + EXIT_KERNEL_SECURITY_FALLBACK + ld r2,_CCR(r1) ld r4,_NIP(r1) ld r5,_MSR(r1)
@@ -653,11 +661,16 @@ _ASM_NOKPROBE_SYMBOL(fast_interrupt_return) kuap_check_amr r3, r4 ld r5,_MSR(r1) andi. r0,r5,MSR_PR - bne .Lfast_user_interrupt_return_amr - kuap_kernel_restore r3, r4 + beq 1f + kuap_user_restore r3, r4 + b .Lfast_user_interrupt_return +1: andi. r0,r5,MSR_RI + beq- 2f + kuap_kernel_restore r3, r4 li r3,0 /* 0 return value, no EMULATE_STACK_STORE */ - bne+ .Lfast_kernel_interrupt_return + b .Lfast_kernel_interrupt_return +2: addi r3,r1,STACK_FRAME_OVERHEAD bl unrecoverable_exception b . /* should not get here */
@@ -673,10 +686,9 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) bl interrupt_exit_user_prepare cmpdi r3,0 bne- .Lrestore_nvgprs - -.Lfast_user_interrupt_return_amr: - kuap_user_restore r3, r4 .Lfast_user_interrupt_return: + EXIT_KERNEL_SECURITY_FALLBACK + ld r11,_NIP(r1) ld r12,_MSR(r1) BEGIN_FTR_SECTION
arch/powerpc/kernel/exceptions-64s.S +44 -133
219 lines not on this page · raw message
arch/powerpc/kernel/vmlinux.lds.S +14 -0
@@ -165,6 +165,20 @@ SECTIONS *(__rfi_flush_fixup) __stop___rfi_flush_fixup = .; } + + . = ALIGN(8); + __enter_security_fallback_fixup : AT(ADDR(__enter_security_fallback_fixup) - LOAD_OFFSET) { + __start___enter_security_fallback_fixup = .; + *(__enter_security_fallback_fixup) + __stop___enter_security_fallback_fixup = .; + } + + . = ALIGN(8); + __exit_security_fallback_fixup : AT(ADDR(__exit_security_fallback_fixup) - LOAD_OFFSET) { + __start___exit_security_fallback_fixup = .; + *(__exit_security_fallback_fixup) + __stop___exit_security_fallback_fixup = .; + } #endif /* CONFIG_PPC64 */ #ifdef CONFIG_PPC_BARRIER_NOSPEC
arch/powerpc/kvm/book3s_hv_rmhandlers.S +4 -0
@@ -1078,6 +1078,8 @@ fast_guest_return: bl kvmhv_accumulate_time #endif + ENTER_GUEST_SECURITY_FALLBACK + /* Enter guest */ BEGIN_FTR_SECTION
@@ -1346,6 +1348,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) li r0, MSR_RI mtmsrd r0, 1 + ENTER_KERNEL_SECURITY_FALLBACK + #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING addi r3, r9, VCPU_TB_RMINTR mr r4, r9
arch/powerpc/lib/feature-fixups.c +88 -104
308 lines not on this page · raw message
Signature · 1 line
2.23.0