powerpc: branch cache flush changes
Subsystems (2)
Nicholas Piggin
reviewercover powerpc: branch cache flush changes
This series allows the link stack to be flushed with the speical bcctr 2,0,0 flush instruction that also flushes the count cache if the processor supports it.
Firmware does not support this at the moment, but I've tested it in simulator with a patched firmware to advertise support.
Thanks,
Nick
Nicholas Piggin (7):
powerpc/security: re-name count cache flush to branch cache flush powerpc/security: change link stack flush state to the flush type enum powerpc/security: make display of branch cache flush more consistent powerpc/security: split branch cache flush toggle from code patching powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h powerpc/security: Allow for processors that flush the link stack using
the special bcctr
powerpc/64s: advertise hardware link stack flush
arch/powerpc/include/asm/asm-prototypes.h | 4 +- arch/powerpc/include/asm/hvcall.h | 1 + arch/powerpc/include/asm/ppc-opcode.h | 2 + arch/powerpc/include/asm/security_features.h | 2 + arch/powerpc/include/uapi/asm/kvm.h | 1 + arch/powerpc/kernel/entry_64.S | 13 +- arch/powerpc/kernel/security.c | 139 +++++++++++-------- arch/powerpc/kvm/powerpc.c | 9 +- arch/powerpc/platforms/powernv/setup.c | 3 + arch/powerpc/platforms/pseries/setup.c | 3 + tools/arch/powerpc/include/uapi/asm/kvm.h | 1 + 11 files changed, 106 insertions(+), 72 deletions(-)
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 1/7 powerpc/security: re-name count cache flush to branch cache flush
The count cache flush mostly refers to both count cache and link stack flushing. As a first step to untangling these a bit, re-name the bits that apply to both.
--- arch/powerpc/include/asm/asm-prototypes.h | 4 +-- arch/powerpc/kernel/entry_64.S | 7 ++--- arch/powerpc/kernel/security.c | 36 +++++++++++------------ 3 files changed, 23 insertions(+), 24 deletions(-)
arch/powerpc/include/asm/asm-prototypes.h +2 -2
@@ -144,13 +144,13 @@ void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);void _kvmppc_save_tm_pr (struct kvm_vcpu *vcpu,u64 guest_msr);/* Patch sites */ -extern s32 patch__call_flush_count_cache; +extern s32 patch__call_flush_branch_caches;extern s32 patch__flush_count_cache_return;extern s32 patch__flush_link_stack_return;extern s32 patch__call_kvm_flush_link_stack;extern s32 patch__memset_nocache, patch__memcpy_nocache; -extern long flush_count_cache; +extern long flush_branch_caches;extern long kvm_flush_link_stack;#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
arch/powerpc/kernel/entry_64.S +3 -4
@@ -259,8 +259,7 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs); #define FLUSH_COUNT_CACHE \ 1: nop; \ - patch_site 1b, patch__call_flush_count_cache - + patch_site 1b, patch__call_flush_branch_caches #define BCCTR_FLUSH .long 0x4c400420
@@ -271,8 +270,8 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs); .endm .balign 32 -.global flush_count_cache -flush_count_cache: +.global flush_branch_caches +flush_branch_caches: /* Save LR into r9 */ mflr r9
arch/powerpc/kernel/security.c +18 -18
@@ -21,12 +21,12 @@u64 powerpc_security_features__read_mostly =SEC_FTR_DEFAULT ; -enum count_cache_flush_type { -COUNT_CACHE_FLUSH_NONE =0x1 , -COUNT_CACHE_FLUSH_SW =0x2 , -COUNT_CACHE_FLUSH_HW =0x4 , +enum branch_cache_flush_type { +BRANCH_CACHE_FLUSH_NONE =0x1 , +BRANCH_CACHE_FLUSH_SW =0x2 , +BRANCH_CACHE_FLUSH_HW =0x4 , }; -static enum count_cache_flush_type count_cache_flush_type =COUNT_CACHE_FLUSH_NONE ; +static enum branch_cache_flush_type count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ;static bool link_stack_flush_enabled;bool barrier_nospec_enabled;
@@ -222,10 +222,10 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, cif (link_stack_flush_enabled)seq_buf_printf (&s,", Software link stack flush" ); - }else if (count_cache_flush_type !=COUNT_CACHE_FLUSH_NONE ) { + }else if (count_cache_flush_type !=BRANCH_CACHE_FLUSH_NONE ) {seq_buf_printf (&s,"Mitigation: Software count cache flush" ); -if (count_cache_flush_type ==COUNT_CACHE_FLUSH_HW ) +if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_HW )seq_buf_printf (&s," (hardware accelerated)" );if (link_stack_flush_enabled)
@@ -429,18 +429,18 @@ device_initcall(stf_barrier_debugfs_init);static void no_count_cache_flush (void ) { - count_cache_flush_type =COUNT_CACHE_FLUSH_NONE ; + count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ;pr_info ("count-cache-flush: software flush disabled.\n" ); } -static void toggle_count_cache_flush (bool enable) +static void toggle_branch_cache_flush (bool enable) {if (!security_ftr_enabled (SEC_FTR_FLUSH_COUNT_CACHE ) && !security_ftr_enabled (SEC_FTR_FLUSH_LINK_STACK )) enable = false;if (!enable) { -patch_instruction_site (&patch__call_flush_count_cache, +patch_instruction_site (&patch__call_flush_branch_caches,ppc_inst (PPC_INST_NOP ));#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE patch_instruction_site(&patch__call_kvm_flush_link_stack,
@@ -452,9 +452,9 @@ static void toggle_count_cache_flush(bool enable)return ; } -// This enables the branch from _switch to flush_count_cache -patch_branch_site (&patch__call_flush_count_cache, - (u64)&flush_count_cache,BRANCH_SET_LINK ); +// This enables the branch from _switch to flush_branch_caches +patch_branch_site (&patch__call_flush_branch_caches, + (u64)&flush_branch_caches,BRANCH_SET_LINK );#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE // This enables the branch from guest_exit_cont to kvm_flush_link_stack
@@ -474,13 +474,13 @@ static void toggle_count_cache_flush(bool enable) }if (!security_ftr_enabled (SEC_FTR_BCCTR_FLUSH_ASSIST )) { - count_cache_flush_type =COUNT_CACHE_FLUSH_SW ; + count_cache_flush_type =BRANCH_CACHE_FLUSH_SW ;pr_info ("count-cache-flush: full software flush sequence enabled.\n" );return ; }patch_instruction_site (&patch__flush_count_cache_return,ppc_inst (PPC_INST_BLR )); - count_cache_flush_type =COUNT_CACHE_FLUSH_HW ; + count_cache_flush_type =BRANCH_CACHE_FLUSH_HW ;pr_info ("count-cache-flush: hardware assisted flush sequence enabled\n" ); }
@@ -505,7 +505,7 @@ void setup_count_cache_flush(void) security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE))security_ftr_set (SEC_FTR_FLUSH_LINK_STACK ); -toggle_count_cache_flush (enable); +toggle_branch_cache_flush (enable); }#ifdef CONFIG_DEBUG_FS
@@ -520,14 +520,14 @@ static int count_cache_flush_set(void *data, u64 val)else return -EINVAL ; -toggle_count_cache_flush (enable); +toggle_branch_cache_flush (enable);return 0 ; }static int count_cache_flush_get (void *data,u64 *val) { -if (count_cache_flush_type ==COUNT_CACHE_FLUSH_NONE ) +if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_NONE ) *val =0 ;else *val =1 ;
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 2/7 powerpc/security: change link stack flush state to the flush type enum
In reply to Nicholas PigginPrepare to allow for hardware link stack flushing by using the none/sw/hw type, same as the count cache state.
--- arch/powerpc/kernel/security.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/powerpc/kernel/security.c +5 -5
@@ -27,7 +27,7 @@ enum branch_cache_flush_type { BRANCH_CACHE_FLUSH_HW = 0x4, };static enum branch_cache_flush_type count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ; -static bool link_stack_flush_enabled; +static enum branch_cache_flush_type link_stack_flush_type =BRANCH_CACHE_FLUSH_NONE ;bool barrier_nospec_enabled;static bool no_nospec;
@@ -219,7 +219,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, cif (ccd)seq_buf_printf (&s,"Indirect branch cache disabled" ); -if (link_stack_flush_enabled) +if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_SW )seq_buf_printf (&s,", Software link stack flush" ); }else if (count_cache_flush_type !=BRANCH_CACHE_FLUSH_NONE ) {
@@ -228,7 +228,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, cif (count_cache_flush_type ==BRANCH_CACHE_FLUSH_HW )seq_buf_printf (&s," (hardware accelerated)" ); -if (link_stack_flush_enabled) +if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_SW )seq_buf_printf (&s,", Software link stack flush" ); }else if (btb_flush_enabled) {
@@ -447,7 +447,7 @@ static void toggle_branch_cache_flush(bool enable) ppc_inst(PPC_INST_NOP));#endif pr_info ("link-stack-flush: software flush disabled.\n" ); - link_stack_flush_enabled = false; + link_stack_flush_type =BRANCH_CACHE_FLUSH_NONE ;no_count_cache_flush ();return ; }
@@ -463,7 +463,7 @@ static void toggle_branch_cache_flush(bool enable)#endif pr_info ("link-stack-flush: software flush enabled.\n" ); - link_stack_flush_enabled = true; + link_stack_flush_type =BRANCH_CACHE_FLUSH_SW ;// If we just need to flush the link stack, patch an early return if (!security_ftr_enabled (SEC_FTR_FLUSH_COUNT_CACHE )) {
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 3/7 powerpc/security: make display of branch cache flush more consistent
In reply to Nicholas PigginMake the count-cache and link-stack messages look the same
--- arch/powerpc/kernel/security.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/powerpc/kernel/security.c +4 -4
@@ -430,7 +430,7 @@ device_initcall(stf_barrier_debugfs_init);static void no_count_cache_flush (void ) { count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ; -pr_info ("count-cache-flush: software flush disabled.\n" ); +pr_info ("count-cache-flush: flush disabled.\n" ); }static void toggle_branch_cache_flush (bool enable)
@@ -446,7 +446,7 @@ static void toggle_branch_cache_flush(bool enable)patch_instruction_site (&patch__call_kvm_flush_link_stack,ppc_inst (PPC_INST_NOP ));#endif -pr_info ("link-stack-flush: software flush disabled.\n" ); +pr_info ("link-stack-flush: flush disabled.\n" ); link_stack_flush_type =BRANCH_CACHE_FLUSH_NONE ;no_count_cache_flush ();return ;
@@ -475,13 +475,13 @@ static void toggle_branch_cache_flush(bool enable)if (!security_ftr_enabled (SEC_FTR_BCCTR_FLUSH_ASSIST )) { count_cache_flush_type =BRANCH_CACHE_FLUSH_SW ; -pr_info ("count-cache-flush: full software flush sequence enabled.\n" ); +pr_info ("count-cache-flush: software flush enabled.\n" );return ; }patch_instruction_site (&patch__flush_count_cache_return,ppc_inst (PPC_INST_BLR )); count_cache_flush_type =BRANCH_CACHE_FLUSH_HW ; -pr_info ("count-cache-flush: hardware assisted flush sequence enabled\n" ); +pr_info ("count-cache-flush: hardware flush enabled.\n" ); }void setup_count_cache_flush (void )
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 4/7 powerpc/security: split branch cache flush toggle from code patching
In reply to Nicholas PigginBranch cache flushing code patching has inter-dependencies on both the link stack and the count cache flushing state.
To make the code clearer and to separate the link stack and count cache handling, split the "toggle" (setting up variables and printing enable/disable) from the code patching.
--- arch/powerpc/kernel/security.c | 94 ++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 43 deletions(-)
arch/powerpc/kernel/security.c +51 -43
@@ -427,61 +427,69 @@ static __init int stf_barrier_debugfs_init(void)device_initcall (stf_barrier_debugfs_init);#endif /* CONFIG_DEBUG_FS */ -static void no_count_cache_flush (void ) +static void update_branch_cache_flush (void ) { - count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ; -pr_info ("count-cache-flush: flush disabled.\n" ); -} - -static void toggle_branch_cache_flush (bool enable) -{ -if (!security_ftr_enabled (SEC_FTR_FLUSH_COUNT_CACHE ) && - !security_ftr_enabled (SEC_FTR_FLUSH_LINK_STACK )) - enable = false; - -if (!enable) { -patch_instruction_site (&patch__call_flush_branch_caches, -ppc_inst (PPC_INST_NOP ));#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +// This controls the branch from guest_exit_cont to kvm_flush_link_stack +if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_NONE ) {patch_instruction_site (&patch__call_kvm_flush_link_stack,ppc_inst (PPC_INST_NOP )); -#endif -pr_info ("link-stack-flush: flush disabled.\n" ); - link_stack_flush_type =BRANCH_CACHE_FLUSH_NONE ; -no_count_cache_flush (); -return ; + }else { +patch_branch_site (&patch__call_kvm_flush_link_stack, + (u64)&kvm_flush_link_stack,BRANCH_SET_LINK ); } - -// This enables the branch from _switch to flush_branch_caches -patch_branch_site (&patch__call_flush_branch_caches, - (u64)&flush_branch_caches,BRANCH_SET_LINK ); - -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE -// This enables the branch from guest_exit_cont to kvm_flush_link_stack -patch_branch_site (&patch__call_kvm_flush_link_stack, - (u64)&kvm_flush_link_stack,BRANCH_SET_LINK );#endif -pr_info ("link-stack-flush: software flush enabled.\n" ); - link_stack_flush_type =BRANCH_CACHE_FLUSH_SW ; +// This controls the branch from _switch to flush_branch_caches +if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_NONE && + link_stack_flush_type ==BRANCH_CACHE_FLUSH_NONE ) { +patch_instruction_site (&patch__call_flush_branch_caches, +ppc_inst (PPC_INST_NOP )); + }else { +patch_branch_site (&patch__call_flush_branch_caches, + (u64)&flush_branch_caches,BRANCH_SET_LINK ); + +// If we just need to flush the link stack, early return +if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_NONE ) { +patch_instruction_site (&patch__flush_link_stack_return, +ppc_inst (PPC_INST_BLR )); + +// If we have flush instruction, early return + }else if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_HW ) { +patch_instruction_site (&patch__flush_count_cache_return, +ppc_inst (PPC_INST_BLR )); + } + } +} -// If we just need to flush the link stack, patch an early return -if (!security_ftr_enabled (SEC_FTR_FLUSH_COUNT_CACHE )) { -patch_instruction_site (&patch__flush_link_stack_return, -ppc_inst (PPC_INST_BLR )); -no_count_cache_flush (); -return ; +static void toggle_branch_cache_flush (bool enable) +{ +if (!enable || !security_ftr_enabled (SEC_FTR_FLUSH_COUNT_CACHE )) { +if (count_cache_flush_type !=BRANCH_CACHE_FLUSH_NONE ) { + count_cache_flush_type =BRANCH_CACHE_FLUSH_NONE ; +pr_info ("count-cache-flush: flush disabled.\n" ); + } + }else { +if (security_ftr_enabled (SEC_FTR_BCCTR_FLUSH_ASSIST )) { + count_cache_flush_type =BRANCH_CACHE_FLUSH_HW ; +pr_info ("count-cache-flush: hardware flush enabled.\n" ); + }else { + count_cache_flush_type =BRANCH_CACHE_FLUSH_SW ; +pr_info ("count-cache-flush: software flush enabled.\n" ); + } } -if (!security_ftr_enabled (SEC_FTR_BCCTR_FLUSH_ASSIST )) { - count_cache_flush_type =BRANCH_CACHE_FLUSH_SW ; -pr_info ("count-cache-flush: software flush enabled.\n" ); -return ; +if (!enable || !security_ftr_enabled (SEC_FTR_FLUSH_LINK_STACK )) { +if (link_stack_flush_type !=BRANCH_CACHE_FLUSH_NONE ) { + link_stack_flush_type =BRANCH_CACHE_FLUSH_NONE ; +pr_info ("link-stack-flush: flush disabled.\n" ); + } + }else { + link_stack_flush_type =BRANCH_CACHE_FLUSH_SW ; +pr_info ("link-stack-flush: software flush enabled.\n" ); } -patch_instruction_site (&patch__flush_count_cache_return,ppc_inst (PPC_INST_BLR )); - count_cache_flush_type =BRANCH_CACHE_FLUSH_HW ; -pr_info ("count-cache-flush: hardware flush enabled.\n" ); +update_branch_cache_flush (); }void setup_count_cache_flush (void )
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 5/7 powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h
In reply to Nicholas Piggin--- arch/powerpc/include/asm/ppc-opcode.h | 2 ++ arch/powerpc/kernel/entry_64.S | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-)
arch/powerpc/include/asm/ppc-opcode.h +2 -0
@@ -195,6 +195,7 @@#define OP_LQ 56/* sorted alphabetically */ +#define PPC_INST_BCCTR_FLUSH 0x4c400420#define PPC_INST_BHRBE 0x7c00025c#define PPC_INST_CLRBHRB 0x7c00035c#define PPC_INST_COPY 0x7c20060c
@@ -432,6 +433,7 @@#endif /* Deal with instructions that older assemblers aren't aware of */ +#define PPC_BCCTR_FLUSH stringify_in_c(.long PPC_INST_BCCTR_FLUSH)#define PPC_CP_ABORT stringify_in_c(.long PPC_INST_CP_ABORT)#define PPC_COPY (a, b) stringify_in_c(.long PPC_INST_COPY | \ ___PPC_RA(a) | ___PPC_RB(b))
arch/powerpc/kernel/entry_64.S +2 -4
@@ -261,8 +261,6 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs); 1: nop; \ patch_site 1b, patch__call_flush_branch_caches -#define BCCTR_FLUSH .long 0x4c400420 - .macro nops number .rept \number nop
@@ -293,7 +291,7 @@ flush_branch_caches: li r9,0x7fff mtctr r9 - BCCTR_FLUSH + PPC_BCCTR_FLUSH 2: nop patch_site 2b patch__flush_count_cache_return
@@ -302,7 +300,7 @@ flush_branch_caches: .rept 278 .balign 32 - BCCTR_FLUSH + PPC_BCCTR_FLUSH nops 7 .endr
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 6/7 powerpc/security: Allow for processors that flush the link stack using the special bcctr
In reply to Nicholas PigginIf both count cache and link stack are to be flushed, and can be flushed with the special bcctr, patch that in directly to the flush/branch nop site.
--- arch/powerpc/include/asm/security_features.h | 2 ++ arch/powerpc/kernel/security.c | 27 ++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-)
arch/powerpc/include/asm/security_features.h +2 -0
@@ -63,6 +63,8 @@ static inline bool security_ftr_enabled(u64 feature)// bcctr 2,0,0 triggers a hardware assisted count cache flush #define SEC_FTR_BCCTR_FLUSH_ASSIST 0x0000000000000800ull +// bcctr 2,0,0 triggers a hardware assisted link stack flush +#define SEC_FTR_BCCTR_LINK_FLUSH_ASSIST 0x0000000000002000ull// Features indicating need for Spectre/Meltdown mitigations
arch/powerpc/kernel/security.c +19 -8
@@ -219,24 +219,25 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, cif (ccd)seq_buf_printf (&s,"Indirect branch cache disabled" ); -if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_SW ) -seq_buf_printf (&s,", Software link stack flush" ); - }else if (count_cache_flush_type !=BRANCH_CACHE_FLUSH_NONE ) {seq_buf_printf (&s,"Mitigation: Software count cache flush" );if (count_cache_flush_type ==BRANCH_CACHE_FLUSH_HW )seq_buf_printf (&s," (hardware accelerated)" ); -if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_SW ) -seq_buf_printf (&s,", Software link stack flush" ); - }else if (btb_flush_enabled) {seq_buf_printf (&s,"Mitigation: Branch predictor state flush" ); }else {seq_buf_printf (&s,"Vulnerable" ); } +if (bcs || ccd || count_cache_flush_type !=BRANCH_CACHE_FLUSH_NONE ) { +if (link_stack_flush_type !=BRANCH_CACHE_FLUSH_NONE ) +seq_buf_printf (&s,", Software link stack flush" ); +if (link_stack_flush_type ==BRANCH_CACHE_FLUSH_HW ) +seq_buf_printf (&s," (hardware accelerated)" ); + } +seq_buf_printf (&s,"\n" );return s.len;
@@ -435,6 +436,7 @@ static void update_branch_cache_flush(void)patch_instruction_site (&patch__call_kvm_flush_link_stack,ppc_inst (PPC_INST_NOP )); }else { +// Could use HW flush, but that could also flush count cache patch_branch_site (&patch__call_kvm_flush_link_stack, (u64)&kvm_flush_link_stack,BRANCH_SET_LINK ); }
@@ -445,6 +447,10 @@ static void update_branch_cache_flush(void) link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {patch_instruction_site (&patch__call_flush_branch_caches,ppc_inst (PPC_INST_NOP )); + } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW && + link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) { +patch_instruction_site (&patch__call_flush_branch_caches, +ppc_inst (PPC_INST_BCCTR_FLUSH )); } else {patch_branch_site (&patch__call_flush_branch_caches, (u64)&flush_branch_caches,BRANCH_SET_LINK );
@@ -485,8 +491,13 @@ static void toggle_branch_cache_flush(bool enable)pr_info ("link-stack-flush: flush disabled.\n" ); } }else { - link_stack_flush_type =BRANCH_CACHE_FLUSH_SW ; -pr_info ("link-stack-flush: software flush enabled.\n" ); +if (security_ftr_enabled (SEC_FTR_BCCTR_LINK_FLUSH_ASSIST )) { + link_stack_flush_type =BRANCH_CACHE_FLUSH_HW ; +pr_info ("link-stack-flush: hardware flush enabled.\n" ); + }else { + link_stack_flush_type =BRANCH_CACHE_FLUSH_SW ; +pr_info ("link-stack-flush: software flush enabled.\n" ); + } }update_branch_cache_flush ();
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH 7/7 powerpc/64s: advertise hardware link stack flush
In reply to Nicholas PigginFor testing only at the moment, firmware does not define these bits.
--- arch/powerpc/include/asm/hvcall.h | 1 + arch/powerpc/include/uapi/asm/kvm.h | 1 + arch/powerpc/kvm/powerpc.c | 9 +++++++-- arch/powerpc/platforms/powernv/setup.c | 3 +++ arch/powerpc/platforms/pseries/setup.c | 3 +++ tools/arch/powerpc/include/uapi/asm/kvm.h | 1 + 6 files changed, 16 insertions(+), 2 deletions(-)
arch/powerpc/include/asm/hvcall.h +1 -0
@@ -373,6 +373,7 @@#define H_CPU_CHAR_THREAD_RECONFIG_CTRL (1ull << 57) // IBM bit 6#define H_CPU_CHAR_COUNT_CACHE_DISABLED (1ull << 56) // IBM bit 7#define H_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) // IBM bit 9 +#define H_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53) // IBM bit 10#define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0#define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1
arch/powerpc/include/uapi/asm/kvm.h +1 -0
@@ -464,6 +464,7 @@ struct kvm_ppc_cpu_char {#define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57)#define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56)#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) +#define KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53)#define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63)#define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62)
arch/powerpc/kvm/powerpc.c +7 -2
@@ -2221,7 +2221,8 @@ static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)KVM_PPC_CPU_CHAR_BR_HINT_HONOURED |KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF |KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | -KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST ; +KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST | +KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST ; cp->behaviour_mask =KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR | KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
@@ -2287,13 +2288,17 @@ static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp)if (have_fw_feat (fw_features,"enabled" ,"fw-count-cache-flush-bcctr2,0,0" )) cp->character |=KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST ; +if (have_fw_feat (fw_features,"enabled" , +"fw-link-stack-flush-bcctr2,0,0" )) + cp->character |=KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST ; cp->character_mask =KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | -KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST ; +KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST | +KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST ;if (have_fw_feat (fw_features,"enabled" ,"speculation-policy-favor-security" ))
arch/powerpc/platforms/powernv/setup.c +3 -0
@@ -78,6 +78,9 @@ static void init_fw_feat_flags(struct device_node *np)if (fw_feature_is ("enabled" ,"fw-count-cache-flush-bcctr2,0,0" , np))security_ftr_set (SEC_FTR_BCCTR_FLUSH_ASSIST ); +if (fw_feature_is ("enabled" ,"fw-link-stack-flush-bcctr2,0,0" , np)) +security_ftr_set (SEC_FTR_BCCTR_LINK_FLUSH_ASSIST ); +if (fw_feature_is ("enabled" ,"needs-count-cache-flush-on-context-switch" , np))security_ftr_set (SEC_FTR_FLUSH_COUNT_CACHE );
arch/powerpc/platforms/pseries/setup.c +3 -0
@@ -517,6 +517,9 @@ static void init_cpu_char_feature_flags(struct h_cpu_char_result *result)if (result->character &H_CPU_CHAR_BCCTR_FLUSH_ASSIST )security_ftr_set (SEC_FTR_BCCTR_FLUSH_ASSIST ); +if (result->character &H_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST ) +security_ftr_set (SEC_FTR_BCCTR_LINK_FLUSH_ASSIST ); +if (result->behaviour &H_CPU_BEHAV_FLUSH_COUNT_CACHE )security_ftr_set (SEC_FTR_FLUSH_COUNT_CACHE );
tools/arch/powerpc/include/uapi/asm/kvm.h +1 -0
@@ -464,6 +464,7 @@ struct kvm_ppc_cpu_char {#define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57)#define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56)#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) +#define KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53)#define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63)#define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62)
Signature · 1 line
2.23.0
Quoted 8 lines · Nicholas Piggin
This series allows the link stack to be flushed with the speical bcctr 2,0,0 flush instruction that also flushes the count cache if the processor supports it.
Firmware does not support this at the moment, but I've tested it in simulator with a patched firmware to advertise support.
[...]
Patches 1-6 applied to powerpc/next.
[1/7] powerpc/security: re-name count cache flush to branch cache flush
https://git.kernel.org/powerpc/c/1026798c644bfd3115fc4e32fd5e767cfc30ccf1 [2/7] powerpc/security: change link stack flush state to the flush type enum https://git.kernel.org/powerpc/c/c06ac2771070f465076e87bba262c64fb0b3aca3 [3/7] powerpc/security: make display of branch cache flush more consistent https://git.kernel.org/powerpc/c/1afe00c74ffe6d502bffa81c7d849cb4640d7ae5 [4/7] powerpc/security: split branch cache flush toggle from code patching https://git.kernel.org/powerpc/c/c0036549a9d9a060fa8bc24e31f85503ce08ad5e [5/7] powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h https://git.kernel.org/powerpc/c/70d7cdaf0548ec95fa7204dcdd39cd8e63cee24d [6/7] powerpc/security: Allow for processors that flush the link stack using the special bcctr https://git.kernel.org/powerpc/c/4d24e21cc694e7253a532fe5a9bde12b284f1317
cheers