Add support for PSTATE.SSBS to mitigate Spectre-v4
Subsystems (3)
cover Add support for PSTATE.SSBS to mitigate Spectre-v4
Hi all,
Armv8.5 introduces a new PSTATE bit, SSBS, which can be used to mitigate Spectre-v4 dynamically without trapping to EL3 firmware.
This patch hooks up support for this bit into the existing SSBD framework on CPUs that support it. The code is a maze of double-negatives but appears to work on the FastModel.
Cheers,
Will
--->8
Will Deacon (7):
arm64: Fix silly typo in comment
arm64: cpufeature: Detect SSBS and advertise to userspace
arm64: ssbd: Drop #ifdefs for PR_SPEC_STORE_BYPASS
arm64: entry: Allow handling of undefined instructions from EL1
arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3
KVM: arm64: Set SCTLR_EL2.DSSBS if SSBD is forcefully disabled and
!vhe
arm64: cpu: Move errata and feature enable callbacks closer to callers
arch/arm64/include/asm/cpucaps.h | 3 +- arch/arm64/include/asm/cpufeature.h | 2 +- arch/arm64/include/asm/kvm_host.h | 11 +++++ arch/arm64/include/asm/processor.h | 11 +++-- arch/arm64/include/asm/ptrace.h | 1 + arch/arm64/include/asm/sysreg.h | 19 ++++++-- arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/cpu_errata.c | 32 ++++++++++++- arch/arm64/kernel/cpufeature.c | 92 ++++++++++++++++++++++++++++++++---- arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/process.c | 4 ++ arch/arm64/kernel/ssbd.c | 24 ++++++++-- arch/arm64/kernel/traps.c | 16 +++---- arch/arm64/kvm/hyp/sysreg-sr.c | 11 +++++ arch/arm64/mm/fault.c | 14 ------ 17 files changed, 198 insertions(+), 47 deletions(-)
Signature · 1 line
2.1.4
PATCH 1/7 arm64: Fix silly typo in comment
I was passing through and figuered I'd fix this up:
featuer -> feature
--- arch/arm64/include/asm/cpufeature.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/cpufeature.h +1 -1
@@ -262,7 +262,7 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;/* * CPU feature detected at boot time based on system-wide value of a * feature. It is safe for a late CPU to have this feature even though -* the system hasn't enabled it, although the featuer will not be used +* the system hasn't enabled it, although the feature will not be used * by Linux in this case. If the system has enabled this feature already, * then every late CPU must have it. */
Signature · 1 line
2.1.4
PATCH 2/7 arm64: cpufeature: Detect SSBS and advertise to userspace
In reply to Will DeaconArmv8.5 introduces a new PSTATE bit known as Speculative Store Bypass Safe (SSBS) which can be used as a mitigation against Spectre variant 4.
Additionally, a CPU may provide instructions to manipulate PSTATE.SSBS directly, so that userspace can toggle the SSBS control without trapping to the kernel.
This patch probes for the existence of SSBS and advertise the new instructions to userspace if they exist.
--- arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/include/asm/sysreg.h | 16 ++++++++++++---- arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 19 +++++++++++++++++-- arch/arm64/kernel/cpuinfo.c | 1 + 5 files changed, 33 insertions(+), 7 deletions(-)
arch/arm64/include/asm/cpucaps.h +2 -1
@@ -51,7 +51,8 @@#define ARM64_SSBD 30#define ARM64_MISMATCHED_CACHE_TYPE 31#define ARM64_HAS_STAGE2_FWB 32 +#define ARM64_SSBS 33 -#define ARM64_NCAPS 33 +#define ARM64_NCAPS 34#endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/sysreg.h +12 -4
@@ -419,6 +419,7 @@#define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7)/* Common SCTLR_ELx flags. */ +#define SCTLR_ELx_DSSBS (1UL << 44)#define SCTLR_ELx_EE (1 << 25)#define SCTLR_ELx_IESB (1 << 21)#define SCTLR_ELx_WXN (1 << 19)
@@ -439,7 +440,7 @@ (1 << 10) | (1 << 13) | (1 << 14) | (1 << 15) | \ (1 << 17) | (1 << 20) | (1 << 24) | (1 << 26) | \ (1 << 27) | (1 << 30) | (1 << 31) | \ - (0xffffffffUL << 32)) + (0xffffefffUL << 32))#ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE
@@ -453,7 +454,7 @@#define SCTLR_EL2_SET (SCTLR_ELx_IESB | ENDIAN_SET_EL2 | SCTLR_EL2_RES1)#define SCTLR_EL2_CLEAR (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \ SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_WXN | \ - ENDIAN_CLEAR_EL2 | SCTLR_EL2_RES0) + SCTLR_ELx_DSSBS | ENDIAN_CLEAR_EL2 | SCTLR_EL2_RES0)#if (SCTLR_EL2_SET ^SCTLR_EL2_CLEAR ) !=0xffffffffffffffff #error "Inconsistent SCTLR_EL2 set/clear bits"
@@ -477,7 +478,7 @@ (1 << 29))#define SCTLR_EL1_RES0 ((1 << 6) | (1 << 10) | (1 << 13) | (1 << 17) | \ (1 << 27) | (1 << 30) | (1 << 31) | \ - (0xffffffffUL << 32)) + (0xffffefffUL << 32))#ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL1 (SCTLR_EL1_E0E | SCTLR_ELx_EE)
@@ -494,7 +495,7 @@ ENDIAN_SET_EL1 | SCTLR_EL1_UCI | SCTLR_EL1_RES1)#define SCTLR_EL1_CLEAR (SCTLR_ELx_A | SCTLR_EL1_CP15BEN | SCTLR_EL1_ITD |\ SCTLR_EL1_UMA | SCTLR_ELx_WXN | ENDIAN_CLEAR_EL1 |\ - SCTLR_EL1_RES0) + SCTLR_ELx_DSSBS | SCTLR_EL1_RES0)#if (SCTLR_EL1_SET ^SCTLR_EL1_CLEAR ) !=0xffffffffffffffff #error "Inconsistent SCTLR_EL1 set/clear bits"
@@ -544,6 +545,13 @@#define ID_AA64PFR0_EL0_64BIT_ONLY 0x1#define ID_AA64PFR0_EL0_32BIT_64BIT 0x2 +/* id_aa64pfr1 */ +#define ID_AA64PFR1_SSBS_SHIFT 4 + +#define ID_AA64PFR1_SSBS_PSTATE_NI 0 +#define ID_AA64PFR1_SSBS_PSTATE_ONLY 1 +#define ID_AA64PFR1_SSBS_PSTATE_INSNS 2 +/* id_aa64mmfr0 */ #define ID_AA64MMFR0_TGRAN4_SHIFT 28#define ID_AA64MMFR0_TGRAN64_SHIFT 24
arch/arm64/include/uapi/asm/hwcap.h +1 -0
@@ -48,5 +48,6 @@#define HWCAP_USCAT (1 << 25)#define HWCAP_ILRCPC (1 << 26)#define HWCAP_FLAGM (1 << 27) +#define HWCAP_SSBS (1 << 28)#endif /* _UAPI__ASM_HWCAP_H */
arch/arm64/kernel/cpufeature.c +17 -2
@@ -164,6 +164,11 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { +ARM64_FTR_BITS (FTR_VISIBLE ,FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64PFR1_SSBS_SHIFT ,4 ,ID_AA64PFR1_SSBS_PSTATE_NI ), +ARM64_FTR_END , +}; +static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {S_ARM64_FTR_BITS (FTR_HIDDEN ,FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64MMFR0_TGRAN4_SHIFT ,4 ,ID_AA64MMFR0_TGRAN4_NI ),S_ARM64_FTR_BITS (FTR_HIDDEN ,FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64MMFR0_TGRAN64_SHIFT ,4 ,ID_AA64MMFR0_TGRAN64_NI ),
@@ -371,7 +376,7 @@ static const struct __ftr_reg_entry {/* Op1 = 0, CRn = 0, CRm = 4 */ ARM64_FTR_REG (SYS_ID_AA64PFR0_EL1 , ftr_id_aa64pfr0), -ARM64_FTR_REG (SYS_ID_AA64PFR1_EL1 , ftr_raz), +ARM64_FTR_REG (SYS_ID_AA64PFR1_EL1 , ftr_id_aa64pfr1), ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),/* Op1 = 0, CRn = 0, CRm = 5 */
@@ -657,7 +662,6 @@ void update_cpu_features(int cpu,/* * EL3 is not our concern. -* ID_AA64PFR1 is currently RES0. */ taint |=check_update_ftr_reg (SYS_ID_AA64PFR0_EL1 , cpu, info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
@@ -1222,6 +1226,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_hw_dbm, },#endif + { + .desc = "Speculative Store Bypassing Safe (SSBS)", + .capability = ARM64_SSBS, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64PFR1_EL1, + .field_pos = ID_AA64PFR1_SSBS_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY, + }, {}, };
@@ -1267,6 +1281,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {#ifdef CONFIG_ARM64_SVE HWCAP_CAP (SYS_ID_AA64PFR0_EL1 ,ID_AA64PFR0_SVE_SHIFT ,FTR_UNSIGNED ,ID_AA64PFR0_SVE ,CAP_HWCAP ,HWCAP_SVE ), #endif + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS), {}, };
arch/arm64/kernel/cpuinfo.c +1 -0
@@ -81,6 +81,7 @@ static const char *const hwcap_str[] = {"uscat" ,"ilrcpc" ,"flagm" , +"ssbs" , NULL };
Signature · 1 line
2.1.4
Suzuki K Poulose
reviewerPATCH 2/7 arm64: cpufeature: Detect SSBS and advertise to userspace
Quoted 10 lines · Will Deacon
Armv8.5 introduces a new PSTATE bit known as Speculative Store Bypass Safe (SSBS) which can be used as a mitigation against Spectre variant 4.
Additionally, a CPU may provide instructions to manipulate PSTATE.SSBS directly, so that userspace can toggle the SSBS control without trapping to the kernel.
This patch probes for the existence of SSBS and advertise the new instructions to userspace if they exist.
Quoted 9 lines
Signed-off-by Will Deacon <[email protected]>--- arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/include/asm/sysreg.h | 16 ++++++++++++---- arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 19 +++++++++++++++++-- arch/arm64/kernel/cpuinfo.c | 1 + 5 files changed, 33 insertions(+), 7 deletions(-)
PATCH 3/7 arm64: ssbd: Drop #ifdefs for PR_SPEC_STORE_BYPASS
In reply to Will DeaconNow that we're all merged nicely into mainline, there's no need to check to see if PR_SPEC_STORE_BYPASS is defined.
--- arch/arm64/kernel/ssbd.c | 3 --- 1 file changed, 3 deletions(-)
arch/arm64/kernel/ssbd.c +0 -3
@@ -11,9 +11,7 @@/* * prctl interface for SSBD -* FIXME: Drop the below ifdefery once merged in 4.18. */ -#ifdef PR_SPEC_STORE_BYPASS static int ssbd_prctl_set (struct task_struct *task,unsigned long ctrl) {int state =arm64_get_ssbd_state ();
@@ -107,4 +105,3 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)return -ENODEV ; } } -#endif /* PR_SPEC_STORE_BYPASS */
Signature · 1 line
2.1.4
PATCH 4/7 arm64: entry: Allow handling of undefined instructions from EL1
In reply to Will DeaconRather than panic() when taking an undefined instruction exception from EL1, allow a hook to be registered in case we want to emulate the instruction, like we will for the SSBS PSTATE manipulation instructions.
--- arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/traps.c | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-)
arch/arm64/kernel/entry.S +1 -1
@@ -589,7 +589,7 @@ el1_undef: inherit_daif pstate=x23, tmp=x2 mov x0, sp bl do_undefinstr - ASM_BUG() + kernel_exit 1 el1_dbg: /* * Debug exception handling
arch/arm64/kernel/traps.c +7 -4
@@ -310,10 +310,12 @@ static int call_undef_hook(struct pt_regs *regs)int (*fn)(struct pt_regs *regs,u32 instr) =NULL ;void __user *pc = (void __user *)instruction_pointer (regs); -if (!user_mode (regs)) -return 1 ; - -if (compat_thumb_mode (regs)) { +if (!user_mode (regs)) { +__le32 instr_le; +if (probe_kernel_address ((__force __le32 *)pc, instr_le)) + goto exit; + instr =le32_to_cpu (instr_le); + }else if (compat_thumb_mode (regs)) {/* 16-bit Thumb instruction */ __le16 instr_le;if (get_user (instr_le, (__le16 __user *)pc))
@@ -407,6 +409,7 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs)return ;force_signal_inject (SIGILL ,ILL_ILLOPC , regs->pc); +BUG_ON (!user_mode (regs)); }void cpu_enable_cache_maint_trap (const struct arm64_cpu_capabilities *__unused)
Signature · 1 line
2.1.4
PATCH 5/7 arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3
In reply to Will DeaconOn CPUs with support for PSTATE.SSBS, the kernel can toggle the SSBD state without needing to call into firmware.
This patch hooks into the existing SSBD infrastructure so that SSBS is used on CPUs that support it, but it's all made horribly complicated by the very real possibility of big/little systems that don't uniformly provide the new capability.
--- arch/arm64/include/asm/processor.h | 7 ++++++ arch/arm64/include/asm/ptrace.h | 1 + arch/arm64/include/asm/sysreg.h | 3 +++ arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/cpu_errata.c | 26 +++++++++++++++++++-- arch/arm64/kernel/cpufeature.c | 45 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 4 ++++ arch/arm64/kernel/ssbd.c | 21 +++++++++++++++++ 8 files changed, 106 insertions(+), 2 deletions(-)
arch/arm64/include/asm/processor.h +7 -0
@@ -174,6 +174,10 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, {start_thread_common (regs, pc); regs->pstate = PSR_MODE_EL0t; + +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_SSBS_BIT ; + regs->sp = sp; }
@@ -190,6 +194,9 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, regs->pstate |=PSR_AA32_E_BIT ;#endif +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_AA32_SSBS_BIT ; + regs->compat_sp = sp; }#endif
arch/arm64/include/asm/ptrace.h +1 -0
@@ -50,6 +50,7 @@#define PSR_AA32_I_BIT 0x00000080#define PSR_AA32_A_BIT 0x00000100#define PSR_AA32_E_BIT 0x00000200 +#define PSR_AA32_SSBS_BIT 0x00800000#define PSR_AA32_DIT_BIT 0x01000000#define PSR_AA32_Q_BIT 0x08000000#define PSR_AA32_V_BIT 0x10000000
arch/arm64/include/asm/sysreg.h +3 -0
@@ -86,11 +86,14 @@#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) +#define REG_PSTATE_SSBS_IMM sys_reg(0, 3, 4, 0, 1)#define SET_PSTATE_PAN (x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \ (!!x)<<8 | 0x1f)#define SET_PSTATE_UAO (x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \ (!!x)<<8 | 0x1f) +#define SET_PSTATE_SSBS (x) __emit_inst(0xd5000000 | REG_PSTATE_SSBS_IMM | \ + (!!x)<<8 | 0x1f)#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)
arch/arm64/include/uapi/asm/ptrace.h +1 -0
@@ -46,6 +46,7 @@#define PSR_I_BIT 0x00000080#define PSR_A_BIT 0x00000100#define PSR_D_BIT 0x00000200 +#define PSR_SSBS_BIT 0x00001000#define PSR_PAN_BIT 0x00400000#define PSR_UAO_BIT 0x00800000#define PSR_V_BIT 0x10000000
arch/arm64/kernel/cpu_errata.c +24 -2
@@ -312,6 +312,14 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt,void arm64_set_ssbd_mitigation (bool state) { +if (this_cpu_has_cap (ARM64_SSBS )) { +if (state) + asm volatile(SET_PSTATE_SSBS(0)); +else + asm volatile(SET_PSTATE_SSBS(1)); +return ; + } +switch (psci_ops.conduit) {case PSCI_CONDUIT_HVC :arm_smccc_1_1_hvc (ARM_SMCCC_ARCH_WORKAROUND_2 , state,NULL );
@@ -336,6 +344,11 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,WARN_ON (scope !=SCOPE_LOCAL_CPU ||preemptible ()); +if (this_cpu_has_cap (ARM64_SSBS )) { + required = false; + goto out_printmsg; + } +if (psci_ops.smccc_version ==SMCCC_VERSION_1_0 ) { ssbd_state =ARM64_SSBD_UNKNOWN ;return false;
@@ -384,7 +397,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,switch (ssbd_state) {case ARM64_SSBD_FORCE_DISABLE : -pr_info_once ("%s disabled from command-line\n" , entry->desc);arm64_set_ssbd_mitigation (false); required = false;break ;
@@ -397,7 +409,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,break ;case ARM64_SSBD_FORCE_ENABLE : -pr_info_once ("%s forced from command-line\n" , entry->desc);arm64_set_ssbd_mitigation (true); required = true;break ;
@@ -407,6 +418,17 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, break; } +out_printmsg: +switch (ssbd_state) { +case ARM64_SSBD_FORCE_DISABLE : +pr_info_once ("%s disabled from command-line\n" , entry->desc); +break ; + +case ARM64_SSBD_FORCE_ENABLE : +pr_info_once ("%s forced from command-line\n" , entry->desc); +break ; + } +return required; }#endif /* CONFIG_ARM64_SSBD */
arch/arm64/kernel/cpufeature.c +45 -0
64 lines not on this page · raw message
arch/arm64/kernel/process.c +4 -0
@@ -358,6 +358,10 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,if (IS_ENABLED (CONFIG_ARM64_UAO ) &&cpus_have_const_cap (ARM64_HAS_UAO )) childregs->pstate |=PSR_UAO_BIT ; + +if (arm64_get_ssbd_state () ==ARM64_SSBD_FORCE_DISABLE ) + childregs->pstate |=PSR_SSBS_BIT ; + p->thread.cpu_context.x19 = stack_start; p->thread.cpu_context.x20 = stk_sz; }
arch/arm64/kernel/ssbd.c +21 -0
@@ -3,12 +3,30 @@* Copyright (C) 2018 ARM Ltd, All Rights Reserved. */ +#include <linux/compat.h> #include <linux/errno.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/thread_info.h> #include <asm/cpufeature.h> +static void ssbd_ssbs_enable (struct task_struct *task) +{ +u64 val =is_compat_thread (task_thread_info (task)) ? +PSR_AA32_SSBS_BIT :PSR_SSBS_BIT ; + +task_pt_regs (task)->pstate |= val; +} + +static void ssbd_ssbs_disable (struct task_struct *task) +{ +u64 val =is_compat_thread (task_thread_info (task)) ? +PSR_AA32_SSBS_BIT :PSR_SSBS_BIT ; + +task_pt_regs (task)->pstate &= ~val; +} +/* * prctl interface for SSBD */
@@ -44,12 +62,14 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)return -EPERM ;task_clear_spec_ssb_disable (task);clear_tsk_thread_flag (task,TIF_SSBD ); +ssbd_ssbs_enable (task);break ;case PR_SPEC_DISABLE :if (state ==ARM64_SSBD_FORCE_DISABLE )return -EPERM ;task_set_spec_ssb_disable (task);set_tsk_thread_flag (task,TIF_SSBD ); +ssbd_ssbs_disable (task);break ;case PR_SPEC_FORCE_DISABLE :if (state ==ARM64_SSBD_FORCE_DISABLE )
@@ -57,6 +77,7 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)task_set_spec_ssb_disable (task);task_set_spec_ssb_force_disable (task);set_tsk_thread_flag (task,TIF_SSBD ); +ssbd_ssbs_disable (task);break ;default :return -ERANGE ;
Signature · 1 line
2.1.4
Suzuki K Poulose
reviewerPATCH 5/7 arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3
Hi Will,
The patch looks good to me, some comments below.
Quoted 62 lines · Will Deacon
On CPUs with support for PSTATE.SSBS, the kernel can toggle the SSBD state without needing to call into firmware.
This patch hooks into the existing SSBD infrastructure so that SSBS is used on CPUs that support it, but it's all made horribly complicated by the very real possibility of big/little systems that don't uniformly provide the new capability.
Signed-off-by Will Deacon <[email protected]>--- arch/arm64/include/asm/processor.h | 7 ++++++ arch/arm64/include/asm/ptrace.h | 1 + arch/arm64/include/asm/sysreg.h | 3 +++ arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/cpu_errata.c | 26 +++++++++++++++++++-- arch/arm64/kernel/cpufeature.c | 45 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 4 ++++ arch/arm64/kernel/ssbd.c | 21 +++++++++++++++++ 8 files changed, 106 insertions(+), 2 deletions(-)arch/arm64/include/asm/processor.h +7
-0@@ -174,6 +174,10 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, {start_thread_common (regs, pc); regs->pstate = PSR_MODE_EL0t; + +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_SSBS_BIT ; + regs->sp = sp; }@@ -190,6 +194,9 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, regs->pstate |=PSR_AA32_E_BIT ;#endif +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_AA32_SSBS_BIT ; + regs->compat_sp = sp; }#endif arch/arm64/include/asm/ptrace.h +1
-0@@ -50,6 +50,7 @@#define PSR_AA32_I_BIT 0x00000080#define PSR_AA32_A_BIT 0x00000100#define PSR_AA32_E_BIT 0x00000200 +#define PSR_AA32_SSBS_BIT 0x00800000#define PSR_AA32_DIT_BIT 0x01000000#define PSR_AA32_Q_BIT 0x08000000#define PSR_AA32_V_BIT 0x10000000arch/arm64/include/asm/sysreg.h +0
-0
arch/arm64/include/asm/sysreg.h line 89
Quoted 4 lines
arch/arm64/include/asm/sysreg.h +1
-0@@ -86,11 +86,14 @@#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) +#define REG_PSTATE_SSBS_IMM sys_reg(0, 3, 4, 0, 1)
nit: This is not about the patch, but what we have already established for the macros above.
It looks like these definitions are a bit confusing if you try to correlate it with the Arm ARM. For "msr <pstate_field>, #Immediate", we have :
pstate_field => encoded in Op1:Op2 only
#Immediate => encoded in CRm field.
And Crn = 4, op0 = 0 and they are not specifically part of any standard encoding for the "fields", but for the instruction access. (e.g, they don't appear in an ISS for an ESR_ELx unlike the other sys regs).
So, using the sys_reg() definitions above somehow hides these facts and makes it a bit difficult to verify the code. I am wondering if we should introduce something like :
#define pstate_field(op1, op2) (((op1) << Op1_shift) | ((op2) << Op2_shift))
#define PSTATE_Imm_shift CRm_shift
#define PSTATE_PAN pstate_field(0, 4)
#define PSTATE_UAO pstate_field(0, 3)
#define PSTATE_SSBS pstate_field(3, 1)
#define SET_PSTATE_FIELD(field, val) \ __emit_insn(0xd500401f | field | ((val) << PSTATE_Imm_shift))
#define SET_PSTATE_PAN(x) SET_PSTATE_FIELD(PSTATE_PAN, !!(x))
etc.
Of course this has to be a separate patch which I can send if you think it is worth pursuing.
Or@least we should replace the '8' below in the __emit_inst with a symbolic name and use it wherever we need.
Quoted 27 lines
arch/arm64/include/asm/sysreg.h +2
-0#define SET_PSTATE_PAN (x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM | \ (!!x)<<8 | 0x1f)#define SET_PSTATE_UAO (x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM | \ (!!x)<<8 | 0x1f) +#define SET_PSTATE_SSBS (x) __emit_inst(0xd5000000 | REG_PSTATE_SSBS_IMM | \ + (!!x)<<8 | 0x1f)#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)arch/arm64/include/uapi/asm/ptrace.h +1
-0@@ -46,6 +46,7 @@#define PSR_I_BIT 0x00000080#define PSR_A_BIT 0x00000100#define PSR_D_BIT 0x00000200 +#define PSR_SSBS_BIT 0x00001000#define PSR_PAN_BIT 0x00400000#define PSR_UAO_BIT 0x00800000#define PSR_V_BIT 0x10000000arch/arm64/kernel/cpu_errata.c +0
-0
arch/arm64/kernel/cpu_errata.c void __init arm64_enable_wa2_handling(struct alt_instr *alt, · line 315
Quoted 4 lines
arch/arm64/kernel/cpu_errata.c +1
-0@@ -312,6 +312,14 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt,void arm64_set_ssbd_mitigation (bool state) { +if (this_cpu_has_cap (ARM64_SSBS )) {
nit: This will take a longer time as it iterates over the entire *arm64_features* and *arm64_errata* list until it finds the "cap" requested to run the check. If we tend to use this function a lot more and expect it to do it quickly, we may simply make the direct call to the matches or add a wrapper to do the same.
Quoted 71 lines
arch/arm64/kernel/cpu_errata.c +23
-2+if (state) + asm volatile(SET_PSTATE_SSBS(0)); +else + asm volatile(SET_PSTATE_SSBS(1)); +return ; + } +switch (psci_ops.conduit) {case PSCI_CONDUIT_HVC :arm_smccc_1_1_hvc (ARM_SMCCC_ARCH_WORKAROUND_2 , state,NULL );@@ -336,6 +344,11 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,WARN_ON (scope !=SCOPE_LOCAL_CPU ||preemptible ()); +if (this_cpu_has_cap (ARM64_SSBS )) { + required = false; + goto out_printmsg; + } +if (psci_ops.smccc_version ==SMCCC_VERSION_1_0 ) { ssbd_state =ARM64_SSBD_UNKNOWN ;return false;@@ -384,7 +397,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,switch (ssbd_state) {case ARM64_SSBD_FORCE_DISABLE : -pr_info_once ("%s disabled from command-line\n" , entry->desc);arm64_set_ssbd_mitigation (false); required = false;break ;@@ -397,7 +409,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,break ;case ARM64_SSBD_FORCE_ENABLE : -pr_info_once ("%s forced from command-line\n" , entry->desc);arm64_set_ssbd_mitigation (true); required = true;break ;@@ -407,6 +418,17 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, break; } +out_printmsg: +switch (ssbd_state) { +case ARM64_SSBD_FORCE_DISABLE : +pr_info_once ("%s disabled from command-line\n" , entry->desc); +break ; + +case ARM64_SSBD_FORCE_ENABLE : +pr_info_once ("%s forced from command-line\n" , entry->desc); +break ; + } +return required; }#endif /* CONFIG_ARM64_SSBD */ arch/arm64/kernel/cpufeature.c +7
-0@@ -1039,6 +1039,48 @@ static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)WARN_ON (val & (7 <<27 |7 <<21 )); } +#ifdef CONFIG_ARM64_SSBD +static int ssbs_emulation_handler (struct pt_regs *regs,u32 instr) +{ +if (user_mode (regs)) +return 1 ; + +if (instr &BIT (CRm_shift))
arch/arm64/kernel/cpufeature.c static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused) · line 1058
Quoted 10 lines
arch/arm64/kernel/cpufeature.c +10
-0+ regs->pstate |=PSR_SSBS_BIT ; + else + regs->pstate &= ~PSR_SSBS_BIT; + +arm64_skip_faulting_instruction (regs,4 ); +return 0 ; +} + +static struct undef_hook ssbs_emulation_hook = { + .instr_mask = ~(1U << CRm_shift),
These lines above triggered the above detailed "nit" part.
arch/arm64/kernel/cpufeature.c static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused) · line 1062
Quoted 4 lines
arch/arm64/kernel/cpufeature.c +4
-0+ .instr_val = 0xd500001f | REG_PSTATE_SSBS_IMM, + .fn = ssbs_emulation_handler, +}; +
Suzuki
PATCH 5/7 arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3
Hi Suzuki,
Quoted 93 lines · Suzuki K Poulose
Quoted 60 lines · Will Deacon
On CPUs with support for PSTATE.SSBS, the kernel can toggle the SSBD state without needing to call into firmware.
This patch hooks into the existing SSBD infrastructure so that SSBS is used on CPUs that support it, but it's all made horribly complicated by the very real possibility of big/little systems that don't uniformly provide the new capability.
Signed-off-by Will Deacon <[email protected]>--- arch/arm64/include/asm/processor.h | 7 ++++++ arch/arm64/include/asm/ptrace.h | 1 + arch/arm64/include/asm/sysreg.h | 3 +++ arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/cpu_errata.c | 26 +++++++++++++++++++-- arch/arm64/kernel/cpufeature.c | 45 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 4 ++++ arch/arm64/kernel/ssbd.c | 21 +++++++++++++++++ 8 files changed, 106 insertions(+), 2 deletions(-)arch/arm64/include/asm/processor.h +7
-0@@ -174,6 +174,10 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, {start_thread_common (regs, pc); regs->pstate = PSR_MODE_EL0t; + +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_SSBS_BIT ; + regs->sp = sp; }@@ -190,6 +194,9 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, regs->pstate |=PSR_AA32_E_BIT ;#endif +if (arm64_get_ssbd_state () !=ARM64_SSBD_FORCE_ENABLE ) + regs->pstate |=PSR_AA32_SSBS_BIT ; + regs->compat_sp = sp; }#endif arch/arm64/include/asm/ptrace.h +1
-0@@ -50,6 +50,7 @@#define PSR_AA32_I_BIT 0x00000080#define PSR_AA32_A_BIT 0x00000100#define PSR_AA32_E_BIT 0x00000200 +#define PSR_AA32_SSBS_BIT 0x00800000#define PSR_AA32_DIT_BIT 0x01000000#define PSR_AA32_Q_BIT 0x08000000#define PSR_AA32_V_BIT 0x10000000arch/arm64/include/asm/sysreg.h +0
-0arch/arm64/include/asm/sysreg.h line 88
Quoted 3 lines
arch/arm64/include/asm/sysreg.h +1
-0@@ -86,11 +86,14 @@#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) +#define REG_PSTATE_SSBS_IMM sys_reg(0, 3, 4, 0, 1)nit: This is not about the patch, but what we have already established for the macros above.
It looks like these definitions are a bit confusing if you try to correlate it with the Arm ARM. For "msr <
pstate_field>, #Immediate", we have :
pstate_field=> encoded in Op1:Op2 only
#Immediate => encoded in CRm field.And Crn = 4, op0 = 0 and they are not specifically part of any standard encoding for the "fields", but for the instruction access. (e.g, they don't appear in an ISS for an
ESR_ELxunlike the other sys regs).So, using the
sys_reg()definitions above somehow hides these facts and makes it a bit difficult to verify the code. I am wondering if we should introduce something like :#define
pstate_field(op1, op2)(((op1) <<Op1_shift) | ((op2) <<Op2_shift))
#definePSTATE_Imm_shiftCRm_shift#define
PSTATE_PANpstate_field(0, 4)
#definePSTATE_UAOpstate_field(0, 3)
#definePSTATE_SSBSpstate_field(3, 1)#define SET_PSTATE_FIELD(field, val) \ __emit_insn(0xd500401f | field | ((val) << PSTATE_Imm_shift))
Quoted 6 lines
#define
SET_PSTATE_PAN(x)SET_PSTATE_FIELD(PSTATE_PAN, !!(x))etc.
Of course this has to be a separate patch which I can send if you think it is worth pursuing.
I think this would be a useful cleanup, so please go ahead! I suspect I'll have to backport my stuff to stable, so having the cleanup on top would be best.
Quoted 5 lines
Quoted 5 lines
arch/arm64/kernel/cpu_errata.c +0
-0@@ -312,6 +312,14 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt,
Quoted 8 lines
arch/arm64/kernel/cpu_errata.c void __init arm64_enable_wa2_handling(struct alt_instr *alt, · line 315
Quoted 3 lines
arch/arm64/kernel/cpu_errata.c +1
-0void arm64_set_ssbd_mitigation (bool state) { +if (this_cpu_has_cap (ARM64_SSBS )) {nit: This will take a longer time as it iterates over the entire *arm64_features* and *arm64_errata* list until it finds the "cap" requested to run the check. If we tend to use this function a lot more and expect it to do it quickly, we may simply make the direct call to the matches or add a wrapper to do the same.
I don't think we need to treat this as too much of a hot path, so it should be fine.
Will
PATCH 6/7 KVM: arm64: Set SCTLR_EL2.DSSBS if SSBD is forcefully disabled and !vhe
In reply to Will DeaconWhen running without VHE, it is necessary to set SCTLR_EL2.DSSBS if SSBD has been forcefully disabled on the kernel command-line.
--- arch/arm64/include/asm/kvm_host.h | 11 +++++++++++ arch/arm64/kvm/hyp/sysreg-sr.c | 11 +++++++++++ 2 files changed, 22 insertions(+)
arch/arm64/include/asm/kvm_host.h +11 -0
@@ -389,6 +389,8 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);DECLARE_PER_CPU (kvm_cpu_context_t, kvm_host_cpu_state); +void __kvm_enable_ssbs (void ); +static inline void __cpu_init_hyp_mode (phys_addr_t pgd_ptr,unsigned long hyp_stack_ptr,unsigned long vector_ptr)
@@ -409,6 +411,15 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,*/ BUG_ON (!static_branch_likely (&arm64_const_caps_ready));__kvm_call_hyp ((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2); + +/* +* Disabling SSBD on a non-VHE system requires us to enable SSBS +* at EL2. +*/ +if (!has_vhe () &&this_cpu_has_cap (ARM64_SSBS ) && +arm64_get_ssbd_state () ==ARM64_SSBD_FORCE_DISABLE ) { +kvm_call_hyp (__kvm_enable_ssbs); + } }static inline bool kvm_arch_check_sve_has_vhe (void )
arch/arm64/kvm/hyp/sysreg-sr.c +11 -0
@@ -288,3 +288,14 @@ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) vcpu->arch.sysregs_loaded_on_cpu = false; } + +void __hyp_text __kvm_enable_ssbs(void) +{ +u64 tmp; + + asmvolatile ( +"mrs %0, sctlr_el2\n" +"orr %0, %0, %1\n" +"msr sctlr_el2, %0" + :"=&r" (tmp) :"L" (SCTLR_ELx_DSSBS)); +}
Signature · 1 line
2.1.4
PATCH 6/7 KVM: arm64: Set SCTLR_EL2.DSSBS if SSBD is forcefully disabled and !vhe
Quoted 4 lines · Will Deacon
When running without VHE, it is necessary to set SCTLR_EL2.DSSBS if SSBD has been forcefully disabled on the kernel command-line.
Signed-off-by Will Deacon <[email protected]>
Quoted 56 lines
--- arch/arm64/include/asm/kvm_host.h | 11 +++++++++++ arch/arm64/kvm/hyp/sysreg-sr.c | 11 +++++++++++ 2 files changed, 22 insertions(+)arch/arm64/include/asm/kvm_host.h +11
-0@@ -389,6 +389,8 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);DECLARE_PER_CPU (kvm_cpu_context_t, kvm_host_cpu_state); +void __kvm_enable_ssbs (void ); +static inline void __cpu_init_hyp_mode (phys_addr_t pgd_ptr,unsigned long hyp_stack_ptr,unsigned long vector_ptr)@@ -409,6 +411,15 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,*/ BUG_ON (!static_branch_likely (&arm64_const_caps_ready));__kvm_call_hyp ((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2); + +/* +* Disabling SSBD on a non-VHE system requires us to enable SSBS +* at EL2. +*/ +if (!has_vhe () &&this_cpu_has_cap (ARM64_SSBS ) && +arm64_get_ssbd_state () ==ARM64_SSBD_FORCE_DISABLE ) { +kvm_call_hyp (__kvm_enable_ssbs); + } }static inline bool kvm_arch_check_sve_has_vhe (void )arch/arm64/kvm/hyp/sysreg-sr.c +11
-0@@ -288,3 +288,14 @@ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) vcpu->arch.sysregs_loaded_on_cpu = false; } + +void __hyp_text __kvm_enable_ssbs(void) +{ +u64 tmp; + + asmvolatile ( +"mrs %0, sctlr_el2\n" +"orr %0, %0, %1\n" +"msr sctlr_el2, %0" + :"=&r" (tmp) :"L" (SCTLR_ELx_DSSBS)); +}Signature · 1 line
2.1.4
PATCH 7/7 arm64: cpu: Move errata and feature enable callbacks closer to callers
In reply to Will DeaconThe cpu errata and feature enable callbacks are only called via their respective arm64_cpu_capabilities structure and therefore shouldn't exist in the global namespace.
Move the PAN, RAS and cache maintenance emulation enable callbacks into the same files as their corresponding arm64_cpu_capabilities structures, making them static in the process.
--- arch/arm64/include/asm/processor.h | 4 ---- arch/arm64/kernel/cpu_errata.c | 6 ++++++ arch/arm64/kernel/cpufeature.c | 28 ++++++++++++++++++++++------ arch/arm64/kernel/traps.c | 5 ----- arch/arm64/mm/fault.c | 14 -------------- 5 files changed, 28 insertions(+), 29 deletions(-)
arch/arm64/include/asm/processor.h +0 -4
@@ -251,10 +251,6 @@ static inline void spin_lock_prefetch(const void *ptr)#endif -void cpu_enable_pan (const struct arm64_cpu_capabilities *__unused); -void cpu_enable_cache_maint_trap (const struct arm64_cpu_capabilities *__unused); -void cpu_clear_disr (const struct arm64_cpu_capabilities *__unused); -extern unsigned long __ro_after_init signal_minsigstksz;/* sigframe size */ extern void __init minsigstksz_setup (void );
arch/arm64/kernel/cpu_errata.c +6 -0
@@ -433,6 +433,12 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, }#endif /* CONFIG_ARM64_SSBD */ +static void __maybe_unused +cpu_enable_cache_maint_trap (const struct arm64_cpu_capabilities *__unused) +{ +sysreg_clear_set (sctlr_el1,SCTLR_EL1_UCI ,0 ); +} +#define CAP_MIDR_RANGE (model, v_min, r_min, v_max, r_max) \ .matches = is_affected_midr_range, \ .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
arch/arm64/kernel/cpufeature.c +22 -6
@@ -1081,6 +1081,28 @@ static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused) }#endif /* CONFIG_ARM64_SSBD */ +#ifdef CONFIG_ARM64_PAN +static void cpu_enable_pan (const struct arm64_cpu_capabilities *__unused) +{ +/* +* We modify PSTATE. This won't work from irq context as the PSTATE +* is discarded once we return from the exception. +*/ +WARN_ON_ONCE (in_interrupt ()); + +sysreg_clear_set (sctlr_el1,SCTLR_EL1_SPAN ,0 ); + asm(SET_PSTATE_PAN(1)); +} +#endif /* CONFIG_ARM64_PAN */ + +#ifdef CONFIG_ARM64_RAS_EXTN +static void cpu_clear_disr (const struct arm64_cpu_capabilities *__unused) +{ +/* Firmware may have left a deferred SError in this register. */ +write_sysreg_s (0 ,SYS_DISR_EL1 ); +} +#endif /* CONFIG_ARM64_RAS_EXTN */ +static const struct arm64_cpu_capabilities arm64_features[] = { { .desc ="GIC system register CPU interface" ,
@@ -1815,9 +1837,3 @@ static int __init enable_mrs_emulation(void) }core_initcall (enable_mrs_emulation); - -void cpu_clear_disr (const struct arm64_cpu_capabilities *__unused) -{ -/* Firmware may have left a deferred SError in this register. */ -write_sysreg_s (0 ,SYS_DISR_EL1 ); -}
arch/arm64/kernel/traps.c +0 -5
@@ -412,11 +412,6 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs)BUG_ON (!user_mode (regs)); } -void cpu_enable_cache_maint_trap (const struct arm64_cpu_capabilities *__unused) -{ -sysreg_clear_set (sctlr_el1,SCTLR_EL1_UCI ,0 ); -} -#define __user_cache_maint (insn, address, res) \ if (address >= user_addr_max()) { \ res = -EFAULT; \
arch/arm64/mm/fault.c +0 -14
@@ -864,17 +864,3 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,return rv; }NOKPROBE_SYMBOL (do_debug_exception); - -#ifdef CONFIG_ARM64_PAN -void cpu_enable_pan (const struct arm64_cpu_capabilities *__unused) -{ -/* -* We modify PSTATE. This won't work from irq context as the PSTATE -* is discarded once we return from the exception. -*/ -WARN_ON_ONCE (in_interrupt ()); - -sysreg_clear_set (sctlr_el1,SCTLR_EL1_SPAN ,0 ); - asm(SET_PSTATE_PAN(1)); -} -#endif /* CONFIG_ARM64_PAN */
Signature · 1 line
2.1.4