arm64: entry: Place an SB sequence following an ERET instruction
Subsystems (2)
v3 cover arm64: entry: Place an SB sequence following an ERET instruction
Changes in v3:
- included missing preliminary patch to define the SB barrier instruction, see patch change log for details on how it was back ported into v4.9
Changes in v2:
- added missing hunk in hyp/entry.S per Will's feedback
Will Deacon (2):
arm64: Add support for SB barrier and patch in over DSB; ISB sequences
arm64: entry: Place an SB sequence following an ERET instruction
arch/arm64/include/asm/assembler.h | 13 +++++++++++++ arch/arm64/include/asm/barrier.h | 4 ++++ arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/include/asm/sysreg.h | 13 +++++++++++++ arch/arm64/include/asm/uaccess.h | 3 +-- arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 22 +++++++++++++++++++++- arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/kernel/entry.S | 2 ++ arch/arm64/kvm/hyp/entry.S | 2 ++ arch/arm64/kvm/hyp/hyp-entry.S | 4 ++++ 11 files changed, 64 insertions(+), 4 deletions(-)
Signature · 6 lines
2.7.4 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v3 1/2 arm64: Add support for SB barrier and patch in over DSB; ISB sequences
From: Will Deacon <[email protected]>
commit bd4fb6d270bc423a9a4098108784f7f9254c4e6d upstream
We currently use a DSB; ISB sequence to inhibit speculation in set_fs(). Whilst this works for current CPUs, future CPUs may implement a new SB barrier instruction which acts as an architected speculation barrier.
On CPUs that support it, patch in an SB; NOP sequence over the DSB; ISB sequence and advertise the presence of the new instruction to userspace.
[florian:
- adjust number capabilities
- wire decoding of ftr_id_aa64_isar1
- __emit_inst -> __inst_arm
- sys_insn -> sys_reg]
--- arch/arm64/include/asm/assembler.h | 13 +++++++++++++ arch/arm64/include/asm/barrier.h | 4 ++++ arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/include/asm/sysreg.h | 13 +++++++++++++ arch/arm64/include/asm/uaccess.h | 3 +-- arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 22 +++++++++++++++++++++- arch/arm64/kernel/cpuinfo.c | 1 + 8 files changed, 56 insertions(+), 4 deletions(-)
arch/arm64/include/asm/assembler.h +13 -0
@@ -95,6 +95,19 @@ .endm /* + * Speculation barrier + */ + .macro sb +alternative_if_not ARM64_HAS_SB + dsb nsh +isb +alternative_else +SB_BARRIER_INSN + nop +alternative_endif + .endm + +/* * Sanitise a 64-bit bounded index wrt speculation, returning zero if out * of bounds. */
arch/arm64/include/asm/barrier.h +4 -0
@@ -33,6 +33,10 @@#define csdb () asm volatile("hint #20" : : : "memory") +#define spec_bar () asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \ + SB_BARRIER_INSN"nop\n", \ + ARM64_HAS_SB)) +#define mb () dsb(sy)#define rmb () dsb(ld)#define wmb () dsb(st)
arch/arm64/include/asm/cpucaps.h +2 -1
@@ -38,7 +38,8 @@#define ARM64_HARDEN_BRANCH_PREDICTOR 17#define ARM64_SSBD 18#define ARM64_MISMATCHED_CACHE_TYPE 19 +#define ARM64_HAS_SB 20 -#define ARM64_NCAPS 20 +#define ARM64_NCAPS 21#endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/sysreg.h +13 -0
@@ -86,6 +86,12 @@#define SET_PSTATE_UAO (x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\ (!!x)<<8 | 0x1f) +#define __SYS_BARRIER_INSN (CRm, op2, Rt) \ + __inst_arm(0xd5000000 | sys_reg(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f)) + +#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31) + +/* Common SCTLR_ELx flags. */ #define SCTLR_ELx_EE (1 << 25)#define SCTLR_ELx_I (1 << 12)
@@ -116,6 +122,13 @@#define ID_AA64ISAR0_SHA1_SHIFT 8#define ID_AA64ISAR0_AES_SHIFT 4 +/* id_aa64isar1 */ +#define ID_AA64ISAR1_SB_SHIFT 36 +#define ID_AA64ISAR1_LRCPC_SHIFT 20 +#define ID_AA64ISAR1_FCMA_SHIFT 16 +#define ID_AA64ISAR1_JSCVT_SHIFT 12 +#define ID_AA64ISAR1_DPB_SHIFT 0 +/* id_aa64pfr0 */ #define ID_AA64PFR0_CSV3_SHIFT 60#define ID_AA64PFR0_CSV2_SHIFT 56
arch/arm64/include/asm/uaccess.h +1 -2
@@ -71,8 +71,7 @@ static inline void set_fs(mm_segment_t fs)* Prevent a mispredicted conditional call to set_fs from forwarding * the wrong address limit to access_ok under speculation. */ -dsb (nsh); -isb (); +spec_bar ();/* * Enable/disable UAO so that copy_to_user() etc can access
arch/arm64/include/uapi/asm/hwcap.h +1 -0
@@ -30,5 +30,6 @@#define HWCAP_ATOMICS (1 << 8)#define HWCAP_FPHP (1 << 9)#define HWCAP_ASIMDHP (1 << 10) +#define HWCAP_SB (1 << 29)#endif /* _UAPI__ASM_HWCAP_H */
arch/arm64/kernel/cpufeature.c +21 -1
@@ -93,6 +93,15 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { +ARM64_FTR_BITS (FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64ISAR1_SB_SHIFT ,4 ,0 ), +ARM64_FTR_BITS (FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64ISAR1_LRCPC_SHIFT ,4 ,0 ), +ARM64_FTR_BITS (FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64ISAR1_FCMA_SHIFT ,4 ,0 ), +ARM64_FTR_BITS (FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64ISAR1_JSCVT_SHIFT ,4 ,0 ), +ARM64_FTR_BITS (FTR_STRICT ,FTR_LOWER_SAFE ,ID_AA64ISAR1_DPB_SHIFT ,4 ,0 ), +ARM64_FTR_END , +}; +static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {ARM64_FTR_BITS (FTR_NONSTRICT ,FTR_LOWER_SAFE ,ID_AA64PFR0_CSV3_SHIFT ,4 ,0 ),ARM64_FTR_BITS (FTR_NONSTRICT ,FTR_LOWER_SAFE ,ID_AA64PFR0_CSV2_SHIFT ,4 ,0 ),
@@ -326,7 +335,7 @@ static const struct __ftr_reg_entry {/* Op1 = 0, CRn = 0, CRm = 6 */ ARM64_FTR_REG (SYS_ID_AA64ISAR0_EL1 , ftr_id_aa64isar0), -ARM64_FTR_REG (SYS_ID_AA64ISAR1_EL1 , ftr_aa64raz), +ARM64_FTR_REG (SYS_ID_AA64ISAR1_EL1 , ftr_id_aa64isar1),/* Op1 = 0, CRn = 0, CRm = 7 */ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
@@ -949,6 +958,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .enable = kpti_install_ng_mappings, },#endif + { + .desc = "Speculation barrier (SB)", + .capability = ARM64_HAS_SB, + .def_scope = SCOPE_SYSTEM, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64ISAR1_EL1, + .field_pos = ID_AA64ISAR1_SB_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + }, {}, };
@@ -976,6 +995,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {HWCAP_CAP (SYS_ID_AA64PFR0_EL1 ,ID_AA64PFR0_FP_SHIFT ,FTR_SIGNED ,1 ,CAP_HWCAP ,HWCAP_FPHP ),HWCAP_CAP (SYS_ID_AA64PFR0_EL1 ,ID_AA64PFR0_ASIMD_SHIFT ,FTR_SIGNED ,0 ,CAP_HWCAP ,HWCAP_ASIMD ),HWCAP_CAP (SYS_ID_AA64PFR0_EL1 ,ID_AA64PFR0_ASIMD_SHIFT ,FTR_SIGNED ,1 ,CAP_HWCAP ,HWCAP_ASIMDHP ), + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SB), {}, };
arch/arm64/kernel/cpuinfo.c +1 -0
@@ -63,6 +63,7 @@ static const char *const hwcap_str[] = {"atomics" ,"fphp" ,"asimdhp" , +"sb" , NULL };
Signature · 6 lines
2.7.4 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v3 2/2 arm64: entry: Place an SB sequence following an ERET instruction
In reply to Florian FainelliFrom: Will Deacon <[email protected]>
commit 679db70801da9fda91d26caf13bf5b5ccc74e8e8 upstream
Some CPUs can speculate past an ERET instruction and potentially perform speculative accesses to memory before processing the exception return. Since the register state is often controlled by a lower privilege level at the point of an ERET, this could potentially be used as part of a side-channel attack.
This patch emits an SB sequence after each ERET so that speculation is held up on exception return.
[florian: Adjust hyp-entry.S to account for the label
added change to hyp/entry.S]
--- arch/arm64/kernel/entry.S | 2 ++ arch/arm64/kvm/hyp/entry.S | 2 ++ arch/arm64/kvm/hyp/hyp-entry.S | 4 ++++ 3 files changed, 8 insertions(+)
arch/arm64/kernel/entry.S +2 -0
@@ -255,6 +255,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .else eret .endif + sb .endm .macro get_thread_info, rd
@@ -945,6 +946,7 @@ __ni_sys_trace: mrs x30, far_el1 .endif eret + sb .endm .align 11
arch/arm64/kvm/hyp/entry.S +2 -0
@@ -83,6 +83,7 @@ ENTRY(__guest_enter) // Do not touch any register after this! eret + sb ENDPROC(__guest_enter) ENTRY(__guest_exit)
@@ -195,4 +196,5 @@ alternative_endif ldp x0, x1, [sp], #16 eret + sb ENDPROC(__fpsimd_guest_restore)
arch/arm64/kvm/hyp/hyp-entry.S +4 -0
@@ -97,6 +97,7 @@ el1_sync: // Guest trapped into EL2 do_el2_call 2: eret + sb el1_hvc_guest: /*
@@ -147,6 +148,7 @@ wa_epilogue: mov x0, xzr add sp, sp, #16 eret + sb el1_trap: get_vcpu_ptr x1, x0
@@ -198,6 +200,7 @@ el2_error: b.ne __hyp_panic mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) eret + sb ENTRY(__hyp_do_panic) mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
@@ -206,6 +209,7 @@ ENTRY(__hyp_do_panic) ldr lr, =panic msr elr_el2, lr eret + sb ENDPROC(__hyp_do_panic) ENTRY(__hyp_panic)
Signature · 6 lines
2.7.4 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm