arm64: Workaround Neoverse-N1 #1542419
Subsystems (3)
cover arm64: Workaround Neoverse-N1 #1542419
This series backports the Neoverse-N1 #1542419 erratum workaround to v4.19.116. The series was originally merged in v5.5.
These patches handle user-space. The kernel change was:
commit dd8a1f134884 ("arm64: ftrace: Ensure synchronisation in PLT setup for Neoverse-N1 #1542419"), which has already been picked up by stable. (magic!)
Backporting this stuff past v4.19 isn't straight-forward as the kernel change depends on the work done in:
https://lore.kernel.org/linux-arm-kernel/[email protected]/
which was merged for v4.19.
Thanks,
James
Catalin Marinas (1):
arm64: Silence clang warning on mismatched value/register sizes
James Morse (3):
arm64: errata: Hide CTR_EL0.DIC on systems affected by Neoverse-N1
#1542419
arm64: Fake the IminLine size on systems affected by Neoverse-N1
#1542419arm64: compat: Workaround Neoverse-N1 #1542419 for compat user-space
Marc Zyngier (1):
arm64: Add part number for Neoverse N1
Documentation/arm64/silicon-errata.txt | 1 + arch/arm64/Kconfig | 16 ++++++++++++++++ arch/arm64/include/asm/cache.h | 3 ++- arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/include/asm/cputype.h | 2 ++ arch/arm64/kernel/cpu_errata.c | 22 ++++++++++++++++++++++ arch/arm64/kernel/sys_compat.c | 11 +++++++++++ arch/arm64/kernel/traps.c | 9 +++++++++ 8 files changed, 65 insertions(+), 2 deletions(-)
Signature · 1 line
2.26.1
PATCH 1/5 arm64: Add part number for Neoverse N1
From: Marc Zyngier <[email protected]>
[ Upstream commit 0cf57b86859c49381addb3ce47be70aadf5fd2c0 ]
New CPU, new part number. You know the drill.
--- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+)
arch/arm64/include/asm/cputype.h +2 -0
@@ -80,6 +80,7 @@#define ARM_CPU_PART_CORTEX_A35 0xD04#define ARM_CPU_PART_CORTEX_A55 0xD05#define ARM_CPU_PART_CORTEX_A76 0xD0B +#define ARM_CPU_PART_NEOVERSE_N1 0xD0C#define APM_CPU_PART_POTENZA 0x000
@@ -107,6 +108,7 @@#define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35)#define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55)#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) +#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
Signature · 1 line
2.26.1
PATCH 2/5 arm64: errata: Hide CTR_EL0.DIC on systems affected by Neoverse-N1 #1542419
In reply to James Morse[ Upstream commit 05460849c3b51180d5ada3373d0449aea19075e4 ]
Cores affected by Neoverse-N1 #1542419 could execute a stale instruction when a branch is updated to point to freshly generated instructions.
To workaround this issue we need user-space to issue unnecessary icache maintenance that we can trap. Start by hiding CTR_EL0.DIC.
--- Documentation/arm64/silicon-errata.txt | 1 + arch/arm64/Kconfig | 16 ++++++++++++++++ arch/arm64/include/asm/cpucaps.h | 3 ++- arch/arm64/kernel/cpu_errata.c | 22 ++++++++++++++++++++++ arch/arm64/kernel/traps.c | 3 +++ 5 files changed, 44 insertions(+), 1 deletion(-)
Documentation/arm64/silicon-errata.txt +1 -0
@@ -59,6 +59,7 @@ stable kernels. | ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 | | ARM | Cortex-A55 | #1024718 | ARM64_ERRATUM_1024718 | | ARM | Cortex-A76 | #1463225 | ARM64_ERRATUM_1463225 | +| ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 | | ARM | MMU-500 | #841119,#826419 | N/A | | | | | | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 |
arch/arm64/Kconfig +16 -0
@@ -499,6 +499,22 @@ config ARM64_ERRATUM_1463225 If unsure, say Y. +config ARM64_ERRATUM_1542419 + bool "Neoverse-N1: workaround mis-ordering of instruction fetches" + default y + help + This option adds a workaround for ARM Neoverse-N1 erratum + 1542419. + + Affected Neoverse-N1 cores could execute a stale instruction when + modified by another CPU. The workaround depends on a firmware + counterpart. + + Workaround the issue by hiding the DIC feature from EL0. This + forces user-space to perform cache maintenance. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y
arch/arm64/include/asm/cpucaps.h +2 -1
@@ -53,7 +53,8 @@#define ARM64_HAS_STAGE2_FWB 32#define ARM64_WORKAROUND_1463225 33#define ARM64_SSBS 34 +#define ARM64_WORKAROUND_1542419 35 -#define ARM64_NCAPS 35 +#define ARM64_NCAPS 36#endif /* __ASM_CPUCAPS_H */
arch/arm64/kernel/cpu_errata.c +22 -0
@@ -643,6 +643,18 @@ needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry,return false; } +static bool __maybe_unused +has_neoverse_n1_erratum_1542419 (const struct arm64_cpu_capabilities *entry, +int scope) +{ +u32 midr =read_cpuid_id (); +bool has_dic =read_cpuid_cachetype () &BIT (CTR_DIC_SHIFT ); +const struct midr_range range =MIDR_ALL_VERSIONS (MIDR_NEOVERSE_N1 ); + +WARN_ON (scope !=SCOPE_LOCAL_CPU ||preemptible ()); +return is_midr_in_range (midr, &range) && has_dic; +} +#ifdef CONFIG_HARDEN_EL2_VECTORS static const struct midr_range arm64_harden_el2_vectors[] = {
@@ -834,6 +846,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(tx2_family_cpus), .matches = needs_tx2_tvm_workaround, }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_1542419 + { + /* we depend on the firmware portion for correctness */ + .desc = "ARM erratum 1542419 (kernel portion)", + .capability = ARM64_WORKAROUND_1542419, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = has_neoverse_n1_erratum_1542419, + .cpu_enable = cpu_enable_trap_ctr_access, + }, #endif { }
arch/arm64/kernel/traps.c +3 -0
@@ -481,6 +481,9 @@ static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;unsigned long val =arm64_ftr_reg_user_value (&arm64_ftr_reg_ctrel0); +if (cpus_have_const_cap (ARM64_WORKAROUND_1542419 )) + val &= ~BIT (CTR_DIC_SHIFT ); +pt_regs_write_reg (regs, rt, val);arm64_skip_faulting_instruction (regs,AARCH64_INSN_SIZE );
Signature · 1 line
2.26.1
PATCH 3/5 arm64: Fake the IminLine size on systems affected by Neoverse-N1 #1542419
In reply to James Morse[ Upstream commit ee9d90be9ddace01b7fb126567e4b539fbe1f82f ]
Systems affected by Neoverse-N1 #1542419 support DIC so do not need to perform icache maintenance once new instructions are cleaned to the PoU. For the errata workaround, the kernel hides DIC from user-space, so that the unnecessary cache maintenance can be trapped by firmware.
To reduce the number of traps, produce a fake IminLine value based on PAGE_SIZE.
--- arch/arm64/include/asm/cache.h | 3 ++- arch/arm64/kernel/traps.c | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-)
arch/arm64/include/asm/cache.h +2 -1
@@ -22,6 +22,7 @@#define CTR_L1IP_MASK 3#define CTR_DMINLINE_SHIFT 16#define CTR_IMINLINE_SHIFT 0 +#define CTR_IMINLINE_MASK 0xf#define CTR_ERG_SHIFT 20#define CTR_CWG_SHIFT 24#define CTR_CWG_MASK 15
@@ -29,7 +30,7 @@#define CTR_DIC_SHIFT 29#define CTR_CACHE_MINLINE_MASK \ - (0xf << CTR_DMINLINE_SHIFT | 0xf << CTR_IMINLINE_SHIFT) + (0xf << CTR_DMINLINE_SHIFT | CTR_IMINLINE_MASK << CTR_IMINLINE_SHIFT)#define CTR_L1IP (ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
arch/arm64/kernel/traps.c +7 -1
@@ -481,9 +481,15 @@ static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;unsigned long val =arm64_ftr_reg_user_value (&arm64_ftr_reg_ctrel0); -if (cpus_have_const_cap (ARM64_WORKAROUND_1542419 )) +if (cpus_have_const_cap (ARM64_WORKAROUND_1542419 )) { +/* Hide DIC so that we can trap the unnecessary maintenance...*/ val &= ~BIT (CTR_DIC_SHIFT ); +/* ... and fake IminLine to reduce the number of traps. */ + val &= ~CTR_IMINLINE_MASK ; + val |= (PAGE_SHIFT -2 ) &CTR_IMINLINE_MASK ; + } +pt_regs_write_reg (regs, rt, val);arm64_skip_faulting_instruction (regs,AARCH64_INSN_SIZE );
Signature · 1 line
2.26.1
PATCH 4/5 arm64: compat: Workaround Neoverse-N1 #1542419 for compat user-space
In reply to James Morse[ Upstream commit: 222fc0c8503d98cec3cb2bac2780cdd21a6e31c0 ]
Compat user-space is unable to perform ICIMVAU instructions from user-space. Instead it uses a compat-syscall. Add the workaround for Neoverse-N1 #1542419 to this code path.
--- arch/arm64/kernel/sys_compat.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
arch/arm64/kernel/sys_compat.c +11 -0
@@ -19,6 +19,7 @@*/ #include <linux/compat.h> +#include <linux/cpufeature.h> #include <linux/personality.h> #include <linux/sched.h> #include <linux/sched/signal.h>
@@ -28,6 +29,7 @@#include <asm/cacheflush.h> #include <asm/system_misc.h> +#include <asm/tlbflush.h> #include <asm/unistd.h> static long
@@ -41,6 +43,15 @@ __do_compat_cache_op(unsigned long start, unsigned long end)if (fatal_signal_pending (current))return 0 ; +if (cpus_have_const_cap (ARM64_WORKAROUND_1542419 )) { +/* +* The workaround requires an inner-shareable tlbi. +* We pick the reserved-ASID to minimise the impact. +*/ +__tlbi (aside1is,0 ); +dsb (ish); + } + ret =__flush_cache_user_range (start, start + chunk);if (ret)return ret;
Signature · 1 line
2.26.1
PATCH 5/5 arm64: Silence clang warning on mismatched value/register sizes
In reply to James MorseFrom: Catalin Marinas <[email protected]>
[ Upstream commit: 27a22fbdeedd6c5c451cf5f830d51782bf50c3a2 ]
Clang reports a warning on the __tlbi(aside1is, 0) macro expansion since the value size does not match the register size specified in the inline asm. Construct the ASID value using the __TLBI_VADDR() macro.
--- arch/arm64/kernel/sys_compat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/sys_compat.c +1 -1
@@ -48,7 +48,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)* The workaround requires an inner-shareable tlbi. * We pick the reserved-ASID to minimise the impact. */ -__tlbi (aside1is,0 ); +__tlbi (aside1is,__TLBI_VADDR (0 ,0 ));dsb (ish); }
Signature · 1 line
2.26.1