Skip to contentSkip to conversation

arm64: Define Qualcomm Technologies Falkor v1 CPU

PATCH Superseded Christopher Covington 5 participants 10 messages Dec 7, 2016 – Dec 19, 2016

Subsystems (3)

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 arm64: Define Qualcomm Technologies Falkor v1 CPU

From: Shanker Donthineni <[email protected]>

This patch adds the cputype info for Qualcomm Technologies ARMv8 CPU implementer ID 0x51 and part number for Falkor v1 in cputype.h.

---
 arch/arm64/include/asm/cputype.h | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/include/asm/cputype.h +4 -0
index 26a68dd..ee60561 100644
@@ -71,6 +71,7 @@
 #define ARM_CPU_IMP_APM			0x50
 #define ARM_CPU_IMP_CAVIUM		0x43
 #define ARM_CPU_IMP_BRCM		0x42
+#define ARM_CPU_IMP_QCOM		0x51
 
 #define ARM_CPU_PART_AEM_V8		0xD0F
 #define ARM_CPU_PART_FOUNDATION		0xD00
@@ -84,10 +85,13 @@
 
 #define BRCM_CPU_PART_VULCAN		0x516
 
+#define QCOM_CPU_PART_FALKOR_V1		0x800
+
 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
 #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
 #define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
 #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
+#define MIDR_QCOM_FALKOR_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1)
 
 #ifndef __ASSEMBLY__
 
Signature · 3 lines
Qualcomm Datacenter Technologies, Inc. as an affiliate of Qualcomm
Technologies, Inc. Qualcomm Technologies, Inc. is a member of the Code Aurora
Forum, a Linux Foundation Collaborative Project.
#1 lore raw

PATCH 2/3 arm64: Work around Falkor erratum 1003

From: Shanker Donthineni <[email protected]>

On the Qualcomm Datacenter Technologies Falkor v1 CPU, memory accesses may allocate TLB entries using an incorrect ASID when TTBRx_EL1 is being updated. Changing the TTBRx_EL1[ASID] and TTBRx_EL1[BADDR] fields separately using a reserved ASID will ensure that there are no TLB entries with incorrect ASID after changing the the ASID.

Pseudo code:
write TTBRx_EL1[ASID] to a reserved value
ISB
write TTBRx_EL1[BADDR] to a desired value
ISB
write TTBRx_EL1[ASID] to a desired value
ISB

---
 arch/arm64/Kconfig               | 11 +++++++++++
 arch/arm64/include/asm/cpucaps.h |  3 ++-
 arch/arm64/kernel/cpu_errata.c   |  7 +++++++
 arch/arm64/mm/context.c          | 10 ++++++++++
 arch/arm64/mm/proc.S             | 21 +++++++++++++++++++++
 5 files changed, 51 insertions(+), 1 deletion(-)
arch/arm64/Kconfig +11 -0
index 969ef88..1004a3d 100644
@@ -474,6 +474,17 @@ config CAVIUM_ERRATUM_27456
 
 	  If unsure, say Y.
 
+config QCOM_FALKOR_ERRATUM_E1003
+	bool "Falkor E1003: Incorrect translation due to ASID change"
+	default y
+	help
+	  An incorrect translation TLBI entry may be created while
+	  changing the ASID & translation table address together for
+	  TTBR0_EL1. The workaround for this issue is use a reserved
+	  ASID in cpu_do_switch_mm() before switching to target ASID.
+
+	  If unsure, say Y.
+
 endmenu
 
 
arch/arm64/include/asm/cpucaps.h +2 -1
index 87b4465..cb6a8c2 100644
@@ -34,7 +34,8 @@
 #define ARM64_HAS_32BIT_EL0			13
 #define ARM64_HYP_OFFSET_LOW			14
 #define ARM64_MISMATCHED_CACHE_LINE_SIZE	15
+#define ARM64_WORKAROUND_QCOM_FALKOR_E1003	16
 
-#define ARM64_NCAPS				16
+#define ARM64_NCAPS				17
 
 #endif /* __ASM_CPUCAPS_H */
arch/arm64/kernel/cpu_errata.c +7 -0
index b75e917..3789e2f 100644
@@ -130,6 +130,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.def_scope = SCOPE_LOCAL_CPU,
 		.enable = cpu_enable_trap_ctr_access,
 	},
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1003
+	{
+		.desc = "Qualcomm Falkor erratum E1003",
+		.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
+		MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0x00, 0x00),
+	},
+#endif
 	{
 	}
 };
arch/arm64/mm/context.c +10 -0
index efcf1f7..f8d94ff 100644
@@ -87,6 +87,11 @@ static void flush_context(unsigned int cpu)
 	/* Update the list of reserved ASIDs and the ASID bitmap. */
 	bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 
+	/* Reserve ASID '1' for Falkor erratum E1003 */
+	if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_E1003) &&
+	    cpus_have_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
+		__set_bit(1, asid_map);
+
 	/*
 	 * Ensure the generation bump is observed before we xchg the
 	 * active_asids.
@@ -239,6 +244,11 @@ static int asids_init(void)
 		panic("Failed to allocate bitmap for %lu ASIDs\n",
 		      NUM_USER_ASIDS);
 
+	/* Reserve ASID '1' for Falkor erratum E1003 */
+	if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_E1003) &&
+	    cpus_have_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
+		__set_bit(1, asid_map);
+
 	pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
 	return 0;
 }
arch/arm64/mm/proc.S +21 -0
index 352c73b..b4d6508 100644
@@ -134,6 +134,27 @@ ENDPROC(cpu_do_resume)
 ENTRY(cpu_do_switch_mm)
 	mmid	x1, x1				// get mm->context.id
 	bfi	x0, x1, #48, #16		// set the ASID
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1003
+alternative_if_not ARM64_WORKAROUND_QCOM_FALKOR_E1003
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+alternative_else
+	mrs     x2, ttbr0_el1                   // get cuurent TTBR0_EL1
+	mov     x3, #1                          // reserved ASID
+	bfi     x2, x3, #48, #16                // set the reserved ASID + old BADDR
+	msr     ttbr0_el1, x2                   // update TTBR0_EL1
+	isb
+	bfi     x2, x0, #0, #48                 // set the desired BADDR + reserved ASID
+	msr     ttbr0_el1, x2                   // update TTBR0_EL1
+	isb
+alternative_endif
+#endif
 	msr	ttbr0_el1, x0			// set TTBR0
 	isb
 alternative_if ARM64_WORKAROUND_CAVIUM_27456
Signature · 3 lines
Qualcomm Datacenter Technologies, Inc. as an affiliate of Qualcomm
Technologies, Inc. Qualcomm Technologies, Inc. is a member of the Code Aurora
Forum, a Linux Foundation Collaborative Project.
maintainer
#2 lore raw

Re: [PATCH 2/3] arm64: Work around Falkor erratum 1003

Quoted 8 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

On the Qualcomm Datacenter Technologies Falkor v1 CPU, memory accesses may allocate TLB entries using an incorrect ASID when TTBRx_EL1 is being updated. Changing the TTBRx_EL1[ASID] and TTBRx_EL1[BADDR] fields separately using a reserved ASID will ensure that there are no TLB entries with incorrect ASID after changing the the ASID.

Quoted 7 lines

Pseudo code:
write TTBRx_EL1[ASID] to a reserved value
ISB
write TTBRx_EL1[BADDR] to a desired value
ISB
write TTBRx_EL1[ASID] to a desired value
ISB

While the new ASID probably won't have incorrect TLB entries, the reserved ASID will have random entries from all over the place. That's because in step 1 you change the ASID to the reserved one while leaving the old BADDR in place. There is a brief time before changing the ASID when speculative page table walks will populate the TLB with entries tagged with the reserved ASID. Such entries are never removed during TLB shoot-down for the real ASID, so, depending on how this CPU implements the walk cache, you could end up with intermediate level entries still active and pointing to freed/reused pages. It will eventually hit an entry that looks global with weird consequences.

We've been bitten by this in the past on arm32: 52af9c6cd863 ("ARM:
6943/1: mm: use TTBR1 instead of reserved context ID").

Signature · 1 line
Catalin
#3 lore raw

Re: [PATCH 2/3] arm64: Work around Falkor erratum 1003

Hi Catalin,

Quoted 28 lines · Catalin Marinas
Quoted 8 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

On the Qualcomm Datacenter Technologies Falkor v1 CPU, memory accesses may allocate TLB entries using an incorrect ASID when TTBRx_EL1 is being updated. Changing the TTBRx_EL1[ASID] and TTBRx_EL1[BADDR] fields separately using a reserved ASID will ensure that there are no TLB entries with incorrect ASID after changing the the ASID.

Quoted 7 lines

Pseudo code:
write TTBRx_EL1[ASID] to a reserved value
ISB
write TTBRx_EL1[BADDR] to a desired value
ISB
write TTBRx_EL1[ASID] to a desired value
ISB

While the new ASID probably won't have incorrect TLB entries, the reserved ASID will have random entries from all over the place. That's because in step 1 you change the ASID to the reserved one while leaving the old BADDR in place. There is a brief time before changing the ASID when speculative page table walks will populate the TLB with entries tagged with the reserved ASID. Such entries are never removed during TLB shoot-down for the real ASID, so, depending on how this CPU implements the walk cache, you could end up with intermediate level entries still active and pointing to freed/reused pages. It will eventually hit an entry that looks global with weird consequences.

Quoted 2 lines

We've been bitten by this in the past on arm32: 52af9c6cd863 ("ARM:
6943/1: mm: use TTBR1 instead of reserved context ID").

Thanks for bringing this up, but I'm told the scenario you describe won't happen on the Falkor 1.0 CPU.

Thanks,
Cov

Signature · 3 lines
Qualcomm Datacenter Technologies, Inc. as an affiliate of Qualcomm
Technologies, Inc. Qualcomm Technologies, Inc. is a member of the Code
Aurora Forum, a Linux Foundation Collaborative Project.
reviewer
#4 lore raw

Re: [PATCH 2/3] arm64: Work around Falkor erratum 1003

In reply to Christopher Covington
Quoted 18 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

On the Qualcomm Datacenter Technologies Falkor v1 CPU, memory accesses may allocate TLB entries using an incorrect ASID when TTBRx_EL1 is being updated. Changing the TTBRx_EL1[ASID] and TTBRx_EL1[BADDR] fields separately using a reserved ASID will ensure that there are no TLB entries with incorrect ASID after changing the the ASID.

Pseudo code:
write TTBRx_EL1[ASID] to a reserved value
ISB
write TTBRx_EL1[BADDR] to a desired value
ISB
write TTBRx_EL1[ASID] to a desired value
ISB

Quoted 7 lines
---
 arch/arm64/Kconfig               | 11 +++++++++++
 arch/arm64/include/asm/cpucaps.h |  3 ++-
 arch/arm64/kernel/cpu_errata.c   |  7 +++++++
 arch/arm64/mm/context.c          | 10 ++++++++++
 arch/arm64/mm/proc.S             | 21 +++++++++++++++++++++
 5 files changed, 51 insertions(+), 1 deletion(-)

This needs an update to Documentation/arm64/silicon-errata.txt.

Quoted 39 lines
arch/arm64/mm/context.c +10 -0
index efcf1f7..f8d94ff 100644
@@ -87,6 +87,11 @@ static void flush_context(unsigned int cpu)
 	/* Update the list of reserved ASIDs and the ASID bitmap. */
 	bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 
+	/* Reserve ASID '1' for Falkor erratum E1003 */
+	if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_E1003) &&
+	    cpus_have_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
+		__set_bit(1, asid_map);
+
 	/*
 	 * Ensure the generation bump is observed before we xchg the
 	 * active_asids.
@@ -239,6 +244,11 @@ static int asids_init(void)
 		panic("Failed to allocate bitmap for %lu ASIDs\n",
 		      NUM_USER_ASIDS);
 
+	/* Reserve ASID '1' for Falkor erratum E1003 */
+	if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_E1003) &&
+	    cpus_have_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
+		__set_bit(1, asid_map);
+
 	pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
 	return 0;
 }
arch/arm64/mm/proc.S +3 -0
index 352c73b..b4d6508 100644
@@ -134,6 +134,27 @@ ENDPROC(cpu_do_resume)
 ENTRY(cpu_do_switch_mm)
 	mmid	x1, x1				// get mm->context.id
 	bfi	x0, x1, #48, #16		// set the ASID
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1003
+alternative_if_not ARM64_WORKAROUND_QCOM_FALKOR_E1003
+	nop

arch/arm64/mm/proc.S ENDPROC(cpu_do_resume) · line 149

Quoted 10 lines
arch/arm64/mm/proc.S +10 -0
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+alternative_else
+	mrs     x2, ttbr0_el1                   // get cuurent TTBR0_EL1
+	mov     x3, #1                          // reserved ASID

It might be best to define a FALCOR_E1003_RESERVED_ASID constant somewhere, rather than using 1 directly here and in the ASID allocator.

arch/arm64/mm/proc.S ENDPROC(cpu_do_resume) · line 156

Quoted 7 lines
arch/arm64/mm/proc.S +7 -0
+	bfi     x2, x3, #48, #16                // set the reserved ASID + old BADDR
+	msr     ttbr0_el1, x2                   // update TTBR0_EL1
+	isb
+	bfi     x2, x0, #0, #48                 // set the desired BADDR + reserved ASID
+	msr     ttbr0_el1, x2                   // update TTBR0_EL1
+	isb
+alternative_endif

Please use alternative_if and alternative_else_nop_endif.

As Catalin noted, there are issues with stale and/or conflicting TLB entries allocated with the reserved ASID, so we likely have to invalidate that after the final switch.

Thanks,
Mark.

#5 lore raw

PATCH 1/1 arm64: Work around Falkor erratum 1009

In reply to Christopher Covington

From: Shanker Donthineni <[email protected]>

During a TLB invalidate sequence targeting the inner shareable domain, Falkor may prematurely complete the DSB before all loads and stores using the old translation are observed; instruction fetches are not subject to the conditions of this erratum.

---
 arch/arm64/Kconfig                | 10 +++++++++
 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/tlbflush.h | 43 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/cpu_errata.c    |  7 +++++++
 arch/arm64/kvm/hyp/tlb.c          | 39 ++++++++++++++++++++++++++++++-----
 5 files changed, 96 insertions(+), 6 deletions(-)
arch/arm64/Kconfig +10 -0
index 1004a3d..125440f 100644
@@ -485,6 +485,16 @@ config QCOM_FALKOR_ERRATUM_E1003
 
 	  If unsure, say Y.
 
+config QCOM_FALKOR_ERRATUM_E1009
+	bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
+	default y
+	help
+	  Falkor CPU may prematurely complete a DSB following a TLBI xxIS
+	  invalidate maintenance operations. Repeat the TLBI operation one
+	  more time to fix the issue.
+
+	  If unsure, say Y.
+
 endmenu
 
 
arch/arm64/include/asm/cpucaps.h +2 -1
index cb6a8c2..5357d7f 100644
@@ -35,7 +35,8 @@
 #define ARM64_HYP_OFFSET_LOW			14
 #define ARM64_MISMATCHED_CACHE_LINE_SIZE	15
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003	16
+#define ARM64_WORKAROUND_QCOM_FALKOR_E1009	17
 
-#define ARM64_NCAPS				17
+#define ARM64_NCAPS				18
 
 #endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/tlbflush.h +43 -0

85 lines not on this page · raw message

arch/arm64/kernel/cpu_errata.c +7 -0
index 3789e2f..8013579 100644
@@ -137,6 +137,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0x00, 0x00),
 	},
 #endif
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1009
+	{
+		.desc = "Qualcomm Falkor erratum E1009",
+		.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1009,
+		MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0x00, 0x00),
+	},
+#endif
 	{
 	}
 };
arch/arm64/kvm/hyp/tlb.c +34 -5
index 88e2f2b..dfd3a77 100644
@@ -16,6 +16,7 @@
  */
 
 #include <asm/kvm_hyp.h>
+#include <asm/alternative.h>
 
 void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
 {
@@ -32,7 +33,14 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
 	 * whole of Stage-1. Weep...
 	 */
 	ipa >>= 12;
-	asm volatile("tlbi ipas2e1is, %0" : : "r" (ipa));
+	asm volatile("tlbi ipas2e1is, %0 \n"
+		     ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "dsb ish \n"
+		     "tlbi ipas2e1is, %0 \n",
+		     ARM64_WORKAROUND_QCOM_FALKOR_E1009)
+		     : : "r" (ipa));
 
 	/*
 	 * We have to ensure completion of the invalidation at Stage-2,
@@ -41,7 +49,14 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
 	 * the Stage-1 invalidation happened first.
 	 */
 	dsb(ish);
-	asm volatile("tlbi vmalle1is" : : );
+	asm volatile("tlbi vmalle1is \n"
+		     ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "dsb ish \n"
+		     "tlbi vmalle1is \n",
+		     ARM64_WORKAROUND_QCOM_FALKOR_E1009)
+		     : : );
 	dsb(ish);
 	isb();
 
@@ -57,7 +72,14 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
 	write_sysreg(kvm->arch.vttbr, vttbr_el2);
 	isb();
 
-	asm volatile("tlbi vmalls12e1is" : : );
+	asm volatile("tlbi vmalls12e1is \n"
+		     ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "dsb ish \n"
+		     "tlbi vmalls12e1is \n",
+		     ARM64_WORKAROUND_QCOM_FALKOR_E1009)
+		     : : );
 	dsb(ish);
 	isb();
 
@@ -82,7 +104,14 @@ void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
 void __hyp_text __kvm_flush_vm_context(void)
 {
 	dsb(ishst);
-	asm volatile("tlbi alle1is	\n"
-		     "ic ialluis	  ": : );
+	asm volatile("tlbi alle1is \n"
+		     ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "dsb ish \n"
+		     "tlbi alle1is \n",
+		     ARM64_WORKAROUND_QCOM_FALKOR_E1009)
+		     "ic ialluis \n"
+		     : : );
 	dsb(ish);
 }
Signature · 3 lines
Qualcomm Datacenter Technologies, Inc. as an affiliate of Qualcomm
Technologies, Inc. Qualcomm Technologies, Inc. is a member of the Code Aurora
Forum, a Linux Foundation Collaborative Project.
#6 lore raw

Re: [PATCH] arm64: Work around Falkor erratum 1009

Quoted 24 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

During a TLB invalidate sequence targeting the inner shareable domain, Falkor may prematurely complete the DSB before all loads and stores using the old translation are observed; instruction fetches are not subject to the conditions of this erratum.

---
 arch/arm64/Kconfig                | 10 +++++++++
 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/tlbflush.h | 43 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/cpu_errata.c    |  7 +++++++
 arch/arm64/kvm/hyp/tlb.c          | 39 ++++++++++++++++++++++++++++++-----
 5 files changed, 96 insertions(+), 6 deletions(-)
arch/arm64/Kconfig +0 -0
index 1004a3d..125440f 100644
@@ -485,6 +485,16 @@ config QCOM_FALKOR_ERRATUM_E1003
 
 	  If unsure, say Y.

arch/arm64/Kconfig config QCOM_FALKOR_ERRATUM_E1003 · line 496

Quoted 10 lines
arch/arm64/Kconfig +9 -0
 
+config QCOM_FALKOR_ERRATUM_E1009
+	bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
+	default y
+	help
+	  Falkor CPU may prematurely complete a DSB following a TLBI xxIS
+	  invalidate maintenance operations. Repeat the TLBI operation one
+	  more time to fix the issue.
+
+	  If unsure, say Y.

Call me perverse, but I like this workaround. People often tend to screw up TLBI and DVM sync, but the IPI-based workaround is horribly invasive and fragile. Simply repeating the operation tends to be enough to make the chance of failure small enough to be acceptable.

arch/arm64/include/asm/cpucaps.h line 38

Quoted 9 lines
arch/arm64/include/asm/cpucaps.h +1 -0
index cb6a8c2..5357d7f 100644
@@ -35,7 +35,8 @@
 #define ARM64_HYP_OFFSET_LOW			14
 #define ARM64_MISMATCHED_CACHE_LINE_SIZE	15
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003	16
+#define ARM64_WORKAROUND_QCOM_FALKOR_E1009	17

Could you rename this to something like ARM64_WORKAROUND_REPEAT_TLBI, so that it could potentially be used by others?

Quoted 18 lines
arch/arm64/include/asm/cpucaps.h +1 -1
-#define ARM64_NCAPS				17
+#define ARM64_NCAPS				18
 
 #endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/tlbflush.h +1 -0
index deab523..03bafc5 100644
@@ -23,6 +23,7 @@
 
 #include <linux/sched.h>
 #include <asm/cputype.h>
+#include <asm/alternative.h>
 
 /*
  * Raw TLBI operations.

arch/arm64/include/asm/tlbflush.h static inline void flush_tlb_all(void) · line 104

Quoted 10 lines
arch/arm64/include/asm/tlbflush.h +7 -0
@@ -94,6 +95,13 @@ static inline void flush_tlb_all(void)
 	dsb(ishst);
 	__tlbi(vmalle1is);
 	dsb(ish);
+	asm volatile(ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "tlbi vmalle1is \n"
+		     "dsb ish \n",
+		     ARM64_WORKAROUND_QCOM_FALKOR_E1009)
+		     : :);

I'd much rather this was part of the __tlbi macro, which would hopefully restrict this to one place in the code.

Will

#7 lore raw

Re: [PATCH] arm64: Work around Falkor erratum 1009

Quoted 41 lines · Will Deacon
Quoted 24 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

During a TLB invalidate sequence targeting the inner shareable domain, Falkor may prematurely complete the DSB before all loads and stores using the old translation are observed; instruction fetches are not subject to the conditions of this erratum.

---
 arch/arm64/Kconfig                | 10 +++++++++
 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/tlbflush.h | 43 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/cpu_errata.c    |  7 +++++++
 arch/arm64/kvm/hyp/tlb.c          | 39 ++++++++++++++++++++++++++++++-----
 5 files changed, 96 insertions(+), 6 deletions(-)
arch/arm64/Kconfig +0 -0
index 1004a3d..125440f 100644
@@ -485,6 +485,16 @@ config QCOM_FALKOR_ERRATUM_E1003
 
 	  If unsure, say Y.

arch/arm64/Kconfig config QCOM_FALKOR_ERRATUM_E1003 · line 496

Quoted 10 lines
arch/arm64/Kconfig +9 -0
 
+config QCOM_FALKOR_ERRATUM_E1009
+	bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
+	default y
+	help
+	  Falkor CPU may prematurely complete a DSB following a TLBI xxIS
+	  invalidate maintenance operations. Repeat the TLBI operation one
+	  more time to fix the issue.
+
+	  If unsure, say Y.

Call me perverse, but I like this workaround. People often tend to screw up TLBI and DVM sync, but the IPI-based workaround is horribly invasive and fragile. Simply repeating the operation tends to be enough to make the chance of failure small enough to be acceptable.

Quoted 12 lines

arch/arm64/include/asm/cpucaps.h line 38

Quoted 9 lines
arch/arm64/include/asm/cpucaps.h +1 -0
index cb6a8c2..5357d7f 100644
@@ -35,7 +35,8 @@
 #define ARM64_HYP_OFFSET_LOW			14
 #define ARM64_MISMATCHED_CACHE_LINE_SIZE	15
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003	16
+#define ARM64_WORKAROUND_QCOM_FALKOR_E1009	17

Could you rename this to something like ARM64_WORKAROUND_REPEAT_TLBI, so that it could potentially be used by others?

And add a parameter to it so that we can generate multiple TLBIs, depending on the level of brokenness? ;-)

	M.
Signature · 1 line
Jazz is not dead. It just smells funny...
reviewer
#8 lore raw

Re: [PATCH] arm64: Work around Falkor erratum 1009

In reply to Christopher Covington
Quoted 9 lines · Christopher Covington

From: Shanker Donthineni <[email protected]>

During a TLB invalidate sequence targeting the inner shareable domain, Falkor may prematurely complete the DSB before all loads and stores using the old translation are observed; instruction fetches are not subject to the conditions of this erratum.

Quoted 7 lines
---
 arch/arm64/Kconfig                | 10 +++++++++
 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/tlbflush.h | 43 +++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/cpu_errata.c    |  7 +++++++
 arch/arm64/kvm/hyp/tlb.c          | 39 ++++++++++++++++++++++++++++++-----
 5 files changed, 96 insertions(+), 6 deletions(-)

Please update Documentation/arm64/silicon-errata.txt respectively.

[...]

Quoted 3 lines

#include <linux/sched.h>
#include <asm/cputype.h>
+#include <asm/alternative.h>

Nit: please keep includes (alphabetically) ordered (at least below the linux/ or asm/ level).

[...]

arch/arm64/include/asm/tlbflush.h static inline void flush_tlb_all(void) · line 102

Quoted 5 lines
arch/arm64/include/asm/tlbflush.h +5 -0
+	asm volatile(ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "tlbi vmalle1is \n"
+		     "dsb ish \n",

As a general note, perhaps we want a C compatible NOP_ALTERNATIVE() so that the nop case can be implicitly generated for sequences like this.

Thanks,
Mark.

maintainer
#9 lore raw

Re: [PATCH] arm64: Work around Falkor erratum 1009

Quoted 9 lines · Mark Rutland

arch/arm64/include/asm/tlbflush.h static inline void flush_tlb_all(void) · line 102

Quoted 5 lines · Christopher Covington
arch/arm64/include/asm/tlbflush.h +5 -0
+	asm volatile(ALTERNATIVE(
+		     "nop \n"
+		     "nop \n",
+		     "tlbi vmalle1is \n"
+		     "dsb ish \n",

As a general note, perhaps we want a C compatible NOP_ALTERNATIVE() so that the nop case can be implicitly generated for sequences like this.

It's also worth checking what cpus_have_const_cap() would generate for the default (no workaround required) case.

Signature · 1 line
Catalin