arm64: Add TLB Conflict Abort Exception handler to KVM
PATCH 1/1 arm64: Add TLB Conflict Abort Exception handler to KVM
Currently, KVM does not handle the case of a stage 2 TLB conflict abort exception. This can legitimately occurs when the guest is eliding full BBM semantics as permitted by BBM level 2. In this case it is possible for a confclit abort to be delivered to EL2. We handle that by invalidating the full TLB.
The Arm ARM specifies that the worst-case invalidation is either a tlbi vmalls12e1 or a tlbi alle1 (as per DDI0487K section D8.16.3). We implement tlbi alle1 by extending the existing __kvm_flush_vm_context() helper to allow for differentiating between inner-shareable and cpu-local invalidations.
This commit applies on top of v6.13-rc2 (fac04efc5c79).
--- arch/arm64/include/asm/esr.h | 8 ++++++++ arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/tlb.c | 9 +++++++-- arch/arm64/kvm/hyp/vhe/tlb.c | 9 +++++++-- arch/arm64/kvm/mmu.c | 13 +++++++++++++ arch/arm64/kvm/vmid.c | 2 +- 7 files changed, 38 insertions(+), 7 deletions(-)
arch/arm64/include/asm/esr.h +8 -0
@@ -121,6 +121,7 @@#define ESR_ELx_FSC_SEA_TTW (n) (0x14 + (n))#define ESR_ELx_FSC_SECC (0x18)#define ESR_ELx_FSC_SECC_TTW (n) (0x1c + (n)) +#define ESR_ELx_FSC_TLBABT (0x30)/* Status codes for individual page table levels */ #define ESR_ELx_FSC_ACCESS_L (n) (ESR_ELx_FSC_ACCESS + (n))
@@ -464,6 +465,13 @@ static inline bool esr_fsc_is_access_flag_fault(unsigned long esr) (esr ==ESR_ELx_FSC_ACCESS_L (0 )); } +static inline bool esr_fsc_is_tlb_conflict_abort (unsigned long esr) +{ + esr = esr & ESR_ELx_FSC; + +return esr == ESR_ELx_FSC_TLBABT; +} +/* Indicate whether ESR.EC==0x1A is for an ERETAx instruction */ static inline bool esr_iss_is_eretax (unsigned long esr) {
arch/arm64/include/asm/kvm_asm.h +1 -1
@@ -222,7 +222,7 @@ DECLARE_KVM_NVHE_SYM(__per_cpu_end);DECLARE_KVM_HYP_SYM (__bp_harden_hyp_vecs);#define __bp_harden_hyp_vecs CHOOSE_HYP_SYM(__bp_harden_hyp_vecs) -extern void __kvm_flush_vm_context (void ); +extern void __kvm_flush_vm_context (bool cpu_local);extern void __kvm_flush_cpu_context (struct kvm_s2_mmu *mmu);extern void __kvm_tlb_flush_vmid_ipa (struct kvm_s2_mmu *mmu,phys_addr_t ipa,int level);
arch/arm64/kvm/hyp/nvhe/hyp-main.c +1 -1
@@ -195,7 +195,7 @@ static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)static void handle___kvm_flush_vm_context (struct kvm_cpu_context *host_ctxt) { -__kvm_flush_vm_context (); +__kvm_flush_vm_context (false); }static void handle___kvm_tlb_flush_vmid_ipa (struct kvm_cpu_context *host_ctxt)
arch/arm64/kvm/hyp/nvhe/tlb.c +7 -2
@@ -261,10 +261,15 @@ void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu)exit_vmid_context (&cxt); } -void __kvm_flush_vm_context (void ) +void __kvm_flush_vm_context (bool cpu_local) {/* Same remark as in enter_vmid_context() */ dsb (ish); -__tlbi (alle1is); + +if (cpu_local) +__tlbi (alle1); +else +__tlbi (alle1is); +dsb (ish); }
arch/arm64/kvm/hyp/vhe/tlb.c +7 -2
@@ -213,10 +213,15 @@ void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu)exit_vmid_context (&cxt); } -void __kvm_flush_vm_context (void ) +void __kvm_flush_vm_context (bool cpu_local) {dsb (ishst); -__tlbi (alle1is); + +if (cpu_local) +__tlbi (alle1); +else +__tlbi (alle1is); +dsb (ish); }
arch/arm64/kvm/mmu.c +13 -0
@@ -1756,6 +1756,19 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) ipa = fault_ipa =kvm_vcpu_get_fault_ipa (vcpu); is_iabt =kvm_vcpu_trap_is_iabt (vcpu); +if (esr_fsc_is_tlb_conflict_abort (esr)) { + +/* Architecturely, at this stage 2 tlb conflict abort, we must +* either perform a `tlbi vmalls12e1`, or a `tlbi alle1`. Due +* to nesting of VMs, we would have to iterate all flattened +* VMIDs to clean out a single guest, so we perform a `tlbi alle1` +* instead to save time. +*/ +__kvm_flush_vm_context (true); + +return 1 ; + } +if (esr_fsc_is_translation_fault (esr)) {/* Beyond sanitised PARange (which is the IPA limit) */ if (fault_ipa >=BIT_ULL (get_kvm_ipa_limit ())) {
arch/arm64/kvm/vmid.c +1 -1
@@ -66,7 +66,7 @@ static void flush_context(void)* the next context-switch, we broadcast TLB flush + I-cache * invalidation over the inner shareable domain on rollover. */ -kvm_call_hyp (__kvm_flush_vm_context); +kvm_call_hyp (__kvm_flush_vm_context, false); }static bool check_update_reserved_vmid (u64 vmid,u64 newvmid)
Signature · 1 line
2.45.2
Quoted 5 lines · Mikołaj Lenczewski
Currently, KVM does not handle the case of a stage 2 TLB conflict abort exception. This can legitimately occurs when the guest is eliding full BBM semantics as permitted by BBM level 2. In this case it is possible for a confclit abort to be delivered to EL2. We handle that by invalidating the full TLB.
typo: conflict
Also, a bit of a nitpick, but mentioning that TLB conflict abort routing is implementation defined when S2 is enabled is valuable information.
arch/arm64/kvm/mmu.c int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) · line 1766
Quoted 12 lines
arch/arm64/kvm/mmu.c +8
-0@@ -1756,6 +1756,19 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) ipa = fault_ipa =kvm_vcpu_get_fault_ipa (vcpu); is_iabt =kvm_vcpu_trap_is_iabt (vcpu); +if (esr_fsc_is_tlb_conflict_abort (esr)) { + +/* Architecturely, at this stage 2 tlb conflict abort, we must +* either perform a `tlbi vmalls12e1`, or a `tlbi alle1`. Due +* to nesting of VMs, we would have to iterate all flattened +* VMIDs to clean out a single guest, so we perform a `tlbi alle1` +* instead to save time. +*/
I'm not sure I follow this.
At this point we've taken a TLB conflict abort out of a specific hardware MMU context, and it's unclear to me why a conflict abort in one stage-2 MMU has any bearing on the other stage-2 MMUs that could be associated with this guest.
Even in NV, KVM is always responsible for the maintenance of hardware stage-2 MMUs. So stage-2 TLBI elision in the guest hypervisor should not lead to a stage-2 TLB conflict abort.
TLBI ALLE1 is a larger hammer than what's actually necessary here. Could you perhaps introduce a new invalidation routine, __kvm_tlb_flush_vmid_nsh(), that does a TLBI VMALLS12E1 behind the scenes?
If an NV guest is playing games at stage-1 across VMIDs then it gets to suffer the consequences (additional TLB conflict aborts).
Signature · 2 lines
Thanks, Oliver
Quoted 36 lines · Oliver Upton
Quoted 5 lines · Mikołaj Lenczewski
Currently, KVM does not handle the case of a stage 2 TLB conflict abort exception. This can legitimately occurs when the guest is eliding full BBM semantics as permitted by BBM level 2. In this case it is possible for a confclit abort to be delivered to EL2. We handle that by invalidating the full TLB.
typo: conflict
Also, a bit of a nitpick, but mentioning that TLB conflict abort routing is implementation defined when S2 is enabled is valuable information.
arch/arm64/kvm/mmu.c int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) · line 1766
Quoted 12 lines
arch/arm64/kvm/mmu.c +8
-0@@ -1756,6 +1756,19 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) ipa = fault_ipa =kvm_vcpu_get_fault_ipa (vcpu); is_iabt =kvm_vcpu_trap_is_iabt (vcpu); +if (esr_fsc_is_tlb_conflict_abort (esr)) { + +/* Architecturely, at this stage 2 tlb conflict abort, we must +* either perform a `tlbi vmalls12e1`, or a `tlbi alle1`. Due +* to nesting of VMs, we would have to iterate all flattened +* VMIDs to clean out a single guest, so we perform a `tlbi alle1` +* instead to save time. +*/ I'm not sure I follow this.
At this point we've taken a TLB conflict abort out of a specific hardware MMU context, and it's unclear to me why a conflict abort in one stage-2 MMU has any bearing on the other stage-2 MMUs that could be associated with this guest.
Even in NV, KVM is always responsible for the maintenance of hardware stage-2 MMUs. So stage-2 TLBI elision in the guest hypervisor should not lead to a stage-2 TLB conflict abort.
Quoted 11 lines
TLBI ALLE1 is a larger hammer than what's actually necessary here. Could you perhaps introduce a new invalidation routine,
__kvm_tlb_flush_vmid_nsh(), that does a TLBI VMALLS12E1 behind the scenes?If an NV guest is playing games at stage-1 across VMIDs then it gets to suffer the consequences (additional TLB conflict aborts).
Signature · 2 lines
Thanks, Oliver
Apologies for the very late reply. Thank you for taking the time to review my patch. Will fix the type, and work on fixing the logic there to use a TLBI VMALLS12E1.
Signature · 2 lines
Kind regards, Mikołaj Lenczewski
Marc Zyngier
maintainerQuoted 26 lines · Mikołaj Lenczewski
Currently, KVM does not handle the case of a stage 2 TLB conflict abort exception. This can legitimately occurs when the guest is eliding full BBM semantics as permitted by BBM level 2. In this case it is possible for a confclit abort to be delivered to EL2. We handle that by invalidating the full TLB.
The Arm ARM specifies that the worst-case invalidation is either a
tlbi vmalls12e1or atlbi alle1(as per DDI0487K section D8.16.3). We implementtlbi alle1by extending the existing__kvm_flush_vm_context()helper to allow for differentiating between inner-shareable and cpu-local invalidations.This commit applies on top of v6.13-rc2 (
fac04efc5c79).Signed-off-by Mikołaj Lenczewski <[email protected]>--- arch/arm64/include/asm/esr.h | 8 ++++++++ arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/tlb.c | 9 +++++++-- arch/arm64/kvm/hyp/vhe/tlb.c | 9 +++++++-- arch/arm64/kvm/mmu.c | 13 +++++++++++++ arch/arm64/kvm/vmid.c | 2 +- 7 files changed, 38 insertions(+), 7 deletions(-)
[...]
Quoted 11 lines
arch/arm64/kvm/mmu.c +3
-0@@ -1756,6 +1756,19 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) ipa = fault_ipa =kvm_vcpu_get_fault_ipa (vcpu); is_iabt =kvm_vcpu_trap_is_iabt (vcpu); +if (esr_fsc_is_tlb_conflict_abort (esr)) { + +/* Architecturely, at this stage 2 tlb conflict abort, we must
arch/arm64/kvm/mmu.c int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) · line 1771
Quoted 10 lines
arch/arm64/kvm/mmu.c +10
-0+* either perform a `tlbi vmalls12e1`, or a `tlbi alle1`. Due +* to nesting of VMs, we would have to iterate all flattened +* VMIDs to clean out a single guest, so we perform a `tlbi alle1` +* instead to save time. +*/ +__kvm_flush_vm_context (true); + +return 1 ; + } +
This is broken. At this stage, you are preemptible, so whatever invalidation you are performing might be happening on the wrong CPU (and I really don't want to see a broadcast invalidation).
I really don't see why this can't be handled as a fixup in the inner run loop, which would save *a lot* of cycles and do the right thing.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 57 lines · Marc Zyngier
Quoted 26 lines · Mikołaj Lenczewski
Currently, KVM does not handle the case of a stage 2 TLB conflict abort exception. This can legitimately occurs when the guest is eliding full BBM semantics as permitted by BBM level 2. In this case it is possible for a confclit abort to be delivered to EL2. We handle that by invalidating the full TLB.
The Arm ARM specifies that the worst-case invalidation is either a
tlbi vmalls12e1or atlbi alle1(as per DDI0487K section D8.16.3). We implementtlbi alle1by extending the existing__kvm_flush_vm_context()helper to allow for differentiating between inner-shareable and cpu-local invalidations.This commit applies on top of v6.13-rc2 (
fac04efc5c79).Signed-off-by Mikołaj Lenczewski <[email protected]>--- arch/arm64/include/asm/esr.h | 8 ++++++++ arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/tlb.c | 9 +++++++-- arch/arm64/kvm/hyp/vhe/tlb.c | 9 +++++++-- arch/arm64/kvm/mmu.c | 13 +++++++++++++ arch/arm64/kvm/vmid.c | 2 +- 7 files changed, 38 insertions(+), 7 deletions(-)[...]
Quoted 11 lines
arch/arm64/kvm/mmu.c +3
-0@@ -1756,6 +1756,19 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) ipa = fault_ipa =kvm_vcpu_get_fault_ipa (vcpu); is_iabt =kvm_vcpu_trap_is_iabt (vcpu); +if (esr_fsc_is_tlb_conflict_abort (esr)) { + +/* Architecturely, at this stage 2 tlb conflict abort, we must arch/arm64/kvm/mmu.c int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) · line 1771
Quoted 10 lines
arch/arm64/kvm/mmu.c +10
-0+* either perform a `tlbi vmalls12e1`, or a `tlbi alle1`. Due +* to nesting of VMs, we would have to iterate all flattened +* VMIDs to clean out a single guest, so we perform a `tlbi alle1` +* instead to save time. +*/ +__kvm_flush_vm_context (true); + +return 1 ; + } +This is broken. At this stage, you are preemptible, so whatever invalidation you are performing might be happening on the wrong CPU (and I really don't want to see a broadcast invalidation).
Quoted 9 lines
I really don't see why this can't be handled as a fixup in the inner run loop, which would save *a lot* of cycles and do the right thing.
Thanks,
M.Signature · 1 line
Without deviation from the norm, progress is not possible.
Aplogies for my very late reply. Thank you for tatking the time to review my patch. Will work on this.
Signature · 2 lines
Kind regards, Mikołaj Lenczewski