Skip to contentSkip to conversation

KVM: arm64: Fix AT instruction handling

PATCH In review Andre Przywara 4 participants 9 messages Sep 1, 2020 Stable branch

lore
Share & tools
lore
#0 lore raw

cover KVM: arm64: Fix AT instruction handling

In some architectural corner cases, AT instructions can generate an exception, which KVM is not really ready to handle properly. Teach the code to handle this situation gracefully.

This is a backport of the respective upstream patches to v5.4(.61). James prepared these already, but we were lacking the upstream commit ID. I am sending this on his behalf, since he is off this week.

The last two of the patches were tagged Cc: stable already, but did not apply cleanly, hence this specific backport.

Cheers,
Andre.

James Morse (3):
KVM: arm64: Add kvm_extable for vaxoricism code
KVM: arm64: Survive synchronous exceptions caused by AT instructions
KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception

 arch/arm64/include/asm/kvm_arm.h |  3 +-
 arch/arm64/include/asm/kvm_asm.h | 43 +++++++++++++++++++++
 arch/arm64/kernel/vmlinux.lds.S  |  8 ++++
 arch/arm64/kvm/hyp/entry.S       | 15 +++++---
 arch/arm64/kvm/hyp/hyp-entry.S   | 65 ++++++++++++++++++++------------
 arch/arm64/kvm/hyp/switch.c      | 39 +++++++++++++++++--
 6 files changed, 138 insertions(+), 35 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/3 KVM: arm64: Add kvm_extable for vaxoricism code

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug. This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions, generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries in the extable. Because the KVM code is copied for VHE and nVHE, we end up with four entries, half of which correspond with code that isn't mapped.

---
 arch/arm64/include/asm/kvm_asm.h | 15 ++++++++++
 arch/arm64/kernel/vmlinux.lds.S  |  8 +++++
 arch/arm64/kvm/hyp/entry.S       | 15 ++++++----
 arch/arm64/kvm/hyp/hyp-entry.S   | 51 +++++++++++++++++++-------------
 arch/arm64/kvm/hyp/switch.c      | 31 +++++++++++++++++++
 5 files changed, 94 insertions(+), 26 deletions(-)
arch/arm64/include/asm/kvm_asm.h +15 -0
index 44a243754c1b..d929b4c4df1f 100644
@@ -113,6 +113,21 @@ extern u32 __kvm_get_mdcr_el2(void);
 	kern_hyp_va	\vcpu
 .endm
 
+/*
+ * KVM extable for unexpected exceptions.
+ * In the same format _asm_extable, but output to a different section so that
+ * it can be mapped to EL2. The KVM version is not sorted. The caller must
+ * ensure:
+ * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
+ * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
+ */
+.macro	_kvm_extable, from, to
+	.pushsection	__kvm_ex_table, "a"
+	.align		3
+	.long		(\from - .), (\to - .)
+	.popsection
+.endm
+
 #endif
 
 #endif /* __ARM_KVM_ASM_H__ */
arch/arm64/kernel/vmlinux.lds.S +8 -0
index 8d0374ffdd2d..4f77de8ce138 100644
@@ -24,6 +24,13 @@ ENTRY(_text)
 
 jiffies = jiffies_64;
 
+
+#define HYPERVISOR_EXTABLE					\
+	. = ALIGN(SZ_8);					\
+	__start___kvm_ex_table = .;				\
+	*(__kvm_ex_table)					\
+	__stop___kvm_ex_table = .;
+
 #define HYPERVISOR_TEXT					\
 	/*						\
 	 * Align to 4 KB so that			\
@@ -39,6 +46,7 @@ jiffies = jiffies_64;
 	__hyp_idmap_text_end = .;			\
 	__hyp_text_start = .;				\
 	*(.hyp.text)					\
+	HYPERVISOR_EXTABLE				\
 	__hyp_text_end = .;
 
 #define IDMAP_TEXT					\
arch/arm64/kvm/hyp/entry.S +9 -6
index e5cc8d66bf53..dc3d7bc2292f 100644
@@ -173,20 +173,23 @@ alternative_endif
 	// This is our single instruction exception window. A pending
 	// SError is guaranteed to occur at the earliest when we unmask
 	// it, and at the latest just after the ISB.
-	.global	abort_guest_exit_start
 abort_guest_exit_start:
 
 	isb
 
-	.global	abort_guest_exit_end
 abort_guest_exit_end:
 
 	msr	daifset, #4	// Mask aborts
+	ret
+
+	_kvm_extable	abort_guest_exit_start, 9997f
+	_kvm_extable	abort_guest_exit_end, 9997f
+9997:
+	msr	daifset, #4	// Mask aborts
+	mov	x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
 
-	// If the exception took place, restore the EL1 exception
-	// context so that we can report some information.
-	// Merge the exception code with the SError pending bit.
-	tbz	x0, #ARM_EXIT_WITH_SERROR_BIT, 1f
+	// restore the EL1 exception context so that we can report some
+	// information. Merge the exception code with the SError pending bit.
 	msr	elr_el2, x2
 	msr	esr_el2, x3
 	msr	spsr_el2, x4
arch/arm64/kvm/hyp/hyp-entry.S +31 -20

64 lines not on this page · raw message

arch/arm64/kvm/hyp/switch.c +31 -0
index d76a3d39b269..ecfbd7d29d6b 100644
@@ -14,6 +14,7 @@
 
 #include <asm/arch_gicv3.h>
 #include <asm/cpufeature.h>
+#include <asm/extable.h>
 #include <asm/kprobes.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
@@ -25,6 +26,9 @@
 #include <asm/processor.h>
 #include <asm/thread_info.h>
 
+extern struct exception_table_entry __start___kvm_ex_table;
+extern struct exception_table_entry __stop___kvm_ex_table;
+
 /* Check whether the FP regs were dirtied while in the host-side run loop: */
 static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)
 {
@@ -791,3 +795,30 @@ void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
 
 	unreachable();
 }
+
+asmlinkage void __hyp_text kvm_unexpected_el2_exception(void)
+{
+	unsigned long addr, fixup;
+	struct kvm_cpu_context *host_ctxt;
+	struct exception_table_entry *entry, *end;
+	unsigned long elr_el2 = read_sysreg(elr_el2);
+
+	entry = hyp_symbol_addr(__start___kvm_ex_table);
+	end = hyp_symbol_addr(__stop___kvm_ex_table);
+	host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
+
+	while (entry < end) {
+		addr = (unsigned long)&entry->insn + entry->insn;
+		fixup = (unsigned long)&entry->fixup + entry->fixup;
+
+		if (addr != elr_el2) {
+			entry++;
+			continue;
+		}
+
+		write_sysreg(fixup, elr_el2);
+		return;
+	}
+
+	hyp_panic(host_ctxt);
+}
Signature · 1 line
2.17.1
Reviewed-by on v1 1/3
#2 lore raw

Re: [PATCH stable v5.4 1/3] KVM: arm64: Add kvm_extable for vaxoricism code

Hi Andre,

Quoted 14 lines · Andre Przywara

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug. This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions, generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

Quoted 8 lines

The SError vaxorcism code is a one instruction window, so has two entries in the extable. Because the KVM code is copied for VHE and nVHE, we end up with four entries, half of which correspond with code that isn't mapped.

Can you make sure these patches do carry the sign-off chain as we have in mainline? In particular, this is missing:

You can add your own SoB after this.

Thanks,

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#3 lore raw

Re: [PATCH stable v5.4 1/3] KVM: arm64: Add kvm_extable for vaxoricism code

Quoted 28 lines · Marc Zyngier
Quoted 22 lines · Andre Przywara

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug.
This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions,
generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries
in the extable. Because the KVM code is copied for VHE and nVHE, we end up
with four entries, half of which correspond with code that isn't mapped.

Quoted 4 lines
Quoted 7 lines

Can you make sure these patches do carry the sign-off chain as we have in mainline? In particular, this is missing:

You can add your own SoB after this.

Good point. James prepared the backports before we merged the patches into mainline.

BTW, I also corrected a subject typo: s/vaxoricism/vaxorcism/ (not that this is a real word ;)).

Signature · 1 line
Catalin
#4 lore raw

Re: [PATCH stable v5.4 1/3] KVM: arm64: Add kvm_extable for vaxoricism code

In reply to Marc Zyngier
Quoted 28 lines · Marc Zyngier

Hi Andre,

Quoted 20 lines · Andre Przywara

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug. This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions,
generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries in the extable. Because the KVM code is copied for VHE and nVHE, we end up
with four entries, half of which correspond with code that isn't mapped.

Quoted 4 lines
Quoted 7 lines

Can you make sure these patches do carry the sign-off chain as we have in mainline? In particular, this is missing:

You can add your own SoB after this.

Sure, I wasn't sure your review would apply to this version as well. I took the backports from James' kernel.org repo, where they were lacking any of those tags.
So shall I copy all the tags from mainline to all backport versions? Or only to those where the changes were trivial? The backports to before 5.3 seem to be more involved.

Cheers,
Andre.

#5 lore raw

Re: [PATCH stable v5.4 1/3] KVM: arm64: Add kvm_extable for vaxoricism code

Quoted 35 lines · André Przywara
Quoted 34 lines · Marc Zyngier

Hi Andre,

Quoted 20 lines · Andre Przywara

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug. This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions,
generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries in the extable. Because the KVM code is copied for VHE and nVHE, we end up
with four entries, half of which correspond with code that isn't mapped.

Quoted 4 lines

Can you make sure these patches do carry the sign-off chain as we have in mainline? In particular, this is missing:

Quoted 8 lines
Quoted 1 line

You can add your own SoB after this.

Sure, I wasn't sure your review would apply to this version as well. I took the backports from James' kernel.org repo, where they were lacking any of those tags.
So shall I copy all the tags from mainline to all backport versions? Or only to those where the changes were trivial? The backports to before 5.3 seem to be more involved.

My understanding is that the commit log should stay as close to the original as possible, with potential additions to indicate deviations from the upstream patch.

I'm happy to *add* another RB once I have reviewed the patch, just like I add another SoB when I backport a patch on which I signed-off the first place.

Thanks,

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#6 lore raw

Re: [PATCH stable v5.4 1/3] KVM: arm64: Add kvm_extable for vaxoricism code

In reply to André Przywara
Quoted 35 lines · André Przywara
Quoted 34 lines · Marc Zyngier

Hi Andre,

Quoted 20 lines · Andre Przywara

From: James Morse <[email protected]>

commit e9ee186bb735bfc17fa81dbc9aebf268aee5b41e upstream.

KVM has a one instruction window where it will allow an SError exception to be consumed by the hypervisor without treating it as a hypervisor bug. This is used to consume asynchronous external abort that were caused by the guest.

As we are about to add another location that survives unexpected exceptions,
generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries in the extable. Because the KVM code is copied for VHE and nVHE, we end up
with four entries, half of which correspond with code that isn't mapped.

Quoted 4 lines

Can you make sure these patches do carry the sign-off chain as we have in mainline? In particular, this is missing:

Quoted 6 lines
Quoted 1 line

You can add your own SoB after this.

Sure, I wasn't sure your review would apply to this version as well. I took the backports from James' kernel.org repo, where they were lacking any of those tags.
So shall I copy all the tags from mainline to all backport versions?

Yes.

thanks,

greg k-h

#7 lore raw

PATCH 2/3 KVM: arm64: Survive synchronous exceptions caused by AT instructions

In reply to Andre Przywara

From: James Morse <[email protected]>

commit 88a84ccccb3966bcc3f309cdb76092a9892c0260 upstream.

KVM doesn't expect any synchronous exceptions when executing, any such exception leads to a panic(). AT instructions access the guest page tables, and can cause a synchronous external abort to be taken.

The arm-arm is unclear on what should happen if the guest has configured the hardware update of the access-flag, and a memory type in TCR_EL1 that does not support atomic operations. B2.2.6 "Possible implementation restrictions on using atomic instructions" from DDI0487F.a lists synchronous external abort as a possible behaviour of atomic instructions that target memory that isn't writeback cacheable, but the page table walker may behave differently.

Make KVM robust to synchronous exceptions caused by AT instructions. Add a get_user() style helper for AT instructions that returns -EFAULT if an exception was generated.

While KVM's version of the exception table mixes synchronous and asynchronous exceptions, only one of these can occur at each location.

Re-enter the guest when the AT instructions take an exception on the assumption the guest will take the same exception. This isn't guaranteed to make forward progress, as the AT instructions may always walk the page tables, but guest execution may use the translation cached in the TLB.

This isn't a problem, as since commit 5dcd0fdbb492 ("KVM: arm64: Defer guest entry when an asynchronous exception is pending"), KVM will return to the host to process IRQs allowing the rest of the system to keep running.

---
 arch/arm64/include/asm/kvm_asm.h | 28 ++++++++++++++++++++++++++++
 arch/arm64/kvm/hyp/hyp-entry.S   | 14 ++++++++++----
 arch/arm64/kvm/hyp/switch.c      |  8 ++++----
 3 files changed, 42 insertions(+), 8 deletions(-)
arch/arm64/include/asm/kvm_asm.h +28 -0
index d929b4c4df1f..64d79b288434 100644
@@ -88,6 +88,34 @@ extern u32 __kvm_get_mdcr_el2(void);
 		*__hyp_this_cpu_ptr(sym);				\
 	 })
 
+#define __KVM_EXTABLE(from, to)						\
+	"	.pushsection	__kvm_ex_table, \"a\"\n"		\
+	"	.align		3\n"					\
+	"	.long		(" #from " - .), (" #to " - .)\n"	\
+	"	.popsection\n"
+
+
+#define __kvm_at(at_op, addr)						\
+( { 									\
+	int __kvm_at_err = 0;						\
+	u64 spsr, elr;							\
+	asm volatile(							\
+	"	mrs	%1, spsr_el2\n"					\
+	"	mrs	%2, elr_el2\n"					\
+	"1:	at	"at_op", %3\n"					\
+	"	isb\n"							\
+	"	b	9f\n"						\
+	"2:	msr	spsr_el2, %1\n"					\
+	"	msr	elr_el2, %2\n"					\
+	"	mov	%w0, %4\n"					\
+	"9:\n"								\
+	__KVM_EXTABLE(1b, 2b)						\
+	: "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr)		\
+	: "r" (addr), "i" (-EFAULT));					\
+	__kvm_at_err;							\
+} )
+
+
 #else /* __ASSEMBLY__ */
 
 .macro hyp_adr_this_cpu reg, sym, tmp
arch/arm64/kvm/hyp/hyp-entry.S +10 -4
index c8cdadb38719..f36aad0f207b 100644
@@ -166,13 +166,19 @@ el1_error:
 	b	__guest_exit
 
 el2_sync:
-	/* Check for illegal exception return, otherwise panic */
+	/* Check for illegal exception return */
 	mrs	x0, spsr_el2
+	tbnz	x0, #20, 1f
 
-	/* if this was something else, then panic! */
-	tst	x0, #PSR_IL_BIT
-	b.eq	__hyp_panic
+	save_caller_saved_regs_vect
+	stp     x29, x30, [sp, #-16]!
+	bl	kvm_unexpected_el2_exception
+	ldp     x29, x30, [sp], #16
+	restore_caller_saved_regs_vect
+
+	eret
 
+1:
 	/* Let's attempt a recovery from the illegal exception return */
 	get_vcpu_ptr	x1, x0
 	mov	x0, #ARM_EXCEPTION_IL
arch/arm64/kvm/hyp/switch.c +4 -4
index ecfbd7d29d6b..5602d447abff 100644
@@ -261,10 +261,10 @@ static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
 	 * saved the guest context yet, and we may return early...
 	 */
 	par = read_sysreg(par_el1);
-	asm volatile("at s1e1r, %0" : : "r" (far));
-	isb();
-
-	tmp = read_sysreg(par_el1);
+	if (!__kvm_at("s1e1r", far))
+		tmp = read_sysreg(par_el1);
+	else
+		tmp = SYS_PAR_EL1_F; /* back to the guest */
 	write_sysreg(par, par_el1);
 
 	if (unlikely(tmp & SYS_PAR_EL1_F))
Signature · 1 line
2.17.1
#8 lore raw

PATCH 3/3 KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception

In reply to Andre Przywara

From: James Morse <[email protected]>

commit 71a7f8cb1ca4ca7214a700b1243626759b6c11d4 upstream.

AT instructions do a translation table walk and return the result, or the fault in PAR_EL1. KVM uses these to find the IPA when the value is not provided by the CPU in HPFAR_EL1.

If a translation table walk causes an external abort it is taken as an exception, even if it was due to an AT instruction. (DDI0487F.a's D5.2.11 "Synchronous faults generated by address translation instructions")

While we previously made KVM resilient to exceptions taken due to AT instructions, the device access causes mismatched attributes, and may occur speculatively. Prevent this, by forbidding a walk through memory described as device at stage2. Now such AT instructions will report a stage2 fault.

Such a fault will cause KVM to restart the guest. If the AT instructions always walk the page tables, but guest execution uses the translation cached in the TLB, the guest can't make forward progress until the TLB entry is evicted. This isn't a problem, as since commit 5dcd0fdbb492 ("KVM: arm64:
Defer guest entry when an asynchronous exception is pending"), KVM will return to the host to process IRQs allowing the rest of the system to keep running.

---
 arch/arm64/include/asm/kvm_arm.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
arch/arm64/include/asm/kvm_arm.h +2 -1
index ddf9d762ac62..a4ffd9b55e72 100644
@@ -72,11 +72,12 @@
  * IMO:		Override CPSR.I and enable signaling with VI
  * FMO:		Override CPSR.F and enable signaling with VF
  * SWIO:	Turn set/way invalidates into set/way clean+invalidate
+ * PTW:		Take a stage2 fault if a stage1 walk steps in device memory
  */
 #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
 			 HCR_TVM | HCR_BSU_IS | HCR_FB | HCR_TAC | \
 			 HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
-			 HCR_FMO | HCR_IMO)
+			 HCR_FMO | HCR_IMO | HCR_PTW )
 #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
 #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
 #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
Signature · 1 line
2.17.1