Skip to contentSkip to conversation

nSVM: Check for optional commands and reserved encodins of TLB_CONTROL in nested guests

PATCH Applied Krish Sadhukhan 4 participants 10 messages Sep 21, 2021 – Sep 6, 2023

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover nSVM: Check for optional commands and reserved encodins of TLB_CONTROL in nested guests

According to section "TLB Flush" in APM vol 2,

"Support for TLB_CONTROL commands other than the first two, is
optional and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].

All encodings of TLB_CONTROL not defined in the APM are reserved."

Patch# 1: Exposes FLUSHBYASID CPUID feature to nestsed guests. Patch# 2: Adds KVM checks for ptional commands and reserved encodings of
TLB_CONTROL.
Patch# 3: Adds #defines for valid encodings of TLB_CONTROL. Patch# 4: Adds #define for FLUSHBYASID CPUID bit.
Patch# 5: Adds kvm-unit-tests for optional commands and reserved encodings
of TLB_CONTROL.

[PATCH 1/5] nSVM: Expose FLUSHBYASID CPUID feature to nested guests
[PATCH 2/5] nSVM: Check for optional commands and reserved encodings of
[TEST PATCH 3/5] SVM: Add #defines for valid encodings of TLB_CONTROL VMCB
[TEST PATCH 4/5] X86: Add #define for FLUSHBYASID CPUID bit
[TEST PATCH 5/5] nSVM: Test optional commands and reserved encodings of TLB_CONTROL in nested VMCB

 arch/x86/kvm/svm/nested.c | 19 +++++++++++++++++++
 arch/x86/kvm/svm/svm.c    |  3 +++
 2 files changed, 22 insertions(+)

Krish Sadhukhan (2):
nSVM: Expose FLUSHBYASID CPUID feature to nested guests
nSVM: Check for optional commands and reserved encodings of TLB_CONTROL in nested VMCB

 lib/x86/processor.h |  1 +
 x86/svm.h           |  5 +++++
 x86/svm_tests.c     | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+)

Krish Sadhukhan (3):
SVM: Add #defines for valid encodings of TLB_CONTROL VMCB field
X86: Add #define for FLUSHBYASID CPUID bit
nSVM: Test optional commands and reserved encodings of TLB_CONTROL in nested VMCB

#1 lore raw

PATCH 1/5 nSVM: Expose FLUSHBYASID CPUID feature to nested guests

The FLUSHBYASID CPUID feature controls the availability of commands 0x3 and 0x7 of TLB_CONTROL. If FLUSHBYASID is supported by the VCPU, those TLB_CONTROL commands will be available to nested guests. Therefore, expose FLUSHBYASID CPUID feature to nested guests.

---
 arch/x86/kvm/svm/svm.c | 3 +++
 1 file changed, 3 insertions(+)
arch/x86/kvm/svm/svm.c +3 -0
index 1a70e11f0487..0f8748af8569 100644
@@ -931,6 +931,9 @@ static __init void svm_set_cpu_caps(void)
 
 		/* Nested VM can receive #VMEXIT instead of triggering #GP */
 		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
+
+		if (boot_cpu_has(X86_FEATURE_FLUSHBYASID))
+			kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
 	}
 
 	/* CPUID 0x80000008 */
Signature · 1 line
2.27.0
maintainer
#2 lore raw

Re: [PATCH 1/5] nSVM: Expose FLUSHBYASID CPUID feature to nested guests

Quoted 3 lines · Krish Sadhukhan

The FLUSHBYASID CPUID feature controls the availability of commands 0x3 and 0x7 of TLB_CONTROL. If FLUSHBYASID is supported by the VCPU, those TLB_CONTROL commands will be available to nested guests.

The hypervisor would have to implement them. Right now the VMCB12 tlb_ctl is not used.

        /* Also overwritten later if necessary.  */
        svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;

So this won't work.

Paolo

Quoted 23 lines

Therefore,
expose FLUSHBYASID CPUID feature to nested guests.

---
  arch/x86/kvm/svm/svm.c | 3 +++
  1 file changed, 3 insertions(+)
arch/x86/kvm/svm/svm.c +3 -0
index 1a70e11f0487..0f8748af8569 100644
@@ -931,6 +931,9 @@ static __init void svm_set_cpu_caps(void)
 
  		/* Nested VM can receive #VMEXIT instead of triggering #GP */
  		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
+
+		if (boot_cpu_has(X86_FEATURE_FLUSHBYASID))
+			kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
  	}
 
  	/* CPUID 0x80000008 */
#3 lore raw

PATCH 2/5 nSVM: Check for optional commands and reserved encodings of TLB_CONTROL in nested VMCB

In reply to Krish Sadhukhan

According to section "TLB Flush" in APM vol 2,

"Support for TLB_CONTROL commands other than the first two, is
optional and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].

All encodings of TLB_CONTROL not defined in the APM are reserved."

---
 arch/x86/kvm/svm/nested.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
arch/x86/kvm/svm/nested.c +19 -0
index 5e13357da21e..028cc2a1f028 100644
@@ -235,6 +235,22 @@ static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
 	    kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
 }
 
+static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl)
+{
+	switch(tlb_ctl) {
+		case TLB_CONTROL_DO_NOTHING:
+		case TLB_CONTROL_FLUSH_ALL_ASID:
+			return true;
+		case TLB_CONTROL_FLUSH_ASID:
+		case TLB_CONTROL_FLUSH_ASID_LOCAL:
+			if (guest_cpuid_has(vcpu, X86_FEATURE_FLUSHBYASID))
+				return true;
+			fallthrough;
+		default:
+			return false;
+	}
+}
+
 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 				       struct vmcb_control_area *control)
 {
@@ -254,6 +270,9 @@ static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 					   IOPM_SIZE)))
 		return false;
 
+	if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl)))
+		return false;
+
 	return true;
 }
 
Signature · 1 line
2.27.0
maintainer Applied
#4 lore raw

Re: [PATCH 2/5] nSVM: Check for optional commands and reserved encodings of TLB_CONTROL in nested VMCB

Quoted 21 lines · Krish Sadhukhan

According to section "TLB Flush" in APM vol 2,

"Support for TLB_CONTROL commands other than the first two, is
optional and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].

All encodings of TLB_CONTROL not defined in the APM are reserved."

---
  arch/x86/kvm/svm/nested.c | 19 +++++++++++++++++++
  1 file changed, 19 insertions(+)
arch/x86/kvm/svm/nested.c +1 -0
index 5e13357da21e..028cc2a1f028 100644
@@ -235,6 +235,22 @@ static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
  	    kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
  }
 
+static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl)

arch/x86/kvm/svm/nested.c static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) · line 248

Quoted 10 lines
arch/x86/kvm/svm/nested.c +10 -0
+{
+	switch(tlb_ctl) {
+		case TLB_CONTROL_DO_NOTHING:
+		case TLB_CONTROL_FLUSH_ALL_ASID:
+			return true;
+		case TLB_CONTROL_FLUSH_ASID:
+		case TLB_CONTROL_FLUSH_ASID_LOCAL:
+			if (guest_cpuid_has(vcpu, X86_FEATURE_FLUSHBYASID))
+				return true;
+			fallthrough;

Since nested FLUSHBYASID is not supported yet, this second set of case labels can go away.

Queued with that change, thanks.

Paolo

Quoted 19 lines
arch/x86/kvm/svm/nested.c +8 -0
+		default:
+			return false;
+	}
+}
+
  static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
  				       struct vmcb_control_area *control)
  {
@@ -254,6 +270,9 @@ static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
  					   IOPM_SIZE)))
  		return false;
 
+	if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl)))
+		return false;
+
  	return true;
  }
 
#5 lore raw

Re: [PATCH 2/5] nSVM: Check for optional commands and reserved encodings of TLB_CONTROL in nested VMCB

Quoted 13 lines · Paolo Bonzini
Quoted 12 lines · Krish Sadhukhan

According to section "TLB Flush" in APM vol 2,

     "Support for TLB_CONTROL commands other than the first two, is optional and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].

      All encodings of TLB_CONTROL not defined in the APM are reserved."

---
  arch/x86/kvm/svm/nested.c | 19 +++++++++++++++++++

  1 file changed, 19 insertions(+)
Quoted 9 lines
Quoted 9 lines
arch/x86/kvm/svm/nested.c +0 -0
index 5e13357da21e..028cc2a1f028 100644
@@ -235,6 +235,22 @@ static bool nested_svm_check_bitmap_pa(struct
 kvm_vcpu *vcpu, u64 pa, u32 size)
           kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
   }

  +static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu

, u8

Quoted 12 lines
Quoted 11 lines

tlb_ctl)
+{
+    switch(tlb_ctl) {
+        case TLB_CONTROL_DO_NOTHING:
+        case TLB_CONTROL_FLUSH_ALL_ASID:
+            return true;
+        case TLB_CONTROL_FLUSH_ASID:
+        case TLB_CONTROL_FLUSH_ASID_LOCAL:
+            if (guest_cpuid_has(vcpu, X86_FEATURE_FLUSHBYASID)) +                return true;
+            fallthrough;

Quoted 7 lines

Since nested FLUSHBYASID is not supported yet, this second set of case labels can go away.

Queued with that change, thanks.

Paolo

Are there any plans to support FLUSHBYASID in the future? It seems VMWare Workstation and ESXi require this feature to run on top of KVM [1]. This means that after the introduction of this check these VMs fail to boot and report missing features. Hence, upgrading to a newer kernel version is not an option for some users.

Sorry if I misunderstood something or if
this is not the right place to
ask.

[1]: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2008583

#6 lore raw

Re: [PATCH 2/5] nSVM: Check for optional commands and reserved encodings of TLB_CONTROL in nested VMCB

Quoted 18 lines · Stefan Sterz
Quoted 17 lines · Paolo Bonzini
Quoted 10 lines · Krish Sadhukhan

+{
+    switch(tlb_ctl) {
+        case TLB_CONTROL_DO_NOTHING:
+        case TLB_CONTROL_FLUSH_ALL_ASID:
+            return true;
+        case TLB_CONTROL_FLUSH_ASID:
+        case TLB_CONTROL_FLUSH_ASID_LOCAL:
+            if (guest_cpuid_has(vcpu, X86_FEATURE_FLUSHBYASID)) +                return true;
+            fallthrough;

Since nested FLUSHBYASID is not supported yet, this second set of case labels can go away.

Queued with that change, thanks.

Quoted 8 lines
Quoted 2 lines

Paolo

Are there any plans to support FLUSHBYASID in the future? It seems VMWare Workstation and ESXi require this feature to run on top of KVM [1]. This means that after the introduction of this check these VMs fail to boot and report missing features. Hence, upgrading to a newer kernel version is not an option for some users.

IIUC, there's two different issues. KVM "broke" Workstation 16 by adding a bogus consistency check. And Workstation 17 managed to make things worse by trying to do the right thing (assert that a feature is supported instead of blindly using it).

I say the above consistency check is bogus because I can't find anything in the APM that states that TLB_CONTROL is actually checked. Unless something is called out in "Canonicalization and Consistency Checks" or listed as MBZ (Must Be Zero), AMD behavior is typically to let software shoot itself in the foot.

So unless I'm missing something, commit 174a921b6975 ("nSVM: Check for reserved encodings of TLB_CONTROL in nested VMCB") should be reverted.

However, that doesn't help Workstation 17. On the other hand, I don't see how Workstation 17 could ever have worked on KVM, since KVM has never advertised FLUSHBYASID to L1.

That said, "supporting" FLUSHBYASID is trivial, KVM just needs to advertise the bit to userspace. That'll work because KVM's TLB flush "handling" for nSVM is to just flush everything on every transition (it's been a TODO for a long, long time).

Quoted 3 lines

Sorry if I misunderstood something or if this is not the right place to ask.

[1]: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2008583

#7 lore raw

PATCH 3/5 SVM: Add #defines for valid encodings of TLB_CONTROL VMCB field

In reply to Krish Sadhukhan
---
 x86/svm.h | 5 +++++
 1 file changed, 5 insertions(+)
x86/svm.h +5 -0
index ae35d08..660956d 100644
@@ -360,6 +360,11 @@ struct __attribute__ ((__packed__)) vmcb {
 
 #define MSR_BITMAP_SIZE 8192
 
+#define TLB_CONTROL_DO_NOTHING			0
+#define TLB_CONTROL_FLUSH_ALL_ASID		1
+#define TLB_CONTROL_FLUSH_ASID			3
+#define TLB_CONTROL_FLUSH_ASID_LOCAL		7
+
 struct svm_test {
 	const char *name;
 	bool (*supported)(void);
Signature · 1 line
2.27.0
#8 lore raw

PATCH 4/5 X86: Add #define for FLUSHBYASID CPUID bit

In reply to Krish Sadhukhan
---
 lib/x86/processor.h | 1 +
 1 file changed, 1 insertion(+)
lib/x86/processor.h +1 -0
index f380321..9c716de 100644
@@ -188,6 +188,7 @@ static inline bool is_intel(void)
 #define	X86_FEATURE_AMD_IBPB		(CPUID(0x80000008, 0, EBX, 12))
 #define	X86_FEATURE_NPT			(CPUID(0x8000000A, 0, EDX, 0))
 #define	X86_FEATURE_NRIPS		(CPUID(0x8000000A, 0, EDX, 3))
+#define	X86_FEATURE_FLUSHBYASID		(CPUID(0x8000000A, 0, EDX, 6))
 #define	X86_FEATURE_VGIF		(CPUID(0x8000000A, 0, EDX, 16))
 
 
Signature · 1 line
2.27.0
#9 lore raw

PATCH 5/5 nSVM: Test optional commands and reserved encodings of TLB_CONTROL in nested VMCB

In reply to Krish Sadhukhan

According to section "TLB Flush" in APM vol 2,

"Support for TLB_CONTROL commands, other than 0x0 and 0x1, is optional
and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].

All encodings of TLB_CONTROL not defined in the APM are reserved."

---
 x86/svm_tests.c | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
x86/svm_tests.c +36 -0
index b998b24..fa1bb89 100644
@@ -2562,6 +2562,41 @@ static void guest_rflags_test_db_handler(struct ex_regs *r)
 	r->rflags &= ~X86_EFLAGS_TF;
 }
 
+/*
+ * Support for TLB_CONTROL commands, other than 0x0 and 0x1, is optional
+ * and is indicated by CPUID Fn8000_000A_EDX[FlushByAsid].
+ * All encodings of TLB_CONTROL not defined in the APM are reserved.
+ */
+static void test_tlb_ctl(void)
+{
+	u64 tlb_ctl_saved = vmcb->control.tlb_ctl;
+	u64 i;
+
+	if (!this_cpu_has(X86_FEATURE_FLUSHBYASID)) {
+		int ret;
+		vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
+		ret = svm_vmrun();
+		report(ret == SVM_EXIT_ERR, "Test TLB_CONTROL: %x, CPU doesn't support FLUSH_ASID (encoding 0x3), VMRUN failure expected",
+		    vmcb->control.tlb_ctl);
+
+		vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID_LOCAL;
+		report(svm_vmrun() == SVM_EXIT_ERR, "Test TLB_CONTROL: %x, CPU doesn't support FLUSH_ASID_LOCAL (encoding 0x7), VMRUN failure expected",
+		    vmcb->control.tlb_ctl);
+	}
+
+	/*
+	 * Test reserved encodings up to 0xf only
+	 */
+	for (i = 0; i <= 0xf; i++) {
+		if (i == 0x2 || (i > 3 && i < 7) || i > 7) {
+			vmcb->control.tlb_ctl = i;
+			report(svm_vmrun() == SVM_EXIT_ERR, "Test TLB_CONTROL: %x, reserved encoding used, VMRUN failure expected", vmcb->control.tlb_ctl);
+		}
+	}
+
+	vmcb->control.tlb_ctl = tlb_ctl_saved;
+}
+
 static void svm_guest_state_test(void)
 {
 	test_set_guest(basic_guest_main);
@@ -2572,6 +2607,7 @@ static void svm_guest_state_test(void)
 	test_dr();
 	test_msrpm_iopm_bitmap_addrs();
 	test_canonicalization();
+	test_tlb_ctl();
 }
 
 extern void guest_rflags_test_guest(struct svm_test *test);
Signature · 1 line
2.27.0