Skip to contentSkip to conversation

KVM: SVM: Inhibit APIC virtualization for X2APIC guest

PATCH Superseded Oliver Upton 2 participants 4 messages Feb 28, 2020

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 KVM: SVM: Inhibit APIC virtualization for X2APIC guest

The AVIC does not support guest use of the x2APIC interface. Currently, KVM simply chooses to squash the x2APIC feature in the guest's CPUID If the AVIC is enabled. Doing so prevents KVM from running a guest with greater than 255 vCPUs, as such a guest necessitates the use of the x2APIC interface.

Instead, inhibit AVIC enablement on a per-VM basis whenever the x2APIC feature is set in the guest's CPUID. Since this changes the behavior of KVM as seen by userspace, add a module parameter, avic_per_vm, to opt-in for the new behavior. If this parameter is set, report x2APIC as available on KVM_GET_SUPPORTED_CPUID. Without opt-in, continue to suppress x2APIC from the guest's CPUID.

---

Parent commit: a93236fcbe1d ("KVM: s390: rstify new ioctls in api.rst")

 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm.c              | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
index 98959e8cd448..9d40132a3ae2 100644
@@ -890,6 +890,7 @@ enum kvm_irqchip_mode {
 #define APICV_INHIBIT_REASON_NESTED     2
 #define APICV_INHIBIT_REASON_IRQWIN     3
 #define APICV_INHIBIT_REASON_PIT_REINJ  4
+#define APICV_INHIBIT_REASON_X2APIC	5
 
 struct kvm_arch {
 	unsigned long n_used_mmu_pages;
arch/x86/kvm/svm.c +16 -3
index ad3f5b178a03..95c03c75f51a 100644
@@ -382,6 +382,10 @@ module_param(sev, int, 0444);
 static bool __read_mostly dump_invalid_vmcb = 0;
 module_param(dump_invalid_vmcb, bool, 0644);
 
+/* enable/disable opportunistic use of the AVIC on a per-VM basis */
+static bool __read_mostly avic_per_vm;
+module_param(avic_per_vm, bool, 0444);
+
 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 
 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
@@ -6027,7 +6031,15 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
 	if (!kvm_vcpu_apicv_active(vcpu))
 		return;
 
-	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
+	/*
+	 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
+	 * is exposed to the guest, disable AVIC.
+	 */
+	if (avic_per_vm && guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
+		kvm_request_apicv_update(vcpu->kvm, false,
+					 APICV_INHIBIT_REASON_X2APIC);
+	else
+		guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
 
 	/*
 	 * Currently, AVIC does not work with nested virtualization.
@@ -6044,7 +6056,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 {
 	switch (func) {
 	case 0x1:
-		if (avic)
+		if (avic && !avic_per_vm)
 			entry->ecx &= ~F(X2APIC);
 		break;
 	case 0x80000001:
@@ -7370,7 +7382,8 @@ static bool svm_check_apicv_inhibit_reasons(ulong bit)
 			  BIT(APICV_INHIBIT_REASON_HYPERV) |
 			  BIT(APICV_INHIBIT_REASON_NESTED) |
 			  BIT(APICV_INHIBIT_REASON_IRQWIN) |
-			  BIT(APICV_INHIBIT_REASON_PIT_REINJ);
+			  BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
+			  BIT(APICV_INHIBIT_REASON_X2APIC);
 
 	return supported & BIT(bit);
 }
Signature · 1 line
2.25.1.481.gfbce0eb801-goog
maintainer
#1 lore raw
Quoted 13 lines · Oliver Upton

The AVIC does not support guest use of the x2APIC interface. Currently, KVM simply chooses to squash the x2APIC feature in the guest's CPUID If the AVIC is enabled. Doing so prevents KVM from running a guest with greater than 255 vCPUs, as such a guest necessitates the use of the x2APIC interface.

Instead, inhibit AVIC enablement on a per-VM basis whenever the x2APIC feature is set in the guest's CPUID. Since this changes the behavior of KVM as seen by userspace, add a module parameter, avic_per_vm, to opt-in for the new behavior. If this parameter is set, report x2APIC as available on KVM_GET_SUPPORTED_CPUID. Without opt-in, continue to suppress x2APIC from the guest's CPUID.

Quoted 2 lines

Since AVIC is not enabled by default, let's do this always and flip the default to avic=1 in 5.7. People using avic=1 will have to disable x2apic manually instead but the default will be the same in practice (AVIC not enabled). And then we can figure out:

- how to do emulation of x2apic when avic is enabled (so it will cause vmexits but still use the AVIC for e.g. assigned devices)

- a PV CPUID leaf to tell <=255 vCPUs on AMD virtualization _not_ to use x2apic.

Paolo

Quoted 72 lines

---

Parent commit: a93236fcbe1d ("KVM: s390: rstify new ioctls in api.rst")

 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm.c              | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
index 98959e8cd448..9d40132a3ae2 100644
@@ -890,6 +890,7 @@ enum kvm_irqchip_mode {
 #define APICV_INHIBIT_REASON_NESTED     2
 #define APICV_INHIBIT_REASON_IRQWIN     3
 #define APICV_INHIBIT_REASON_PIT_REINJ  4
+#define APICV_INHIBIT_REASON_X2APIC	5
 
 struct kvm_arch {
 	unsigned long n_used_mmu_pages;
arch/x86/kvm/svm.c +16 -3
index ad3f5b178a03..95c03c75f51a 100644
@@ -382,6 +382,10 @@ module_param(sev, int, 0444);
 static bool __read_mostly dump_invalid_vmcb = 0;
 module_param(dump_invalid_vmcb, bool, 0644);
 
+/* enable/disable opportunistic use of the AVIC on a per-VM basis */
+static bool __read_mostly avic_per_vm;
+module_param(avic_per_vm, bool, 0444);
+
 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 
 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
@@ -6027,7 +6031,15 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
 	if (!kvm_vcpu_apicv_active(vcpu))
 		return;
 
-	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
+	/*
+	 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
+	 * is exposed to the guest, disable AVIC.
+	 */
+	if (avic_per_vm && guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
+		kvm_request_apicv_update(vcpu->kvm, false,
+					 APICV_INHIBIT_REASON_X2APIC);
+	else
+		guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
 
 	/*
 	 * Currently, AVIC does not work with nested virtualization.
@@ -6044,7 +6056,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 {
 	switch (func) {
 	case 0x1:
-		if (avic)
+		if (avic && !avic_per_vm)
 			entry->ecx &= ~F(X2APIC);
 		break;
 	case 0x80000001:
@@ -7370,7 +7382,8 @@ static bool svm_check_apicv_inhibit_reasons(ulong bit)
 			  BIT(APICV_INHIBIT_REASON_HYPERV) |
 			  BIT(APICV_INHIBIT_REASON_NESTED) |
 			  BIT(APICV_INHIBIT_REASON_IRQWIN) |
-			  BIT(APICV_INHIBIT_REASON_PIT_REINJ);
+			  BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
+			  BIT(APICV_INHIBIT_REASON_X2APIC);
 
 	return supported & BIT(bit);
 }
#2 lore raw

Hi Paolo,

Quoted 14 lines · Paolo Bonzini
Quoted 13 lines · Oliver Upton

The AVIC does not support guest use of the x2APIC interface. Currently, KVM simply chooses to squash the x2APIC feature in the guest's CPUID If the AVIC is enabled. Doing so prevents KVM from running a guest with greater than 255 vCPUs, as such a guest necessitates the use of the x2APIC interface.

Instead, inhibit AVIC enablement on a per-VM basis whenever the x2APIC feature is set in the guest's CPUID. Since this changes the behavior of KVM as seen by userspace, add a module parameter, avic_per_vm, to opt-in for the new behavior. If this parameter is set, report x2APIC as available on KVM_GET_SUPPORTED_CPUID. Without opt-in, continue to suppress x2APIC from the guest's CPUID.

Quoted 8 lines
Quoted 2 lines

Since AVIC is not enabled by default, let's do this always and flip the default to avic=1 in 5.7. People using avic=1 will have to disable x2apic manually instead but the default will be the same in practice (AVIC not enabled). And then we can figure out:

I'll drop the new module param in v2 and adopt this suggested behavior.

Quoted 6 lines

- how to do emulation of x2apic when avic is enabled (so it will cause vmexits but still use the AVIC for e.g. assigned devices)

- a PV CPUID leaf to tell <=255 vCPUs on AMD virtualization _not_ to use x2apic.

If a VMM didn't want the guest to use x2APIC in the first place, shouldn't it instead omit x2APIC from the guest CPUID? I can see a point for this if folks are inclined to use the same CPUID for VMs regardless of shape. Just a thought to tackle later down the road :)

Thanks for the review, I'll send a new patch out shortly.

--
Thanks,
Oliver

Quoted 75 lines

Paolo

Quoted 72 lines

---

Parent commit: a93236fcbe1d ("KVM: s390: rstify new ioctls in api.rst")

 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/svm.c              | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -0
index 98959e8cd448..9d40132a3ae2 100644
@@ -890,6 +890,7 @@ enum kvm_irqchip_mode {
 #define APICV_INHIBIT_REASON_NESTED     2
 #define APICV_INHIBIT_REASON_IRQWIN     3
 #define APICV_INHIBIT_REASON_PIT_REINJ  4
+#define APICV_INHIBIT_REASON_X2APIC	5
 
 struct kvm_arch {
 	unsigned long n_used_mmu_pages;
arch/x86/kvm/svm.c +16 -3
index ad3f5b178a03..95c03c75f51a 100644
@@ -382,6 +382,10 @@ module_param(sev, int, 0444);
 static bool __read_mostly dump_invalid_vmcb = 0;
 module_param(dump_invalid_vmcb, bool, 0644);
 
+/* enable/disable opportunistic use of the AVIC on a per-VM basis */
+static bool __read_mostly avic_per_vm;
+module_param(avic_per_vm, bool, 0444);
+
 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 
 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
@@ -6027,7 +6031,15 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
 	if (!kvm_vcpu_apicv_active(vcpu))
 		return;
 
-	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
+	/*
+	 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
+	 * is exposed to the guest, disable AVIC.
+	 */
+	if (avic_per_vm && guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
+		kvm_request_apicv_update(vcpu->kvm, false,
+					 APICV_INHIBIT_REASON_X2APIC);
+	else
+		guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
 
 	/*
 	 * Currently, AVIC does not work with nested virtualization.
@@ -6044,7 +6056,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 {
 	switch (func) {
 	case 0x1:
-		if (avic)
+		if (avic && !avic_per_vm)
 			entry->ecx &= ~F(X2APIC);
 		break;
 	case 0x80000001:
@@ -7370,7 +7382,8 @@ static bool svm_check_apicv_inhibit_reasons(ulong bit)
 			  BIT(APICV_INHIBIT_REASON_HYPERV) |
 			  BIT(APICV_INHIBIT_REASON_NESTED) |
 			  BIT(APICV_INHIBIT_REASON_IRQWIN) |
-			  BIT(APICV_INHIBIT_REASON_PIT_REINJ);
+			  BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
+			  BIT(APICV_INHIBIT_REASON_X2APIC);
 
 	return supported & BIT(bit);
 }
maintainer
#3 lore raw
Quoted 7 lines · Oliver Upton
Quoted 3 lines

- a PV CPUID leaf to tell <=255 vCPUs on AMD virtualization _not_ to use x2apic.

If a VMM didn't want the guest to use x2APIC in the first place, shouldn't it instead omit x2APIC from the guest CPUID? I can see a point for this if folks are inclined to use the same CPUID for VMs regardless of shape. Just a thought to tackle later down the road :)

Yes, it's mostly for simplicity of configuration.

Paolo