Skip to contentSkip to conversation

x86/hyperv: make vapic support x2apic mode

PATCH Superseded Roman Kagan 3 participants 3 messages Oct 9, 2019

Subsystems (4)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 x86/hyperv: make vapic support x2apic mode

Now that there's Hyper-V IOMMU driver, Linux can switch to x2apic mode when supported by the vcpus.

However, the apic access functions for Hyper-V enlightened apic assume xapic mode only.

As a result, Linux fails to bring up secondary cpus when run as a guest in QEMU/KVM with both hv_apic and x2apic enabled.

According to Michael Kelley, when in x2apic mode, the Hyper-V synthetic apic MSRs behave exactly the same as the corresponding architectural x2apic MSRs, so there's no need to override the apic accessors. The only exception is hv_apic_eoi_write, which benefits from lazy EOI when available; however, its implementation works for both xapic and x2apic modes.

Fixes 29217a474683 ("iommu/hyper-v: Add Hyper-V stub IOMMU driver")
Fixes 6b48cb5f8347 ("X86/Hyper-V: Enlighten APIC access")

---
v2 -> v3:
- do not introduce x2apic-capable hv_apic accessors; leave original x2apic accessors instead

v1 -> v2:
- add ifdefs to handle !CONFIG_X86_X2APIC

 arch/x86/hyperv/hv_apic.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
arch/x86/hyperv/hv_apic.c +13 -4
index 5c056b8aebef..26eeff5bd535 100644
@@ -261,10 +261,19 @@ void __init hv_apic_init(void)
 
 	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
 		pr_info("Hyper-V: Using MSR based APIC access\n");
+		/*
+		 * With x2apic, architectural x2apic MSRs are equivalent to the
+		 * respective synthetic MSRs, so there's no need to override
+		 * the apic accessors.  The only exception is
+		 * hv_apic_eoi_write, because it benefits from lazy EOI when
+		 * available, but it works for both xapic and x2apic modes.
+		 */
 		apic_set_eoi_write(hv_apic_eoi_write);
-		apic->read      = hv_apic_read;
-		apic->write     = hv_apic_write;
-		apic->icr_write = hv_apic_icr_write;
-		apic->icr_read  = hv_apic_icr_read;
+		if (!x2apic_enabled()) {
+			apic->read      = hv_apic_read;
+			apic->write     = hv_apic_write;
+			apic->icr_write = hv_apic_icr_write;
+			apic->icr_read  = hv_apic_icr_read;
+		}
 	}
 }
Signature · 1 line
2.21.0
Reviewed-by on v3 1/1
#1 lore raw
Quoted 37 lines · Roman Kagan

Now that there's Hyper-V IOMMU driver, Linux can switch to x2apic mode when supported by the vcpus.

However, the apic access functions for Hyper-V enlightened apic assume xapic mode only.

As a result, Linux fails to bring up secondary cpus when run as a guest in QEMU/KVM with both hv_apic and x2apic enabled.

According to Michael Kelley, when in x2apic mode, the Hyper-V synthetic apic MSRs behave exactly the same as the corresponding architectural x2apic MSRs, so there's no need to override the apic accessors. The only exception is hv_apic_eoi_write, which benefits from lazy EOI when available; however, its implementation works for both xapic and x2apic modes.

Fixes 29217a474683 ("iommu/hyper-v: Add Hyper-V stub IOMMU driver")
Fixes 6b48cb5f8347 ("X86/Hyper-V: Enlighten APIC access")

---
v2 -> v3:
- do not introduce x2apic-capable hv_apic accessors; leave original x2apic accessors instead

v1 -> v2:
- add ifdefs to handle !CONFIG_X86_X2APIC

 arch/x86/hyperv/hv_apic.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
arch/x86/hyperv/hv_apic.c +0 -0
index 5c056b8aebef..26eeff5bd535 100644

arch/x86/hyperv/hv_apic.c void __init hv_apic_init(void) · line 263

Quoted 3 lines
arch/x86/hyperv/hv_apic.c +0 -0
@@ -261,10 +261,19 @@ void __init hv_apic_init(void)
 
 	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
 		pr_info("Hyper-V: Using MSR based APIC access\n");

This pr_info() becomes a bit misleading in x2apic mode, maybe do something like

pr_info("Hyper-V: using Enlightened APIC (%s mode)",

        x2apic_enabled() ? "x2apic" : "xapic");
Quoted 10 lines
arch/x86/hyperv/hv_apic.c +7 -2
+		/*
+		 * With x2apic, architectural x2apic MSRs are equivalent to the
+		 * respective synthetic MSRs, so there's no need to override
+		 * the apic accessors.  The only exception is
+		 * hv_apic_eoi_write, because it benefits from lazy EOI when
+		 * available, but it works for both xapic and x2apic modes.
+		 */
 		apic_set_eoi_write(hv_apic_eoi_write);
-		apic->read      = hv_apic_read;
-		apic->write     = hv_apic_write;

arch/x86/hyperv/hv_apic.c void __init hv_apic_init(void) · line 279

Quoted 10 lines
arch/x86/hyperv/hv_apic.c +6 -2
-		apic->icr_write = hv_apic_icr_write;
-		apic->icr_read  = hv_apic_icr_read;
+		if (!x2apic_enabled()) {
+			apic->read      = hv_apic_read;
+			apic->write     = hv_apic_write;
+			apic->icr_write = hv_apic_icr_write;
+			apic->icr_read  = hv_apic_icr_read;
+		}
 	}
 }
Signature · 1 line
Vitaly
Reviewed-by
#2 lore raw

From: Vitaly Kuznetsov <[email protected]> Sent: Wednesday, October 9, 2019 8:27 AM

Quoted 41 lines
Quoted 38 lines · Roman Kagan

Now that there's Hyper-V IOMMU driver, Linux can switch to x2apic mode when supported by the vcpus.

However, the apic access functions for Hyper-V enlightened apic assume xapic mode only.

As a result, Linux fails to bring up secondary cpus when run as a guest in QEMU/KVM with both hv_apic and x2apic enabled.

According to Michael Kelley, when in x2apic mode, the Hyper-V synthetic apic MSRs behave exactly the same as the corresponding architectural x2apic MSRs, so there's no need to override the apic accessors. The only exception is hv_apic_eoi_write, which benefits from lazy EOI when available; however, its implementation works for both xapic and x2apic modes.

Fixes 29217a474683 ("iommu/hyper-v: Add Hyper-V stub IOMMU driver")
Fixes 6b48cb5f8347 ("X86/Hyper-V: Enlighten APIC access")

---
v2 -> v3:
- do not introduce x2apic-capable hv_apic accessors; leave original x2apic accessors instead

v1 -> v2:
- add ifdefs to handle !CONFIG_X86_X2APIC

 arch/x86/hyperv/hv_apic.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
arch/x86/hyperv/hv_apic.c +0 -0
index 5c056b8aebef..26eeff5bd535 100644
@@ -261,10 +261,19 @@ void __init hv_apic_init(void)
 
Quoted 8 lines

arch/x86/hyperv/hv_apic.c void __init hv_apic_init(void) · line 263

Quoted 2 lines
arch/x86/hyperv/hv_apic.c +0 -0
 	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
 		pr_info("Hyper-V: Using MSR based APIC access\n");

This pr_info() becomes a bit misleading in x2apic mode, maybe do something like

pr_info("Hyper-V: using Enlightened APIC (%s mode)",

        x2apic_enabled() ? "x2apic" : "xapic");

Yes, I like this. But tweak the capitalization of the message:

pr_info("Hyper-V: Using enlightened APIC (%s mode)",

Quoted 15 lines
Quoted 15 lines
arch/x86/hyperv/hv_apic.c +9 -4
+		/*
+		 * With x2apic, architectural x2apic MSRs are equivalent to the
+		 * respective synthetic MSRs, so there's no need to override
+		 * the apic accessors.  The only exception is
+		 * hv_apic_eoi_write, because it benefits from lazy EOI when
+		 * available, but it works for both xapic and x2apic modes.
+		 */
 		apic_set_eoi_write(hv_apic_eoi_write);
-		apic->read      = hv_apic_read;
-		apic->write     = hv_apic_write;
-		apic->icr_write = hv_apic_icr_write;
-		apic->icr_read  = hv_apic_icr_read;
+		if (!x2apic_enabled()) {
+			apic->read      = hv_apic_read;
Quoted 9 lines

arch/x86/hyperv/hv_apic.c void __init hv_apic_init(void) · line 279

Quoted 6 lines
arch/x86/hyperv/hv_apic.c +4 -0
+			apic->write     = hv_apic_write;
+			apic->icr_write = hv_apic_icr_write;
+			apic->icr_read  = hv_apic_icr_read;
+		}
 	}
 }