Skip to contentSkip to conversation

xen: HVM X2APIC support

PATCH Superseded Sheng Yang 3 participants 6 messages Dec 2, 2010 – Dec 3, 2010

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Sheng Yang

author
#0 lore raw

PATCH 1/1 xen: HVM X2APIC support

This patch is similiar to Gleb Natapov's patch for KVM, which enable the hypervisor to emulate x2apic feature for the guest. By this way, the emulation of lapic would be simpler with x2apic interface(MSR), and faster.

---
 arch/x86/include/asm/xen/hypervisor.h |   33 +++++++++++++++++++++++++++++++++
 arch/x86/kernel/apic/apic.c           |    4 +++-
 arch/x86/xen/enlighten.c              |   19 -------------------
 3 files changed, 36 insertions(+), 20 deletions(-)
arch/x86/include/asm/xen/hypervisor.h +33 -0
index 396ff4c..e862874 100644
@@ -37,4 +37,37 @@
 extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
+#include <asm/processor.h>
+
+static inline uint32_t xen_cpuid_base(void)
+{
+	uint32_t base, eax, ebx, ecx, edx;
+	char signature[13];
+
+	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+		cpuid(base, &eax, &ebx, &ecx, &edx);
+		*(uint32_t *)(signature + 0) = ebx;
+		*(uint32_t *)(signature + 4) = ecx;
+		*(uint32_t *)(signature + 8) = edx;
+		signature[12] = 0;
+
+		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
+			return base;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_XEN
+static inline bool xen_para_available(void)
+{
+	return 0;
+}
+#else
+static inline bool xen_para_available(void)
+{
+	return (xen_cpuid_base() != 0);
+}
+#endif
+
 #endif /* _ASM_X86_XEN_HYPERVISOR_H */
arch/x86/kernel/apic/apic.c +3 -1
index 3f838d5..1b68221 100644
@@ -52,6 +52,7 @@
 #include <asm/mce.h>
 #include <asm/kvm_para.h>
 #include <asm/tsc.h>
+#include <asm/xen/hypervisor.h>
 
 unsigned int num_processors;
 
@@ -1476,7 +1477,8 @@ void __init enable_IR_x2apic(void)
 		/* IR is required if there is APIC ID > 255 even when running
 		 * under KVM
 		 */
-		if (max_physical_apicid > 255 || !kvm_para_available())
+		if (max_physical_apicid > 255 ||
+		    (!kvm_para_available() && !xen_para_available()))
 			goto nox2apic;
 		/*
 		 * without IR all CPUs can be addressed by IOAPIC/MSI
arch/x86/xen/enlighten.c +0 -19
index 235c0f4..819c365 100644
@@ -1245,25 +1245,6 @@ asmlinkage void __init xen_start_kernel(void)
 #endif
 }
 
-static uint32_t xen_cpuid_base(void)
-{
-	uint32_t base, eax, ebx, ecx, edx;
-	char signature[13];
-
-	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
-		cpuid(base, &eax, &ebx, &ecx, &edx);
-		*(uint32_t *)(signature + 0) = ebx;
-		*(uint32_t *)(signature + 4) = ecx;
-		*(uint32_t *)(signature + 8) = edx;
-		signature[12] = 0;
-
-		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
-			return base;
-	}
-
-	return 0;
-}
-
 static int init_hvm_pv_info(int *major, int *minor)
 {
 	uint32_t eax, ebx, ecx, edx, pages, msr, base;
Signature · 1 line
1.7.0.1
#1 lore raw
Quoted 3 lines · Sheng Yang

This patch is similiar to Gleb Natapov's patch for KVM, which enable the hypervisor to emulate x2apic feature for the guest. By this way, the emulation of lapic would be simpler with x2apic interface(MSR), and faster.

We have a set of patches to directly use event channels from within hvm domains, completely bypassing the apic altogether. Do we need this as well?

Quoted 37 lines
---
 arch/x86/include/asm/xen/hypervisor.h |   33 +++++++++++++++++++++++++++++++++
 arch/x86/kernel/apic/apic.c           |    4 +++-
 arch/x86/xen/enlighten.c              |   19 -------------------
 3 files changed, 36 insertions(+), 20 deletions(-)
arch/x86/include/asm/xen/hypervisor.h +22 -0
index 396ff4c..e862874 100644
@@ -37,4 +37,37 @@
 extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
+#include <asm/processor.h>
+
+static inline uint32_t xen_cpuid_base(void)
+{
+	uint32_t base, eax, ebx, ecx, edx;
+	char signature[13];
+
+	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+		cpuid(base, &eax, &ebx, &ecx, &edx);
+		*(uint32_t *)(signature + 0) = ebx;
+		*(uint32_t *)(signature + 4) = ecx;
+		*(uint32_t *)(signature + 8) = edx;
+		signature[12] = 0;
+
+		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
+			return base;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_XEN

arch/x86/include/asm/xen/hypervisor.h line 71

Quoted 10 lines
arch/x86/include/asm/xen/hypervisor.h +10 -0
+static inline bool xen_para_available(void)
+{
+	return 0;
+}
+#else
+static inline bool xen_para_available(void)
+{
+	return (xen_cpuid_base() != 0);
+}
+#endif

So this returns true if you're running a kernel without CONFIG_XEN under Xen? Does that assume that all versions of Xen implement x2apic emulation? Why wouldn't we also want this for CONFIG_XEN kernels?

J

Quoted 54 lines
arch/x86/include/asm/xen/hypervisor.h +1 -0
+
 #endif /* _ASM_X86_XEN_HYPERVISOR_H */
arch/x86/kernel/apic/apic.c +3 -1
index 3f838d5..1b68221 100644
@@ -52,6 +52,7 @@
 #include <asm/mce.h>
 #include <asm/kvm_para.h>
 #include <asm/tsc.h>
+#include <asm/xen/hypervisor.h>
 
 unsigned int num_processors;
 
@@ -1476,7 +1477,8 @@ void __init enable_IR_x2apic(void)
 		/* IR is required if there is APIC ID > 255 even when running
 		 * under KVM
 		 */
-		if (max_physical_apicid > 255 || !kvm_para_available())
+		if (max_physical_apicid > 255 ||
+		    (!kvm_para_available() && !xen_para_available()))
 			goto nox2apic;
 		/*
 		 * without IR all CPUs can be addressed by IOAPIC/MSI
arch/x86/xen/enlighten.c +0 -19
index 235c0f4..819c365 100644
@@ -1245,25 +1245,6 @@ asmlinkage void __init xen_start_kernel(void)
 #endif
 }
 
-static uint32_t xen_cpuid_base(void)
-{
-	uint32_t base, eax, ebx, ecx, edx;
-	char signature[13];
-
-	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
-		cpuid(base, &eax, &ebx, &ecx, &edx);
-		*(uint32_t *)(signature + 0) = ebx;
-		*(uint32_t *)(signature + 4) = ecx;
-		*(uint32_t *)(signature + 8) = edx;
-		signature[12] = 0;
-
-		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
-			return base;
-	}
-
-	return 0;
-}
-
 static int init_hvm_pv_info(int *major, int *minor)
 {
 	uint32_t eax, ebx, ecx, edx, pages, msr, base;
author
#2 lore raw
Quoted 9 lines · On Thursday 02 December 2010 14:28:16 Jeremy Fitzhardinge wrote:
Quoted 4 lines · Sheng Yang

This patch is similiar to Gleb Natapov's patch for KVM, which enable the hypervisor to emulate x2apic feature for the guest. By this way, the emulation of lapic would be simpler with x2apic interface(MSR), and faster.

We have a set of patches to directly use event channels from within hvm domains, completely bypassing the apic altogether. Do we need this as well?

This is for other HVMs. And the pvhvm still have limitation like it can't use MSI/MSI-X assigned device.

Quoted 47 lines
Quoted 47 lines

---

 arch/x86/include/asm/xen/hypervisor.h |   33
 +++++++++++++++++++++++++++++++++ arch/x86/kernel/apic/apic.c          
 |    4 +++-
 arch/x86/xen/enlighten.c              |   19 -------------------
 3 files changed, 36 insertions(+), 20 deletions(-)
(quoted diff) +0 -0

b/arch/x86/include/asm/xen/hypervisor.h index 396ff4c..e862874 100644

arch/x86/include/asm/xen/hypervisor.h +28 -0
@@ -37,4 +37,37 @@
 
 extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
+#include <asm/processor.h>
+
+static inline uint32_t xen_cpuid_base(void)
+{
+	uint32_t base, eax, ebx, ecx, edx;
+	char signature[13];
+
+	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+		cpuid(base, &eax, &ebx, &ecx, &edx);
+		*(uint32_t *)(signature + 0) = ebx;
+		*(uint32_t *)(signature + 4) = ecx;
+		*(uint32_t *)(signature + 8) = edx;
+		signature[12] = 0;
+
+		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
+			return base;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_XEN
+static inline bool xen_para_available(void)
+{
+	return 0;
+}
+#else
+static inline bool xen_para_available(void)
Quoted 8 lines

arch/x86/include/asm/xen/hypervisor.h line 71

Quoted 4 lines
arch/x86/include/asm/xen/hypervisor.h +4 -0
+{
+	return (xen_cpuid_base() != 0);
+}
+#endif

So this returns true if you're running a kernel without CONFIG_XEN under Xen? Does that assume that all versions of Xen implement x2apic emulation? Why wouldn't we also want this for CONFIG_XEN kernels?

Because only the ones that implement the feature would expose x2apic CPUID.

For CONFIG_XEN(pv or pvhvm), they both use evtchn, so no need for x2apic.

--
regards
Yang, Sheng

Quoted 71 lines

J

Quoted 68 lines
arch/x86/include/asm/xen/hypervisor.h +1 -0
+

#endif /* _ASM_X86_XEN_HYPERVISOR_H */

arch/x86/kernel/apic/apic.c +1 -0
index 3f838d5..1b68221 100644
@@ -52,6 +52,7 @@
 
 #include <asm/mce.h>
 #include <asm/kvm_para.h>
 #include <asm/tsc.h>
 
+#include <asm/xen/hypervisor.h>
 

unsigned int num_processors;

arch/x86/kernel/apic/apic.c +2 -1
@@ -1476,7 +1477,8 @@ void __init enable_IR_x2apic(void)
 
 		/* IR is required if there is APIC ID > 255 even when running
 
 		 * under KVM
 		 */
 
-		if (max_physical_apicid > 255 || !kvm_para_available())
+		if (max_physical_apicid > 255 ||
+		    (!kvm_para_available() && !xen_para_available()))
arch/x86/kernel/apic/apic.c +0 -0
 			goto nox2apic;
 
 		/*
 
 		 * without IR all CPUs can be addressed by IOAPIC/MSI
arch/x86/xen/enlighten.c +0 -19
index 235c0f4..819c365 100644
@@ -1245,25 +1245,6 @@ asmlinkage void __init xen_start_kernel(void)
 
 #endif
 }
 
-static uint32_t xen_cpuid_base(void)
-{
-	uint32_t base, eax, ebx, ecx, edx;
-	char signature[13];
-
-	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
-		cpuid(base, &eax, &ebx, &ecx, &edx);
-		*(uint32_t *)(signature + 0) = ebx;
-		*(uint32_t *)(signature + 4) = ecx;
-		*(uint32_t *)(signature + 8) = edx;
-		signature[12] = 0;
-
-		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
-			return base;
-	}
-
-	return 0;
-}
-
 
 static int init_hvm_pv_info(int *major, int *minor)

{

uint32_t eax, ebx, ecx, edx, pages, msr, base;

#3 lore raw
Quoted 7 lines · Sheng Yang
Quoted 6 lines · On Thursday 02 December 2010 14:28:16 Jeremy Fitzhardinge wrote:
Quoted 4 lines · Sheng Yang

This patch is similiar to Gleb Natapov's patch for KVM, which enable the hypervisor to emulate x2apic feature for the guest. By this way, the emulation of lapic would be simpler with x2apic interface(MSR), and faster.

Quoted 6 lines
Quoted 3 lines

We have a set of patches to directly use event channels from within hvm domains, completely bypassing the apic altogether. Do we need this as well?

This is for other HVMs. And the pvhvm still have limitation like it can't use MSI/MSI-X assigned device.

That is not true: upstream Linux kernels can remap MSI/MSI-X into pirqs, if it doesn't work is a bug :)
If you are interested give a look at arch/x86/pci/xen.c:xen_hvm_setup_msi_irqs.

Quoted 57 lines
Quoted 48 lines
Quoted 48 lines

---

 arch/x86/include/asm/xen/hypervisor.h |   33
 +++++++++++++++++++++++++++++++++ arch/x86/kernel/apic/apic.c          
 |    4 +++-
 arch/x86/xen/enlighten.c              |   19 -------------------
 3 files changed, 36 insertions(+), 20 deletions(-)
(quoted diff) +0 -0

b/arch/x86/include/asm/xen/hypervisor.h index 396ff4c..e862874 100644

arch/x86/include/asm/xen/hypervisor.h +28 -0
@@ -37,4 +37,37 @@
 
 extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
+#include <asm/processor.h>
+
+static inline uint32_t xen_cpuid_base(void)
+{
+	uint32_t base, eax, ebx, ecx, edx;
+	char signature[13];
+
+	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+		cpuid(base, &eax, &ebx, &ecx, &edx);
+		*(uint32_t *)(signature + 0) = ebx;
+		*(uint32_t *)(signature + 4) = ecx;
+		*(uint32_t *)(signature + 8) = edx;
+		signature[12] = 0;
+
+		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
+			return base;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_XEN
+static inline bool xen_para_available(void)
+{
+	return 0;
+}
+#else
+static inline bool xen_para_available(void)
Quoted 8 lines

arch/x86/include/asm/xen/hypervisor.h line 71

Quoted 4 lines
arch/x86/include/asm/xen/hypervisor.h +4 -0
+{
+	return (xen_cpuid_base() != 0);
+}
+#endif

So this returns true if you're running a kernel without CONFIG_XEN under Xen? Does that assume that all versions of Xen implement x2apic emulation? Why wouldn't we also want this for CONFIG_XEN kernels?

Quoted 3 lines

Because only the ones that implement the feature would expose x2apic CPUID.

For CONFIG_XEN(pv or pvhvm), they both use evtchn, so no need for x2apic.

In that case you need to check for CONFIG_XEN_PVHVM and the presence of xen_feature(XENFEAT_hvm_pirqs) because only in this case a PV on HVM guests are able to remap both GSIs and MSIs into evtchns.
So I would do something like this:

#ifdef CONFIG_XEN_PVHVM
static inline bool xen_para_available(void)

{
    if (xen_cpuid_base() != 0 && xen_feature(XENFEAT_hvm_pirqs))
    	return 0;
    else
        return 1;
}
#else
static inline bool xen_para_available(void)
{
	return (xen_cpuid_base() != 0);
}

#endif

This is assuming that enabling x2apic doesn't prevent Linux from receiving evtchns either using the callback vector mechanism or the legacy platform-pci interrupt.
Finally when running as dom0 would this feature create problems in the presence of a real x2apic?

author
#4 lore raw
Quoted 12 lines · On Thursday 02 December 2010 21:54:55 Stefano Stabellini wrote:
Quoted 11 lines · Sheng Yang
Quoted 9 lines · On Thursday 02 December 2010 14:28:16 Jeremy Fitzhardinge wrote:
Quoted 4 lines · Sheng Yang

This patch is similiar to Gleb Natapov's patch for KVM, which enable the hypervisor to emulate x2apic feature for the guest. By this way, the emulation of lapic would be simpler with x2apic interface(MSR), and faster.

We have a set of patches to directly use event channels from within hvm domains, completely bypassing the apic altogether. Do we need this as well?

Quoted 7 lines
Quoted 2 lines

This is for other HVMs. And the pvhvm still have limitation like it can't use MSI/MSI-X assigned device.

That is not true: upstream Linux kernels can remap MSI/MSI-X into pirqs, if it doesn't work is a bug :)
If you are interested give a look at arch/x86/pci/xen.c:xen_hvm_setup_msi_irqs.

That's great!

Quoted 69 lines
Quoted 58 lines
Quoted 49 lines
Quoted 49 lines

---

 arch/x86/include/asm/xen/hypervisor.h |   33

+++++++++++++++++++++++++++++++++ arch/x86/kernel/apic/apic.c

| 4 +++-

 arch/x86/xen/enlighten.c              |   19 -------------------
 3 files changed, 36 insertions(+), 20 deletions(-)
(quoted diff) +0 -0

b/arch/x86/include/asm/xen/hypervisor.h index 396ff4c..e862874 100644

arch/x86/include/asm/xen/hypervisor.h +28 -0
@@ -37,4 +37,37 @@
 
 extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
+#include <asm/processor.h>
+
+static inline uint32_t xen_cpuid_base(void)
+{
+	uint32_t base, eax, ebx, ecx, edx;
+	char signature[13];
+
+	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
+		cpuid(base, &eax, &ebx, &ecx, &edx);
+		*(uint32_t *)(signature + 0) = ebx;
+		*(uint32_t *)(signature + 4) = ecx;
+		*(uint32_t *)(signature + 8) = edx;
+		signature[12] = 0;
+
+		if (!strcmp("XenVMMXenVMM", signature) && ((eax - base) >= 2))
+			return base;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_XEN
+static inline bool xen_para_available(void)
+{
+	return 0;
+}
+#else
+static inline bool xen_para_available(void)
Quoted 8 lines

arch/x86/include/asm/xen/hypervisor.h line 71

Quoted 4 lines
arch/x86/include/asm/xen/hypervisor.h +4 -0
+{
+	return (xen_cpuid_base() != 0);
+}
+#endif

So this returns true if you're running a kernel without CONFIG_XEN under Xen? Does that assume that all versions of Xen implement x2apic emulation? Why wouldn't we also want this for CONFIG_XEN kernels?

Quoted 4 lines

Because only the ones that implement the feature would expose x2apic CPUID.

For CONFIG_XEN(pv or pvhvm), they both use evtchn, so no need for x2apic.

In that case you need to check for CONFIG_XEN_PVHVM and the presence of xen_feature(XENFEAT_hvm_pirqs) because only in this case a PV on HVM guests are able to remap both GSIs and MSIs into evtchns.
So I would do something like this:

Quoted 7 lines

#ifdef CONFIG_XEN_PVHVM
static inline bool xen_para_available(void)

{
    if (xen_cpuid_base() != 0 && xen_feature(XENFEAT_hvm_pirqs))
    	return 0;
    else
        return 1;

I suppose only HVM guest without XENFEAT_hvm_pirqs may need this. But does this code covered PV guest as well? We don't need cover them.

Quoted 6 lines

}
#else
static inline bool xen_para_available(void)

{
	return (xen_cpuid_base() != 0);
}
Quoted 6 lines

#endif

This is assuming that enabling x2apic doesn't prevent Linux from receiving evtchns either using the callback vector mechanism or the legacy platform-pci interrupt.

I suppose only legacy platform-pci would need this, because it would use lapic. Callback vector method would use evtchn so this won't be enabled.

Quoted 2 lines

Finally when running as dom0 would this feature create problems in the presence of a real x2apic?

I don't think this can be enabled on dom0.

This one target on HVM domain, maybe also PVHVM domain without XENFEAT_hvm_pirqs, but not the domains using evtchn.

--
regards
Yang, Sheng

#5 lore raw
Quoted 3 lines · Sheng Yang

I suppose only HVM guest without XENFEAT_hvm_pirqs may need this. But does this code covered PV guest as well? We don't need cover them.

We can add a check on xen_pv_domain() to rule PV guests out.

Quoted 9 lines
Quoted 9 lines

}
#else
static inline bool xen_para_available(void)

{
	return (xen_cpuid_base() != 0);
}

#endif

Quoted 7 lines
Quoted 3 lines

This is assuming that enabling x2apic doesn't prevent Linux from receiving evtchns either using the callback vector mechanism or the legacy platform-pci interrupt.

I suppose only legacy platform-pci would need this, because it would use lapic. Callback vector method would use evtchn so this won't be enabled.

Right, should add a check on xen_have_vector_callback too.

Quoted 8 lines
Quoted 2 lines

Finally when running as dom0 would this feature create problems in the presence of a real x2apic?

I don't think this can be enabled on dom0.

This one target on HVM domain, maybe also PVHVM domain without XENFEAT_hvm_pirqs, but not the domains using evtchn.

Ok, a check on xen_pv_domain() will disable x2apic for both pv guests and dom0:

#ifdef CONFIG_XEN_PVHVM
static inline bool xen_para_available(void)

{
    if (xen_pv_domain())
        return 0;
    if (xen_cpuid_base() != 0 &&
            xen_feature(XENFEAT_hvm_pirqs) &&
            xen_have_vector_callback)
        return 0;
    else
        return 1;
}