Hybrid extension support in Xen
PATCH v2 1/1 Hybrid extension support in Xen
Hi Keir
Here is the second version of Hybrid extension support in Xen. Mostly the patch did:
1. Enable SMP support through VCPU_OP in arch_set_info_guest().
2. A new hypercall in hvm_ops to enable hybrid.
3. Mapping IRQ to VIRQ when deliver to the guest.
4. Inject a guest defined vector to deliver notification for events.
5. Use CPUID leaf 0x40000002 to support hybrid feature
6. Reserve some space at the end of MMIO hole for grant table use.
Please review. Thanks!
Signature · 19 lines
regards Yang, Sheng -- tools/firmware/hvmloader/config.h | 6 ++- tools/firmware/hvmloader/hvmloader.c | 6 +++ xen/arch/x86/domain.c | 9 ++++ xen/arch/x86/hvm/hvm.c | 62 +++++++++++++++++++++++++++++--- xen/arch/x86/hvm/irq.c | 66 ++++++++++++++++++++++++++++------- xen/arch/x86/hvm/vmx/intr.c | 3 + xen/arch/x86/traps.c | 9 ++++ xen/include/asm-x86/hvm/hvm.h | 4 +- xen/include/asm-x86/hvm/irq.h | 4 +- xen/include/public/arch-x86/cpuid.h | 7 +++ xen/include/public/hvm/hvm_op.h | 6 +++ xen/include/public/xen.h | 7 +++ xen/include/xen/sched.h | 9 ++++ 13 files changed, 175 insertions(+), 23 deletions(-)
tools/firmware/hvmloader/config.h +5 -1
@@ -16,8 +16,12 @@/* MMIO hole: Hardcoded defaults, which can be dynamically expanded. */ #define PCI_MEM_START 0xf0000000 -#define PCI_MEM_END 0xfc000000 +#define PCI_MEM_END 0xfbfe0000extern unsigned long pci_mem_start, pci_mem_end; + +/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000/* We reserve 16MB for special BIOS mappings, etc. */ #define RESERVED_MEMBASE 0xfc000000
tools/firmware/hvmloader/hvmloader.c +6 -0
@@ -618,6 +618,12 @@ static void build_e820_table(void) e820[nr].type =E820_RAM ; nr++; +/* Reserved for grant table */ + e820[nr].addr =GNTTAB_MEMBASE ; + e820[nr].size =GNTTAB_MEMSIZE ; + e820[nr].type =E820_RESERVED ; + nr++; +/* * Explicitly reserve space for special pages. * This space starts at RESERVED_MEMBASE an extends to cover various
xen/arch/x86/domain.c +9 -0
@@ -682,7 +682,16 @@ int arch_set_info_guest(if (is_hvm_vcpu (v) ) { +unsigned long eip, cs; +hvm_set_info_guest (v); + + eip =c (user_regs.eip); +if (eip !=0 ) { + cs = eip >>12 <<8 ; +hvm_vcpu_reset_state (v, cs,0 ); + hvm_funcs.set_tsc_offset (v,0 ); + } goto out; }
xen/arch/x86/hvm/hvm.c +56 -6
86 lines not on this page · raw message
xen/arch/x86/hvm/irq.c +54 -12
147 lines not on this page · raw message
xen/arch/x86/hvm/vmx/intr.c +2 -1
@@ -164,7 +164,8 @@ asmlinkage void vmx_intr_assist(void) {HVMTRACE_2D (INJ_VIRQ , intack.vector,/*fake=*/ 0 );vmx_inject_extint (intack.vector); -pt_intr_post (v, intack); +if (intack.source != hvm_intsrc_vector) +pt_intr_post (v, intack); }/* Is there another IRQ to queue up behind this one? */
xen/arch/x86/traps.c +9 -0
@@ -686,6 +686,7 @@ int cpuid_hypervisor_leaves( uint32_t idstruct domain *d = current->domain;/* Optionally shift out of the way of Viridian architectural leaves. */ uint32_t base =is_viridian_domain (d) ?0x40000100 :0x40000000 ; +unsigned int tmp_eax, tmp_ebx, tmp_ecx, tmp_edx; idx -= base;if ( idx >3 )
@@ -716,6 +717,14 @@ int cpuid_hypervisor_leaves( uint32_t id *edx =0 ;/* Features 2 */ if ( !is_hvm_vcpu (current) ) *ecx |=XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD ; + +/* Check if additional feature specified, e.g. Hybrid */ +if ( !is_viridian_domain (d) ) { +domain_cpuid (d,0x40000002 ,0 , + &tmp_eax, &tmp_ebx, &tmp_ecx, &tmp_edx); +if (tmp_edx !=0 ) + *edx = tmp_edx &XEN_CPUID_FEAT2_MASK ; + }break ;case 3 :
xen/include/asm-x86/hvm/hvm.h +3 -1
@@ -33,7 +33,8 @@ enum hvm_intsrc { hvm_intsrc_pic, hvm_intsrc_lapic, hvm_intsrc_nmi, - hvm_intsrc_mce + hvm_intsrc_mce, + hvm_intsrc_vector, };struct hvm_intack {uint8_t source;/* enum hvm_intsrc */
@@ -44,6 +45,7 @@ struct hvm_intack {#define hvm_intack_lapic (vec) ( (struct hvm_intack) { hvm_intsrc_lapic, vec } )#define hvm_intack_nmi ( (struct hvm_intack) { hvm_intsrc_nmi, 2 } )#define hvm_intack_mce ( (struct hvm_intack) { hvm_intsrc_mce, 18 } ) +#define hvm_intack_vector (vec)( (struct hvm_intack) { hvm_intsrc_vector, vec } )enum hvm_intblk { hvm_intblk_none,/* not blocked (deliverable) */ hvm_intblk_shadow,/* MOV-SS or STI shadow */
xen/include/asm-x86/hvm/irq.h +3 -1
@@ -54,12 +54,14 @@ struct hvm_irq {enum { HVMIRQ_callback_none, HVMIRQ_callback_gsi, - HVMIRQ_callback_pci_intx + HVMIRQ_callback_pci_intx, + HVMIRQ_callback_vector, } callback_via_type; };union {uint32_t gsi;struct {uint8_t dev, intx; } pci; +uint32_t vector; } callback_via;/* Number of INTx wires asserting each PCI-ISA link. */
xen/include/public/arch-x86/cpuid.h +7 -0
@@ -65,4 +65,11 @@#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) +/* Mask unsupported CPUID specified by user */ +#define XEN_CPUID_FEAT2_MASK 0x3ul +#define _XEN_CPUID_FEAT2_HYBRID 0 +#define XEN_CPUID_FEAT2_HYBRID (1u<<0) +#define _XEN_CPUID_FEAT2_HYBRID_EVTCHN 1 +#define XEN_CPUID_FEAT2_HYBRID_EVTCHN (1u<<1) +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
xen/include/public/hvm/hvm_op.h +6 -0
@@ -127,6 +127,12 @@ typedef struct xen_hvm_set_mem_type xen_typedef struct xen_hvm_set_mem_type xen_hvm_set_mem_type_t ;DEFINE_XEN_GUEST_HANDLE (xen_hvm_set_mem_type_t); +#define HVMOP_enable_hybrid 9 +struct xen_hvm_hybrid_type { +domid_t domid; +uint32_t flags; +#define HVM_HYBRID_EVTCHN (1ull<<1) +};#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
xen/include/public/xen.h +6 -1
@@ -158,7 +158,12 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_END 39 +#define VIRQ_EMUL_PIN_NUM 16 +#define VIRQ_EMUL_PIN (x) (VIRQ_EMUL_PIN_START + x) + +#define NR_VIRQS 40/* * HYPERVISOR_mmu_update(reqs, count, pdone, foreigndom)
xen/include/xen/sched.h +9 -0
@@ -309,6 +309,10 @@ struct domain/* Memory paging support */ struct mem_event_domain mem_event; + +#define XEN_HYBRID_ENABLED (1u << 0) +#define XEN_HYBRID_EVTCHN_ENABLED (1u << 1) +uint64_t hybrid_enabled; };struct domain_setup_info
@@ -595,6 +599,11 @@ uint64_t get_cpu_idle_time(unsigned int#define is_hvm_vcpu (v) (is_hvm_domain(v->domain))#define need_iommu (d) ((d)->need_iommu) +#define is_hybrid_domain (d) ((d)->hybrid_enabled & XEN_HYBRID_ENABLED) +#define is_hybrid_vcpu (v) (is_hybrid_domain(v->domain)) +#define is_hybrid_evtchn_enabled_domain (d) (is_hybrid_domain(d) && \ + (d)->hybrid_enabled & XEN_HYBRID_EVTCHN_ENABLED) +void set_vcpu_migration_delay (unsigned int delay);unsigned int get_vcpu_migration_delay (void );
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
tools/firmware/hvmloader/config.h line 24
Quoted 4 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000
Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Ian.
xen/arch/x86/hvm/hvm.c static hvm_hypercall_t *hvm_hypercall32_ · line 2337
Quoted 13 lines · Sheng Yang
+static
hvm_hypercall_t*hvm_hypercall_hybrid64_table[NR_hypercalls] = {(quoted diff) +2
-0+ [ __HYPERVISOR_memory_op ] = (hvm_hypercall_t *)hvm_memory_op, + [ __HYPERVISOR_grant_table_op ] = (hvm_hypercall_t*)hvm_grant_table_op,
xen/arch/x86/hvm/hvm.c +7
-0+HYPERCALL (xen_version), +HYPERCALL (console_io), +HYPERCALL (vcpu_op), +HYPERCALL (sched_op), +HYPERCALL (event_channel_op), + HYPERCALL(hvm_op), +};
Why not just expand the exiting hvm hypercall table to incorporate these new hypercalls?
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.
Have you verified that all of the operations you are making available are safe to be called from (hybrid)hvm mode?
Ian.
Quoted 10 lines · Sheng Yang
1. Enable SMP support through
VCPU_OPinarch_set_info_guest().
2. A new hypercall inhvm_opsto enable hybrid.
3. Mapping IRQ to VIRQ when deliver to the guest.
4. Inject a guest defined vector to deliver notification for events.
5. Use CPUID leaf0x40000002to support hybrid feature
6. Reserve some space at the end of MMIO hole for grant table use.Please review. Thanks!
Why a complete new hypercall table; could you not just use the existing HVM table and add the extra hypercalls in?
Doing it this way would negate the need for the concept of a 'hybrid' HVM guest would it not? HVM guests that were aware of the hybrid extensions would use them, those that were unaware would not.
Also, why the extra E820 space for the grant table? You don't need the table that early so could you not just use fake PCI BAR space or somesuch and do the XENMEM_add_to_physmap hypercall to add XENMAPSPACE_grant_table later on?
Paul
Signature · 10 lines
=============================== Paul Durrant, Software Engineer Citrix Systems (R&D) Ltd. First Floor, Building 101 Cambridge Science Park Milton Road Cambridge CB4 0FY United Kingdom ===============================
Quoted 10 lines · On Tuesday 02 February 2010 19:22:21 Ian Campbell wrote:
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Thanks for so quick and detail comments. :)
And this one is purposed, because we don't want to depends on QEmu. As you see, we now have PV drivers, QEmu is an alternative way to provide device model now. We think that still involving QEmu as a requirement is somehow strange. So this reserved region is there, to drop the dependence with QEmu to provide PV driver(PV driver depends on QEmu is still strange, right?).
Signature · 2 lines
regards Yang, Sheng
Quoted 16 lines · On Tuesday 02 February 2010 19:26:55 Ian Campbell wrote:
xen/arch/x86/hvm/hvm.c static hvm_hypercall_t *hvm_hypercall32_ · line 2337
Quoted 12 lines · Sheng Yang
+static
hvm_hypercall_t*hvm_hypercall_hybrid64_table[NR_hypercalls] = {(quoted diff) +2
-0+ [ __HYPERVISOR_memory_op ] = (hvm_hypercall_t *)hvm_memory_op, + [ __HYPERVISOR_grant_table_op ] = (hvm_hypercall_t*)hvm_grant_table_op,
xen/arch/x86/hvm/hvm.c +7
-0+HYPERCALL (xen_version), +HYPERCALL (console_io), +HYPERCALL (vcpu_op), +HYPERCALL (sched_op), +HYPERCALL (event_channel_op), + HYPERCALL(hvm_op), +};Why not just expand the exiting hvm hypercall table to incorporate these new hypercalls?
I am just afraid the normal HVM guest called these hypercalls would result in some chaos, so add a limitation(for hybrid only) here. (I admit it didn't much improve the security for a malicious guest...)
Quoted 5 lines
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.
There is still something different from normal HVM guest. For example, to use PV timer, we should clean the tsc offset in HVM domain; and for event delivery, we would use predefined VIRQ rather than emulated IOAPIC/APIC. These code are exclusive, we need them wrapped with flag(which we called "hybrid"). The word "mode" here may be is inaccuracy, a "extension" should be more proper. I would change the phrase next time.
Quoted 3 lines
Have you verified that all of the operations you are making available are safe to be called from (hybrid)hvm mode?
You mean malicious guest attack? Sorry, I haven't try it yet...
Signature · 2 lines
regards Yang, Sheng
Quoted 11 lines · Sheng Yang
Quoted 10 lines · On Tuesday 02 February 2010 19:22:21 Ian Campbell wrote:
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Quoted 7 lines
Thanks for so quick and detail comments. :)
And this one is purposed, because we don't want to depends on QEmu. As you see, we now have PV drivers, QEmu is an alternative way to provide device model now. We think that still involving QEmu as a requirement is somehow strange. So this reserved region is there, to drop the dependence with QEmu to provide PV
driver(PV driver depends on QEmu is still strange, right?).
So with your patchset you can run an HVM guest with no qemu process at all? What about the other emulated devices which have no PV equivalent? How does the VM boot, does the BIOS have pv INT 13 handler?
Ian.
Quoted 5 lines · Sheng Yang
Here is the second version of Hybrid extension support in Xen. Mostly the patch did:
1. Enable SMP support through
VCPU_OPinarch_set_info_guest().
What does that mean and why would we want it?
-- Keir
Quoted 21 lines · On Tuesday 02 February 2010 21:19:32 Ian Campbell wrote:
Quoted 11 lines · Sheng Yang
Quoted 9 lines · On Tuesday 02 February 2010 19:22:21 Ian Campbell wrote:
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Quoted 8 lines
Thanks for so quick and detail comments. :)
And this one is purposed, because we don't want to depends on QEmu. As you see, we now have PV drivers, QEmu is an alternative way to provide device model now. We think that still involving QEmu as a requirement is somehow strange. So this reserved region is there, to drop the dependence with QEmu to provide PV
driver(PV driver depends on QEmu is still strange, right?).
Quoted 3 lines
So with your patchset you can run an HVM guest with no qemu process at all? What about the other emulated devices which have no PV equivalent? How does the VM boot, does the BIOS have pv INT 13 handler?
No, not currently... Sorry for confusion.
We just think QEmu provided the availability of PV driver seems unelegant, so we want to decouple them. Because what QEmu provided is a PCI IRQ and a MMIO region for grant table. And event channel is available without that PCI IRQ, so we think decouple MMIO from QEmu should be more elegant...
Signature · 2 lines
regards Yang, Sheng
Quoted 10 lines · Ian Campbell
tools/firmware/hvmloader/config.h line 24
Quoted 4 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
I didn't even find where these get used, except to reserve an area in e820, and it wasn't clear why that reservation is necessary.
-- Keir
Quoted 7 lines · On Tuesday 02 February 2010 21:23:11 Keir Fraser wrote:
Quoted 4 lines · Sheng Yang
Here is the second version of Hybrid extension support in Xen. Mostly the patch did:
1. Enable SMP support through
VCPU_OPinarch_set_info_guest().What does that mean and why would we want it?
It's not a good description, sorry... We need a entry address and boot up code AP. For that purpose, I reuse VCPUOP_initialise and VCPUOP_up hypercall. The former guest set the entry address of AP, the later one boot up them. For this, I add some code in arch_set_info_guest() to initialize the AP(mostly what we did after INIT-SIPI-SIPI sequence), that's what I means here...
Signature · 2 lines
regards Yang, Sheng
Quoted 26 lines · Sheng Yang
Quoted 21 lines · On Tuesday 02 February 2010 21:19:32 Ian Campbell wrote:
Quoted 11 lines · Sheng Yang
Quoted 9 lines · On Tuesday 02 February 2010 19:22:21 Ian Campbell wrote:
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Quoted 8 lines
Thanks for so quick and detail comments. :)
And this one is purposed, because we don't want to depends on QEmu. As you see, we now have PV drivers, QEmu is an alternative way to provide device model now. We think that still involving QEmu as a requirement is somehow strange. So this reserved region is there, to drop the dependence with QEmu to provide PV
driver(PV driver depends on QEmu is still strange, right?).Quoted 3 lines
So with your patchset you can run an HVM guest with no qemu process at all? What about the other emulated devices which have no PV equivalent? How does the VM boot, does the BIOS have pv INT 13 handler?
Quoted 6 lines
No, not currently... Sorry for confusion.
We just think QEmu provided the availability of PV driver seems unelegant, so we want to decouple them. Because what QEmu provided is a PCI IRQ and a MMIO region for grant table. And event channel is available without that PCI IRQ, so we think decouple MMIO from QEmu should be more elegant...
OK, but in that case I think we should have a mechanism for the guest to query the location of the grant table pages (hypercall, MSR etc) rather than hardcoding a magic address. It may well end up being hardcoded on the tools/hypervisor side for now but there is no reason to expose that to the guest.
I wonder if we could turn things around and have the guest pick some pages and tell the hypervisor to put the grant table there, removing the need to reserve any of the physical address space up front. How do full-PV guests find their grant table, is that a mechanism which could be re-used here instead of reserving magic regions?
Ian.
Quoted 11 lines · On Tuesday 02 February 2010 21:35:23 Keir Fraser wrote:
Quoted 9 lines · Ian Campbell
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Quoted 3 lines
I didn't even find where these get used, except to reserve an area in e820, and it wasn't clear why that reservation is necessary.
It has been used in the last [6/6] patch of Linux kernel side, which would use the pages to map grant table. It works the same as the MMIO region in PVonHVM device. Reserve it in BIOS because we think it's more elegant than depends on QEmu to provide the reserved memory space.
Signature · 2 lines
regards Yang, Sheng
Quoted 18 lines · Sheng Yang
Quoted 16 lines · On Tuesday 02 February 2010 19:26:55 Ian Campbell wrote:
xen/arch/x86/hvm/hvm.c static hvm_hypercall_t *hvm_hypercall32_ · line 2337
Quoted 12 lines · Sheng Yang
+static
hvm_hypercall_t*hvm_hypercall_hybrid64_table[NR_hypercalls] = {(quoted diff) +2
-0+ [ __HYPERVISOR_memory_op ] = (hvm_hypercall_t *)hvm_memory_op, + [ __HYPERVISOR_grant_table_op ] = (hvm_hypercall_t*)hvm_grant_table_op,
xen/arch/x86/hvm/hvm.c +7
-0+HYPERCALL (xen_version), +HYPERCALL (console_io), +HYPERCALL (vcpu_op), +HYPERCALL (sched_op), +HYPERCALL (event_channel_op), + HYPERCALL(hvm_op), +};Why not just expand the exiting hvm hypercall table to incorporate these new hypercalls?
Quoted 3 lines
I am just afraid the normal HVM guest called these hypercalls would result in some chaos, so add a limitation(for hybrid only) here. (I admit it didn't much improve the security for a malicious guest...)
I don't think this limitation adds any meaningful security or reduction in chaos. A non-hybrid aware HVM guest simply won't make those hypercalls.
Quoted 11 lines
Quoted 4 lines
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.
There is still something different from normal HVM guest. For example, to use PV timer, we should clean the tsc offset in HVM domain; and for event delivery, we would use predefined VIRQ rather than emulated IOAPIC/APIC. These code are exclusive, we need them wrapped with
flag(which we called "hybrid"). The word "mode" here may be is inaccuracy, a "extension" should be more proper. I would change the phrase next time.
But the old mechanisms (emulated IOAPIC etc) are still present until the enable_hybrid HVMOP is called, aren't they? Why can't you perform the switch at the point at which the new feature is requested by the guest e.g. when the VIRQ is configured?
It looks like you are using evtchn's for all interrupt injection, including any emulated or passthrough devices which may be present. Using evtchn's for PV devices obviously makes sense but I think this needs to coexist with emulated interrupt injection for non-PV devices so the IOAPIC/APIC should not be mutually exclusive with using PV evtchns.
Ian.
Quoted 13 lines · Sheng Yang
xen/arch/x86/hvm/irq.c +4
-2@@ -46,8 +46,18 @@ static void __hvm_pci_intx_assert(if ( (hvm_irq->pci_link_assert_count[link]++ ==0 ) && isa_irq && (hvm_irq->gsi_assert_count[isa_irq]++ ==0 ) ) { -vioapic_irq_positive_edge (d, isa_irq); -vpic_irq_positive_edge (d, isa_irq); +if ( !is_hybrid_evtchn_enabled_domain (d) ) + { +vioapic_irq_positive_edge (d, isa_irq); +vpic_irq_positive_edge (d, isa_irq);
xen/arch/x86/hvm/irq.c static void __hvm_pci_intx_assert( · line 62
Quoted 10 lines
xen/arch/x86/hvm/irq.c +8
-0+ } +else + { +/* TODO fix the critical region here */ +spin_unlock (&d->arch.hvm_domain.irq_lock); +send_guest_global_virq (d,VIRQ_EMUL_PIN (isa_irq)); +spin_lock (&d->arch.hvm_domain.irq_lock); + } } }
This can't be right, at least not without a big comment explaining why it is safe to drop the lock here...
Ian.
Quoted 19 lines · On Tuesday 02 February 2010 21:52:55 Ian Campbell wrote:
Quoted 18 lines · Sheng Yang
xen/arch/x86/hvm/irq.c +9
-2@@ -46,8 +46,18 @@ static void __hvm_pci_intx_assert(if ( (hvm_irq->pci_link_assert_count[link]++ ==0 ) && isa_irq && (hvm_irq->gsi_assert_count[isa_irq]++ ==0 ) ) { -vioapic_irq_positive_edge (d, isa_irq); -vpic_irq_positive_edge (d, isa_irq); +if ( !is_hybrid_evtchn_enabled_domain (d) ) + { +vioapic_irq_positive_edge (d, isa_irq); +vpic_irq_positive_edge (d, isa_irq); + } +else + { +/* TODO fix the critical region here */ +spin_unlock (&d->arch.hvm_domain.irq_lock);
Quoted 8 lines
xen/arch/x86/hvm/irq.c static void __hvm_pci_intx_assert( · line 62
Quoted 5 lines
xen/arch/x86/hvm/irq.c +3
-0+send_guest_global_virq (d,VIRQ_EMUL_PIN (isa_irq)); +spin_lock (&d->arch.hvm_domain.irq_lock); + } } }This can't be right, at least not without a big comment explaining why it is safe to drop the lock here...
... Would address this in the next version.
Signature · 2 lines
regards Yang, Sheng
Quoted 3 lines · Ian Campbell
I wonder if we could turn things around and have the guest pick some pages and tell the hypervisor to put the grant table there, removing the need to reserve any of the physical address space up front.
Yes; this is how all existing PV-on-HVM drivers work, using the grant-table hypercall. I don't see the need to add another mechanism.
Tim.
Signature · 3 lines
Tim Deegan <[email protected]> Principal Software Engineer, Citrix Systems (R&D) Ltd. [Company #02300071, SL9 0DZ, UK.]
Quoted 7 lines · Sheng Yang
Quoted 3 lines
I didn't even find where these get used, except to reserve an area in e820, and it wasn't clear why that reservation is necessary.
It has been used in the last [6/6] patch of Linux kernel side, which would use the pages to map grant table. It works the same as the MMIO region in PVonHVM device. Reserve it in BIOS because we think it's more elegant than depends on QEmu to provide the reserved memory space.
Hmm. Can't this be done in phases? It seems unnecessary to be making changes solely to remove qemu dependencies in the intial patchset, when the patchset does not actually achieve that aim.
I think an HVM guest with no PCI space is a little way off, and perhaps we can find a better way than hardcoding an address in two places.
-- Keir
Quoted 9 lines · Sheng Yang
Quoted 3 lines
Quoted 1 line
1. Enable SMP support through
VCPU_OPinarch_set_info_guest().What does that mean and why would we want it?
It's not a good description, sorry... We need a entry address and boot up code AP. For that purpose, I reuse
VCPUOP_initialiseandVCPUOP_uphypercall. The former guest set the entry address of AP, the later one boot up them. For this, I add some code inarch_set_info_guest()to initialize the AP(mostly what we did after INIT-SIPI-SIPI sequence), that's what I means here...
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
-- Keir
Quoted 6 lines · Ian Campbell
It looks like you are using evtchn's for all interrupt injection, including any emulated or passthrough devices which may be present. Using evtchn's for PV devices obviously makes sense but I think this needs to coexist with emulated interrupt injection for non-PV devices so the IOAPIC/APIC should not be mutually exclusive with using PV evtchns.
Indeed.
Evtchn's for PV devices are more than welcome, but this:
Quoted 14 lines
xen/arch/x86/hvm/irq.c +5
-2@@ -46,8 +46,18 @@ static void __hvm_pci_intx_assert(if ( (hvm_irq->pci_link_assert_count[link]++ ==0 ) && isa_irq && (hvm_irq->gsi_assert_count[isa_irq]++ ==0 ) ) { -vioapic_irq_positive_edge (d, isa_irq); -vpic_irq_positive_edge (d, isa_irq); +if ( !is_hybrid_evtchn_enabled_domain (d) ) + { +vioapic_irq_positive_edge (d, isa_irq); +vpic_irq_positive_edge (d, isa_irq); + }
xen/arch/x86/hvm/irq.c static void __hvm_pci_intx_assert( · line 63
Quoted 10 lines
xen/arch/x86/hvm/irq.c +7
-0+else + { +/* TODO fix the critical region here */ +spin_unlock (&d->arch.hvm_domain.irq_lock); +send_guest_global_virq (d,VIRQ_EMUL_PIN (isa_irq)); +spin_lock (&d->arch.hvm_domain.irq_lock); + } } }
and this:
Quoted 7 lines
xen/arch/x86/hvm/irq.c +1
-2@@ -101,11 +114,21 @@ void hvm_isa_irq_assert(if ( !__test_and_set_bit (isa_irq, &hvm_irq->isa_irq.i) && (hvm_irq->gsi_assert_count[gsi]++ ==0 ) ) { -vioapic_irq_positive_edge (d, gsi); -vpic_irq_positive_edge (d, isa_irq); +if ( !is_hybrid_evtchn_enabled_domain (d) )
xen/arch/x86/hvm/irq.c void hvm_isa_irq_assert( · line 127
Quoted 10 lines
xen/arch/x86/hvm/irq.c +8
-0+ { +vioapic_irq_positive_edge (d, gsi); +vpic_irq_positive_edge (d, isa_irq); + } + else + { + send_virq =1 ; + } }
and this:
xen/arch/x86/hvm/irq.c void hvm_isa_irq_deassert( · line 152
Quoted 13 lines
xen/arch/x86/hvm/irq.c +4
-1@@ -120,7 +143,10 @@ void hvm_isa_irq_deassert(if (__test_and_clear_bit (isa_irq, &hvm_irq->isa_irq.i) && (--hvm_irq->gsi_assert_count[gsi] ==0 ) ) -vpic_irq_negative_edge (d, isa_irq); + { +if ( !is_hybrid_evtchn_enabled_domain (d) ) +vpic_irq_negative_edge (d, isa_irq); + }spin_unlock (&d->arch.hvm_domain.irq_lock); }
I think they are a very bad idea and a maintenance pain, especially when you think about pci passthrought.
Quoted 19 lines · On Tuesday 02 February 2010 21:52:44 Ian Campbell wrote:
Quoted 18 lines · Sheng Yang
Quoted 16 lines · On Tuesday 02 February 2010 19:26:55 Ian Campbell wrote:
xen/arch/x86/hvm/hvm.c static hvm_hypercall_t *hvm_hypercall32_ · line 2337
Quoted 12 lines · Sheng Yang
+static
hvm_hypercall_t*hvm_hypercall_hybrid64_table[NR_hypercalls] = {(quoted diff) +2
-0+ [ __HYPERVISOR_memory_op ] = (hvm_hypercall_t *)hvm_memory_op, + [ __HYPERVISOR_grant_table_op ] = (hvm_hypercall_t*)hvm_grant_table_op,
xen/arch/x86/hvm/hvm.c +7
-0+HYPERCALL (xen_version), +HYPERCALL (console_io), +HYPERCALL (vcpu_op), +HYPERCALL (sched_op), +HYPERCALL (event_channel_op), + HYPERCALL(hvm_op), +};Why not just expand the exiting hvm hypercall table to incorporate these new hypercalls?
Quoted 7 lines
Quoted 3 lines
I am just afraid the normal HVM guest called these hypercalls would result in some chaos, so add a limitation(for hybrid only) here. (I admit it didn't much improve the security for a malicious guest...)
I don't think this limitation adds any meaningful security or reduction in chaos. A non-hybrid aware HVM guest simply won't make those hypercalls.
Um, yes... Would update this in the next version.
Quoted 13 lines
Quoted 6 lines
Quoted 5 lines
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.
Quoted 6 lines
There is still something different from normal HVM guest. For example, to use PV timer, we should clean the tsc offset in HVM domain; and for event delivery, we would use predefined VIRQ rather than emulated IOAPIC/APIC. These code are exclusive, we need them wrapped with
flag(which we called "hybrid"). The word "mode" here may be is inaccuracy, a "extension" should be more proper. I would change the phrase next time.
Quoted 4 lines
But the old mechanisms (emulated IOAPIC etc) are still present until the
enable_hybridHVMOP is called, aren't they? Why can't you perform the switch at the point at which the new feature is requested by the guest e.g. when the VIRQ is configured?
Yes, they are there before the enable_hybrid HVMOP called. But sorry for I don't quite understand about the point "when the VIRQ is configured?"
Quoted 6 lines
It looks like you are using evtchn's for all interrupt injection, including any emulated or passthrough devices which may be present. Using evtchn's for PV devices obviously makes sense but I think this needs to coexist with emulated interrupt injection for non-PV devices so the IOAPIC/APIC should not be mutually exclusive with using PV evtchns.
Yes. similar to dom0(maybe more like pv_ops dom0). Currently passthrough devices have issue with it, due to lack of MSI/MSI-X support(which would be our next step and can be shared with pv_ops dom0); for INTx method, because event channel is edge triggered, but INTx is level triggered, so they also can't be used. Passthrough devices can be used with some hack, if MSI2INTX translation is worked.
But let IOAPIC/APIC coexisted with event channel is not our target. As we know, the overhead brought by them, notably EOI by LAPIC, impact performance badly. We want event channel that because event channel have much less overhead compared to IOAPIC/LAPIC, a completely virtualization aware solution which elimiate all the unnecessary overhead. That's what we want Xen guest to be benefit.
Signature · 2 lines
regards Yang, Sheng
Quoted 12 lines · On Tuesday 02 February 2010 22:03:31 Keir Fraser wrote:
Quoted 10 lines · Sheng Yang
Quoted 3 lines
Quoted 1 line
1. Enable SMP support through
VCPU_OPinarch_set_info_guest().What does that mean and why would we want it?
It's not a good description, sorry... We need a entry address and boot up code AP. For that purpose, I reuse
VCPUOP_initialiseandVCPUOP_uphypercall. The former guest set the entry address of AP, the later one boot up them. For this, I add some code inarch_set_info_guest()to initialize the AP(mostly what we did after INIT-SIPI-SIPI sequence), that's what I means here...
Quoted 3 lines
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
Because we don't have IOAPIC/LAPIC...
Signature · 2 lines
regards Yang, Sheng
Quoted 9 lines · On Tuesday 02 February 2010 22:01:43 Keir Fraser wrote:
Quoted 7 lines · Sheng Yang
Quoted 2 lines
I didn't even find where these get used, except to reserve an area in e820, and it wasn't clear why that reservation is necessary.
It has been used in the last [6/6] patch of Linux kernel side, which would use the pages to map grant table. It works the same as the MMIO region in PVonHVM device. Reserve it in BIOS because we think it's more elegant than depends on QEmu to provide the reserved memory space.
Quoted 6 lines
Hmm. Can't this be done in phases? It seems unnecessary to be making
changes solely to remove qemu dependencies in the intial patchset, when the patchset does not actually achieve that aim.I think an HVM guest with no PCI space is a little way off, and perhaps we can find a better way than hardcoding an address in two places.
Sure. I would try stick to old QEmu provided MMIO first. (Maybe try Ian Campbell's advice to get some more flexible ones later).
Signature · 2 lines
regards Yang, Sheng
Quoted 7 lines · On Tuesday 02 February 2010 22:00:47 Tim Deegan wrote:
Quoted 3 lines · Ian Campbell
I wonder if we could turn things around and have the guest pick some pages and tell the hypervisor to put the grant table there, removing the need to reserve any of the physical address space up front.
Yes; this is how all existing PV-on-HVM drivers work, using the grant-table hypercall. I don't see the need to add another mechanism.
...Not quite understand. I think the PV-on-HVM drivers reserved the memory space in QEmu provided device's MMIO space, similar to what I did here(except reserved in BIOS)?...
Signature · 2 lines
regards Yang, Sheng
Quoted 35 lines · On Tuesday 02 February 2010 21:50:05 Ian Campbell wrote:
Quoted 26 lines · Sheng Yang
Quoted 21 lines · On Tuesday 02 February 2010 21:19:32 Ian Campbell wrote:
Quoted 11 lines · Sheng Yang
Quoted 9 lines · On Tuesday 02 February 2010 19:22:21 Ian Campbell wrote:
tools/firmware/hvmloader/config.h line 24
Quoted 3 lines · Sheng Yang
tools/firmware/hvmloader/config.h +3
-0+/* Reserve 128KB for grant table */ +#define GNTTAB_MEMBASE 0xfbfe0000 +#define GNTTAB_MEMSIZE 0x20000Why is this necessary? Isn't the grant table contained within one of the BARS on the virtual PCI device? What needs grant tables for prior to the kernel finding the PCI device which necessitates hardcoding these addresses in both guest and hypervisor?
Quoted 8 lines
Thanks for so quick and detail comments. :)
And this one is purposed, because we don't want to depends on QEmu. As you see, we now have PV drivers, QEmu is an alternative way to provide device model now. We think that still involving QEmu as a requirement is somehow strange. So this reserved region is there, to drop the dependence with QEmu to provide PV
driver(PV driver depends on QEmu is still strange, right?).Quoted 3 lines
So with your patchset you can run an HVM guest with no qemu process at all? What about the other emulated devices which have no PV equivalent? How does the VM boot, does the BIOS have pv INT 13 handler?
Quoted 7 lines
No, not currently... Sorry for confusion.
We just think QEmu provided the availability of PV driver seems unelegant, so we want to decouple them. Because what QEmu provided is a PCI IRQ and a MMIO region for grant table. And event channel is available without that PCI IRQ, so we think decouple MMIO from QEmu should be more elegant...
Quoted 5 lines
OK, but in that case I think we should have a mechanism for the guest to query the location of the grant table pages (hypercall, MSR etc) rather than hardcoding a magic address. It may well end up being hardcoded on the
tools/hypervisorside for now but there is no reason to expose that to the guest.
Yes. I would try to use QEmu provided device for this - though I think it may still have problem if I use PCI probe mechanism to find the address, because the initialization of grant table maybe earlier.
Rather than probe as a PCI device, I prefer a query or hardcode in this case.
Quoted 6 lines
I wonder if we could turn things around and have the guest pick some pages and tell the hypervisor to put the grant table there, removing the need to reserve any of the physical address space up front. How do full-PV guests find their grant table, is that a mechanism which could be re-used here instead of reserving magic regions?
I would check if we can follow the PV solution.(I remember I once checked that, but forgot the reason for not doing so now...)
Signature · 2 lines
regards Yang, Sheng
Quoted 4 lines · Sheng Yang
Quoted 3 lines
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
Because we don't have IOAPIC/LAPIC...
Is it necessary to remove the LAPICs completely? If you go very far down the route of ripping emulated stuff out of HVM, it starts to feel like starting with a pure PV guest and HVMing it up is closer in spirit to what you might be aiming for.
-- Keir
Quoted 9 lines · Keir Fraser
Quoted 4 lines
Quoted 3 lines
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
Because we don't have IOAPIC/LAPIC...
Is it necessary to remove the LAPICs completely? If you go very far down the route of ripping emulated stuff out of HVM, it starts to feel like starting with a pure PV guest and HVMing it up is closer in spirit to what you might be aiming for.
The other thing is, removing some of this stuff weakens the argument that the hybrid approach lets us 'ride the wave' of improving hardware virt support. For example, LAPIC is pretty architectural these days, and is ripe for fuller hardware virtualisation. If you put HVM onto event channels and totally rip out the LAPIC, where does that leave us if full LAPIC virt comes along, with all its potential for especially improving device passthru performance?
-- Keir
Quoted 10 lines · On Tuesday 02 February 2010 22:32:09 Keir Fraser wrote:
Quoted 4 lines · Sheng Yang
Quoted 2 lines
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
Because we don't have IOAPIC/LAPIC...
Is it necessary to remove the LAPICs completely? If you go very far down
the route of ripping emulated stuff out of HVM, it starts to feel like starting with a pure PV guest and HVMing it up is closer in spirit to what you might be aiming for.
The performance of interrupt delivery is what we care about. The IOAPIC/LAPIC don't work well now, so we want event channel to eliminate their overhead, notably EOI in LAPIC.
And you know, start from a PV guest is another story...
Signature · 2 lines
regards Yang, Sheng
Quoted 11 lines · On Tuesday 02 February 2010 22:37:06 Keir Fraser wrote:
Quoted 9 lines · Keir Fraser
Quoted 4 lines
Quoted 2 lines
Okay, so that leads to the obvious next question: why do you want to avoid using INIT-SIPI-SIPI?
Because we don't have IOAPIC/LAPIC...
Is it necessary to remove the LAPICs completely? If you go very far down the route of ripping emulated stuff out of HVM, it starts to feel like starting with a pure PV guest and HVMing it up is closer in spirit to what you might be aiming for.
Quoted 7 lines
The other thing is, removing some of this stuff weakens the argument that the hybrid approach lets us 'ride the wave' of improving hardware virt support. For example, LAPIC is pretty architectural these days, and is ripe for fuller hardware virtualisation. If you put HVM onto event channels and totally rip out the LAPIC, where does that leave us if full LAPIC virt
comes along, with all its potential for especially improving device passthru performance?
In fact, our target is improving device passthru performance(before full LAPIC virtualziation ready in every server). And what we mostly targeted is MSI/MSI- X interrupt intensive devices. This didn't include in this patchset(we have experiment patches of course), because that can't avoid touching the some generic kernel code, e.g. MSI related things in ioapic.c, make it harder to be checked in. The code to support hybrid MSI/MSI-X device can be shared with pv_ops dom0, so that is what we planed next step.
And these hybrid features are component based. I admit, sooner or later, these PV solutions would be replaced with hardware features. If you got new enough hardwares, you can simply disable them; if you didn't, you can still benefit from them. Of course, as you know, it takes time to get new enough machine for everyone. So I think, it's better to do something now rather than simply waiting for hardware feature come. :)
Signature · 2 lines
regards Yang, Sheng
Quoted 16 lines · Sheng Yang
Quoted 15 lines · On Tuesday 02 February 2010 21:52:44 Ian Campbell wrote:
Quoted 13 lines · Sheng Yang
Quoted 5 lines · On Tuesday 02 February 2010 19:26:55 Ian Campbell wrote:
On Tue, 2010-02-02 at 08:16 +0000, Sheng Yang wrote:
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.There is still something different from normal HVM guest. For example, to use PV timer, we should clean the tsc offset in HVM domain; and for event delivery, we would use predefined VIRQ rather than emulated IOAPIC/APIC. These code are exclusive, we need them wrapped with
flag(which we called "hybrid"). The word "mode" here may be is inaccuracy, a "extension" should be more proper. I would change the phrase next time.
Quoted 7 lines
Quoted 4 lines
But the old mechanisms (emulated IOAPIC etc) are still present until the
enable_hybridHVMOP is called, aren't they? Why can't you perform the switch at the point at which the new feature is requested by the guest e.g. when the VIRQ is configured?Yes, they are there before the
enable_hybridHVMOP called. But sorry for I don't quite understand about the point "when the VIRQ is configured?"
I just meant that you should enable evtchn mode at the point at which the guest tries to use event channels and not in some specific "enable hybrid mode call", similarly for PV-timer mode etc.
Of course that presupposes that event channels and APIC etc are mutually exclusive, which I don't think is a given.
I'm of the opinion that the word "hybrid" should not occur anywhere in the tools or hypervisor -- instead there should simply be PV functionalities which are made available to HVM guest kernels which request them.
I'm similarly of the opinion that the hybrid-ness should not leak out of the core Xen-aware kernel code, for example the word hybrid should not be used in the front end drivers, rather the differences should be encapsulated in the evtchn code (etc).
Quoted 6 lines
But let IOAPIC/APIC coexisted with event channel is not our target. As we know, the overhead brought by them, notably EOI by LAPIC, impact performance badly. We want event channel that because event channel have much less overhead compared to IOAPIC/LAPIC, a completely virtualization aware solution which elimiate all the unnecessary overhead. That's what we want Xen guest to be benefit.
It certainly makes sense to deliver interrupts to PV drivers via evtchns. I don't think it necessarily follows that all interrupts should be injected via event channels. For example for low throughput emulated devices I think it makes sense to continue to inject interrupts via the emulated *APIC paths such that the treatment of a given device is either consistently emulated or consistently paravirtualised but not some mixture.
Ian.
Quoted 26 lines · On Wednesday 03 February 2010 00:15:50 Ian Campbell wrote:
Quoted 17 lines · Sheng Yang
Quoted 16 lines · On Tuesday 02 February 2010 21:52:44 Ian Campbell wrote:
Quoted 14 lines · Sheng Yang
Quoted 5 lines · On Tuesday 02 February 2010 19:26:55 Ian Campbell wrote:
On Tue, 2010-02-02 at 08:16 +0000, Sheng Yang wrote:
In fact, why is hybrid mode considered a separate mode by the hypervisor at all? Shouldn't it just be an extension to regular HVM mode which guests may choose to use? This seems like it would eliminate a bunch of the random conditionals.There is still something different from normal HVM guest. For example, to use PV timer, we should clean the tsc offset in HVM domain; and for event delivery, we would use predefined VIRQ rather than emulated IOAPIC/APIC. These code are exclusive, we need them wrapped with
flag(which we called "hybrid"). The word "mode" here may be is inaccuracy, a "extension" should be more proper. I would change the phrase next time.Quoted 7 lines
Quoted 4 lines
But the old mechanisms (emulated IOAPIC etc) are still present until the
enable_hybridHVMOP is called, aren't they? Why can't you perform the switch at the point at which the new feature is requested by the guest e.g. when the VIRQ is configured?Yes, they are there before the
enable_hybridHVMOP called. But sorry for I don't quite understand about the point "when the VIRQ is configured?"
Quoted 6 lines
I just meant that you should enable evtchn mode at the point at which the guest tries to use event channels and not in some specific "enable hybrid mode call", similarly for PV-timer mode etc.
Of course that presupposes that event channels and APIC etc are mutually exclusive, which I don't think is a given.
I would give it try. Seems I need to bind tsc offset clean in event channel binding hypercall(if I didn't introduce another hypercall), sounds a little messy.
Quoted 9 lines
I'm of the opinion that the word "hybrid" should not occur anywhere in the tools or hypervisor -- instead there should simply be PV functionalities which are made available to HVM guest kernels which request them.
I'm similarly of the opinion that the hybrid-ness should not leak out of the core Xen-aware kernel code, for example the word hybrid should not be used in the front end drivers, rather the differences should be encapsulated in the evtchn code (etc).
That's reasonable. I would try to make them more like natural PV feature.
Quoted 8 lines
Quoted 7 lines
But let IOAPIC/APIC coexisted with event channel is not our target. As we know, the overhead brought by them, notably EOI by LAPIC, impact performance badly. We want event channel that because event channel have much less overhead compared to IOAPIC/LAPIC, a completely virtualization aware solution which elimiate all the unnecessary overhead. That's what we want Xen guest to be benefit.
Quoted 7 lines
It certainly makes sense to deliver interrupts to PV drivers via evtchns. I don't think it necessarily follows that all interrupts should be injected via event channels. For example for low throughput emulated devices I think it makes sense to continue to inject interrupts via the emulated *APIC paths such that the treatment of a given device is either consistently emulated or consistently paravirtualised but not some mixture.
Well, for us, we want evtchn because we want to improve interrupt intensive passthru device's performance(though too big for the first step, we have experiment patches, but would like to consolidate with the solution of pv_ops dom0). The situation won't change if we still use emulated APIC path...
Signature · 2 lines
regards Yang, Sheng
Quoted 7 lines · Sheng Yang
Quoted 2 lines
I just meant that you should enable evtchn mode at the point at
which
Quoted 1 line
the guest tries to use event channels and not in some specific
"enable
Quoted 2 lines
hybrid mode call", similarly for PV-timer mode etc.
Quoted 7 lines
Quoted 1 line
Of course that presupposes that event channels and APIC etc are
mutually
Quoted 1 line
exclusive, which I don't think is a given.
I would give it try. Seems I need to bind tsc offset clean in event channel binding hypercall(if I didn't introduce another hypercall), sounds a little messy.
I guess to clarify what I really meant was in the code which enables the PV timer for a domain, rather than specifically in the event channel hypercall which implements the binding to the timer evtchn. I'm not sure if enabling the timer is a separate hypercall or is just a side effect of binding to the evtchn.
Ian.
Quoted 5 lines · Sheng Yang
Well, for us, we want evtchn because we want to improve interrupt intensive passthru device's performance(though too big for the first step, we have experiment patches, but would like to consolidate with the solution of
pv_opsdom0). The situation won't change if we still use emulated APIC path...
I think you should keep this as the last step: once you have all the other PV features working on HVM like Ian suggested, send a patch to allow a guest kernel to switch from emulated APIC to evtchn for every device.
If you manage to keep it simple, it could be a win for everyone.
Quoted 7 lines · On Wednesday 03 February 2010 02:27:42 Stefano Stabellini wrote:
Quoted 5 lines · Sheng Yang
Well, for us, we want evtchn because we want to improve interrupt intensive passthru device's performance(though too big for the first step, we have experiment patches, but would like to consolidate with the solution of
pv_opsdom0). The situation won't change if we still use emulated APIC path...
Quoted 6 lines
I think you should keep this as the last step: once you have all the other PV features working on HVM like Ian suggested, send a patch to allow a guest kernel to switch from emulated APIC to evtchn for every device.
If you manage to keep it simple, it could be a win for everyone.
Make event channel coexist with IOAPIC/LAPIC is quite different from what we are doing now, and Xen already have PV-on-HVM for that.
Of course, I also want to have a elegant solution in the end, so I would try hard to keep things simple, and keep the code clear.
Signature · 2 lines
regards Yang, Sheng
Quoted 2 lines · Sheng Yang
Make event channel coexist with IOAPIC/LAPIC is quite different from what we are doing now, and Xen already have PV-on-HVM for that.
Indeed it does. I don't see any advantage in adding more mechanism to both Xen and linux to give us something we basically already have.
Cheers,
Tim.
Signature · 3 lines
Tim Deegan <[email protected]> Principal Software Engineer, Citrix Systems (R&D) Ltd. [Company #02300071, SL9 0DZ, UK.]