PV extension of HVM (Hybrid) for Xen
Subsystems (8)
v5 cover PV extension of HVM (Hybrid) for Xen
Hi, Jeremy & Ian
Here is the fifth version of patchset to enable PV extension of HVM support in Linux kernel of Xen.
The PV extension of HVM is started from real mode like HVM guest, but also with a a range of PV features(e.g. PV timer, event channel, as well as PV drivers). So guest with this feature can takes the advantages of both H/W virtualization and Para-Virtualization.
The first two of the patchset imported several header file from Jeremy's tree and Xen tree, respect to Jeremy and Keir's works.
The whole patchset based on Linux upstream.
You need a line like:
cpuid = [ '0x40000002:edx=0x3' ]
in HVM configuration file to expose hybrid feature to guest, and
CONFIG_XEN
in the guest kernel configuration file to enable the hybrid support.
And the compiled image can be used as native/pv domU/hvm guest/pv feature hvm kernel.
Current the patchset support x86_64 only.
Current base is still on Linux 2.6.33.
Change from v4:
1. Add a new CONFIG_XEN_HVM_PV to enable the feature in kernel
2. Separate the related code form enlighted.c to hvmpv.c
3. Separate the feature "PV clocksource" from evtchn. Now we can support HVM guest with PV clocksource. This would be enabled by default.
4. Drop PV halt and pv drivers in this edition. We can work on that later.
5. Update the patchset following Jeremy's comments.
Change from v3:
1. Rebase to Linux 2.6.33 release.
2. change the name to "PV extension of HVM"
3. Some minor coding polishing.
Change from v2:
1. change the name "hybrid" to "PV featured HVM".
2. Unified the PV driver's judgement of xen_domain() to xen_evtchn_enabled().
3. Move the function(evtchn) initialize hypercall near the real enabling place, rather than a unified place before function enabled.
4. Remove the reserved E820 region for grant table. Use QEmu Xen platform device's MMIO instead.
The major change from v1:
1. SMP support.
2. Modify the entrance point to avoid most of genernic kernel modification.
3. Binding PV timer with event channel mechanism.
--
regards
Yang, Sheng
arch/x86/include/asm/xen/cpuid.h | 73 +++++++++++ arch/x86/include/asm/xen/hypercall.h | 6 + arch/x86/include/asm/xen/hypervisor.h | 6 + arch/x86/kernel/setup.c | 4 + arch/x86/xen/Kconfig | 8 ++ arch/x86/xen/Makefile | 1 + arch/x86/xen/enlighten.c | 6 +- arch/x86/xen/hvmpv.c | 219 +++++++++++++++++++++++++++++++++ arch/x86/xen/irq.c | 28 ++++ arch/x86/xen/smp.c | 92 +++++++++++++- arch/x86/xen/time.c | 4 +- arch/x86/xen/xen-ops.h | 14 ++ drivers/block/xen-blkfront.c | 2 +- drivers/input/xen-kbdfront.c | 2 +- drivers/net/xen-netfront.c | 2 +- drivers/video/xen-fbfront.c | 2 +- drivers/xen/events.c | 74 ++++++++++- drivers/xen/grant-table.c | 2 +- drivers/xen/xenbus/xenbus_probe.c | 4 +- include/xen/events.h | 6 + include/xen/hvm.h | 28 ++++ include/xen/interface/hvm/hvm_op.h | 80 ++++++++++++ include/xen/interface/hvm/params.h | 111 +++++++++++++++++ include/xen/interface/xen.h | 6 +- include/xen/xen.h | 11 ++ 25 files changed, 768 insertions(+), 23 deletions(-)
PATCH v5 1/7 xen: add support for hvm_op
From: Jeremy Fitzhardinge <[email protected]>
Add support for hvm_op hypercall.
--- arch/x86/include/asm/xen/hypercall.h | 6 ++ include/xen/hvm.h | 23 +++++++ include/xen/interface/hvm/hvm_op.h | 72 ++++++++++++++++++++++ include/xen/interface/hvm/params.h | 111 ++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 0 deletions(-) create mode 100644 include/xen/hvm.h create mode 100644 include/xen/interface/hvm/hvm_op.h create mode 100644 include/xen/interface/hvm/params.h
arch/x86/include/asm/xen/hypercall.h +6 -0
@@ -417,6 +417,12 @@ HYPERVISOR_nmi_op(unsigned long op, unsigned long arg)return _hypercall2 (int, nmi_op, op, arg); } +static inline unsigned long __must_check +HYPERVISOR_hvm_op (int op,void *arg) +{ + return _hypercall2(unsigned long, hvm_op, op, arg); +} +static inline void MULTI_fpu_taskswitch (struct multicall_entry *mcl,int set) {
include/xen/hvm.h new file +23 -0
@@ -0,0 +1,23 @@ +/* Simple wrappers around HVM functions */ +#ifndef XEN_HVM_H__ +#define XEN_HVM_H__ + +#include <xen/interface/hvm/params.h> + +static inline unsigned long hvm_get_parameter (int idx) +{ +struct xen_hvm_param xhv; +int r; + + xhv.domid =DOMID_SELF ; + xhv.index = idx; + r =HYPERVISOR_hvm_op (HVMOP_get_param, &xhv); +if (r <0 ) { +printk (KERN_ERR "cannot get hvm parameter %d: %d.\n" , + idx, r); +return 0 ; + } +return xhv.value; +} + +#endif /* XEN_HVM_H__ */
include/xen/interface/hvm/hvm_op.h new file +72 -0
72 lines not on this page · raw message
include/xen/interface/hvm/params.h new file +111 -0
111 lines not on this page · raw message
Signature · 1 line
1.5.4.5
PATCH v5 2/7 xen: Import cpuid.h from Xen
In reply to Sheng YangFrom: Keir Fraser <[email protected]>
Which would be used by CPUID detection later
--- arch/x86/include/asm/xen/cpuid.h | 68 ++++++++++++++++++++++++++++++++++++++ 1 files changed, 68 insertions(+), 0 deletions(-) create mode 100644 arch/x86/include/asm/xen/cpuid.h
arch/x86/include/asm/xen/cpuid.h new file +68 -0
@@ -0,0 +1,68 @@ +/****************************************************************************** +* arch/include/asm/xen/cpuid.h +* +* CPUID interface to Xen. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +* Copyright (c) 2007 Citrix Systems, Inc. +* +* Authors: +* Keir Fraser <[email protected]> +*/ + +#ifndef __ASM_X86_XEN_CPUID_H__ +#define __ASM_X86_XEN_CPUID_H__ + +/* Xen identification leaves start at 0x40000000. */ +#define XEN_CPUID_FIRST_LEAF 0x40000000 +#define XEN_CPUID_LEAF (i) (XEN_CPUID_FIRST_LEAF + (i)) + +/* +* Leaf 1 (0x40000000) +* EAX: Largest Xen-information leaf. All leaves up to an including @EAX +* are supported by the Xen host. +* EBX-EDX: "XenVMMXenVMM" signature, allowing positive identification +* of a Xen host. +*/ +#define XEN_CPUID_SIGNATURE_EBX 0x566e6558/* "XenV" */ +#define XEN_CPUID_SIGNATURE_ECX 0x65584d4d/* "MMXe" */ +#define XEN_CPUID_SIGNATURE_EDX 0x4d4d566e/* "nVMM" */ + +/* +* Leaf 2 (0x40000001) +* EAX[31:16]: Xen major version. +* EAX[15: 0]: Xen minor version. +* EBX-EDX: Reserved (currently all zeroes). +*/ + +/* +* Leaf 3 (0x40000002) +* EAX: Number of hypercall transfer pages. This register is always guaranteed +* to specify one hypercall page. +* EBX: Base address of Xen-specific MSRs. +* ECX: Features 1. Unused bits are set to zero. +* EDX: Features 2. Unused bits are set to zero. +*/ + +/* Does the host support MMU_PT_UPDATE_PRESERVE_AD for this guest? */ +#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0 +#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) + +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
Signature · 1 line
1.5.4.5
PATCH v5 3/7 xen: Make pv drivers only work with xen_pv_domain()
In reply to Sheng YangOtherwise they would still try to enable with HVM domain type.
--- drivers/block/xen-blkfront.c | 2 +- drivers/input/xen-kbdfront.c | 2 +- drivers/net/xen-netfront.c | 2 +- drivers/video/xen-fbfront.c | 2 +- drivers/xen/grant-table.c | 2 +- drivers/xen/xenbus/xenbus_probe.c | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-)
drivers/block/xen-blkfront.c +1 -1
@@ -1068,7 +1068,7 @@ static struct xenbus_driver blkfront = {static int __init xlblk_init (void ) { -if (!xen_domain ()) +if (!xen_pv_domain ())return -ENODEV ;if (register_blkdev (XENVBD_MAJOR ,DEV_NAME )) {
drivers/input/xen-kbdfront.c +1 -1
@@ -338,7 +338,7 @@ static struct xenbus_driver xenkbd_driver = {static int __init xenkbd_init (void ) { -if (!xen_domain ()) +if (!xen_pv_domain ())return -ENODEV ;/* Nothing to do if running in dom0. */
drivers/net/xen-netfront.c +1 -1
@@ -1804,7 +1804,7 @@ static struct xenbus_driver netfront_driver = {static int __init netif_init (void ) { -if (!xen_domain ()) +if (!xen_pv_domain ())return -ENODEV ;if (xen_initial_domain ())
drivers/video/xen-fbfront.c +1 -1
@@ -683,7 +683,7 @@ static struct xenbus_driver xenfb_driver = {static int __init xenfb_init (void ) { -if (!xen_domain ()) +if (!xen_pv_domain ())return -ENODEV ;/* Nothing to do if running in dom0. */
drivers/xen/grant-table.c +1 -1
@@ -510,7 +510,7 @@ static int __devinit gnttab_init(void)unsigned int max_nr_glist_frames, nr_glist_frames;unsigned int nr_init_grefs; -if (!xen_domain ()) +if (!xen_pv_domain ())return -ENODEV ; nr_grant_frames =1 ;
drivers/xen/xenbus/xenbus_probe.c +2 -2
@@ -786,7 +786,7 @@ static int __init xenbus_probe_init(void)DPRINTK ("" ); err = -ENODEV ; -if (!xen_domain ()) +if (!xen_pv_domain ()) goto out_error;/* Register ourselves with the kernel bus subsystem */
@@ -922,7 +922,7 @@ static void wait_for_devices(struct xenbus_driver *xendrv)struct device_driver *drv = xendrv ? &xendrv->driver :NULL ;unsigned int seconds_waited =0 ; -if (!ready_to_wait_for_devices || !xen_domain ()) +if (!ready_to_wait_for_devices || !xen_pv_domain ())return ;while (exists_connecting_device (drv)) {
Signature · 1 line
1.5.4.5
PATCH v5 4/7 xen/hvm: Xen PV extension of HVM initialization
In reply to Sheng YangThe PV extended HVM(once known as Hybrid) is started from real mode like HVM guest, but also with a component based PV feature selection(e.g. PV halt, PV timer, event channel, then PV drivers). So guest can takes the advantages of both H/W virtualization and Para-Virtualization.
This patch introduced the PV extension of HVM guest initialization.
Guest would detect the capability using CPUID 0x40000002.edx, then call HVMOP_enable_pv hypercall to enable pv support in hypervisor.
--- arch/x86/include/asm/xen/cpuid.h | 5 ++ arch/x86/xen/Kconfig | 8 ++ arch/x86/xen/Makefile | 1 + arch/x86/xen/hvmpv.c | 133 ++++++++++++++++++++++++++++++++++++ include/xen/interface/hvm/hvm_op.h | 8 ++ include/xen/xen.h | 9 +++ 6 files changed, 164 insertions(+), 0 deletions(-) create mode 100644 arch/x86/xen/hvmpv.c
arch/x86/include/asm/xen/cpuid.h +5 -0
@@ -65,4 +65,9 @@#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV 0 +#define XEN_CPUID_FEAT2_HVM_PV (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV_EVTCHN 1 +#define XEN_CPUID_FEAT2_HVM_PV_EVTCHN (1u<<1) +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
arch/x86/xen/Kconfig +8 -0
@@ -36,3 +36,11 @@ config XEN_DEBUG_FS help Enable statistics output and various tuning options in debugfs. Enabling this option may incur a significant performance overhead. + +config XEN_HVM_PV + bool "PV extension of HVM" + depends on XEN && X86_64 + default n + help + Enable the PV extension of HVM, so that HVM guest would benefit from + PV features like PV timer and evtchn.
arch/x86/xen/Makefile +1 -0
@@ -17,4 +17,5 @@ obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o +obj-$(CONFIG_XEN_HVM_PV) += hvmpv.o
arch/x86/xen/hvmpv.c new file +133 -0
133 lines not on this page · raw message
include/xen/interface/hvm/hvm_op.h +8 -0
@@ -69,4 +69,12 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_set_pci_link_route);/* Flushes all VCPU TLBs: @arg must be NULL. */ #define HVMOP_flush_tlbs 5 +#define HVMOP_enable_pv 9 +struct xen_hvm_pv_type { +domid_t domid; +uint32_t flags; +#define HVM_PV_CLOCK (1ull<<0) +#define HVM_PV_EVTCHN (1ull<<1) +}; +#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */
include/xen/xen.h +9 -0
@@ -19,6 +19,15 @@ extern enum xen_domain_type xen_domain_type;#define xen_hvm_domain () (xen_domain() && \ xen_domain_type == XEN_HVM_DOMAIN) +#ifdef CONFIG_XEN_HVM_PV + +#define XEN_HVM_PV_EVTCHN_ENABLED (1u << 1) +extern u32 xen_hvm_pv_features; + +#define xen_hvm_pv_evtchn_enabled () \ + (xen_hvm_pv_features & XEN_HVM_PV_EVTCHN_ENABLED) +#endif /* CONFIG_XEN_HVM_PV */ +#ifdef CONFIG_XEN_DOM0 #include <xen/interface/xen.h> #include <asm/xen/hypervisor.h>
Signature · 1 line
1.5.4.5
Re: [PATCH 4/7] xen/hvm: Xen PV extension of HVM initialization
Quoted 7 lines · Sheng Yang
arch/x86/xen/Kconfig +0
-0@@ -36,3 +36,11 @@ config XEN_DEBUG_FS help
arch/x86/xen/Kconfig config XEN_DEBUG_FS · line 46
Quoted 10 lines
arch/x86/xen/Kconfig +8
-0Enable statistics output and various tuning options in debugfs. Enabling this option may incur a significant performance overhead. + +config XEN_HVM_PV + bool "PV extension of HVM" + depends on XEN&& X86_64 + default n + help + Enable the PV extension of HVM, so that HVM guest would benefit from + PV features like PV timer and evtchn.
Unless there's a strong reason to make this a user-visible setting, I'd make it a hidden option which is enabled by default when the dependencies are correct.
Quoted 94 lines
arch/x86/xen/Makefile +1
-0@@ -17,4 +17,5 @@ obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o +obj-$(CONFIG_XEN_HVM_PV) += hvmpv.oarch/x86/xen/hvmpv.c new file +77
-0@@ -0,0 +1,133 @@ +/* +* PV extension of HVM implementation. +* +* Sheng Yang<[email protected]>, Intel Corporation, 2010 +* +*/ +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/percpu.h> +#include <linux/module.h> + +#include <xen/xen.h> +#include <xen/features.h> +#include <xen/events.h> +#include <xen/hvm.h> +#include <xen/interface/xen.h> +#include <xen/interface/version.h> +#include <xen/interface/memory.h> + +#include <asm/xen/cpuid.h> +#include <asm/xen/hypercall.h> +#include <asm/xen/hypervisor.h> + +#include "xen-ops.h" + +u32 xen_hvm_pv_features; +EXPORT_SYMBOL_GPL (xen_hvm_pv_features); + +static const struct pv_info xen_hvm_pv_info__initdata = { + .paravirt_enabled =1 , + .shared_kernel_pmd =0 , + .kernel_rpl =0 , + .name ="Xen" , +}; + +static void __init xen_hvm_pv_banner (void ) +{ +unsigned version =HYPERVISOR_xen_version (XENVER_version,NULL ); +struct xen_extraversion extra; +HYPERVISOR_xen_version (XENVER_extraversion,&extra); + +printk (KERN_INFO "Booting PV extended HVM kernel on %s\n" , + pv_info.name); +printk (KERN_INFO "Xen version: %d.%d%s\n" , + version>>16 , version&0xffff , extra.extraversion); +} + +static int __init xen_para_available (void ) +{ +uint32_t eax, ebx, ecx, edx; +cpuid (XEN_CPUID_LEAF (0 ),&eax,&ebx,&ecx,&edx); + +if (ebx ==XEN_CPUID_SIGNATURE_EBX && + ecx ==XEN_CPUID_SIGNATURE_ECX && + edx ==XEN_CPUID_SIGNATURE_EDX && + ((eax -XEN_CPUID_LEAF (0 ))>=2 )) +return 1 ; + +return 0 ; +} + +static int __init enable_hvm_pv (u64 flags) +{ +struct xen_hvm_pv_type a; + + a.domid =DOMID_SELF ; + a.flags = flags; +return HYPERVISOR_hvm_op (HVMOP_enable_pv,&a); +} + +static int __init init_hvm_pv_info (void ) +{ +uint32_t ecx, edx, pages, msr; +u64 pfn; + +if (!xen_para_available ()) +return -EINVAL ;
arch/x86/xen/hvmpv.c line 87
Quoted 10 lines
arch/x86/xen/hvmpv.c new file +10
-0+ +cpuid (XEN_CPUID_LEAF (2 ),&pages,&msr,&ecx,&edx); + +/* Check if hvm_pv mode is supported */ +if (!(edx&XEN_CPUID_FEAT2_HVM_PV )) +return -ENODEV ; + +/* We only support 1 page of hypercall for now */ +if (pages !=1 ) +return -ENOMEM ;
Why does it matter? If you're only using hypercalls in the first page, it doesn't matter whether Xen exports more pages beyond that.
arch/x86/xen/hvmpv.c line 90
Quoted 4 lines
arch/x86/xen/hvmpv.c new file +3
-0+ + pfn =__pa (hypercall_page); +wrmsrl (msr, pfn);
wrmsr_safe() with error check, just in case it fails?
arch/x86/xen/hvmpv.c line 97
Quoted 8 lines
arch/x86/xen/hvmpv.c new file +7
-0+ +xen_setup_features (); + + x86_init.oem.banner = xen_hvm_pv_banner; + pv_info = xen_hvm_pv_info; + + xen_domain_type =XEN_HVM_DOMAIN ;
Set this very last, once everything else has successfully set up.
Quoted 7 lines
arch/x86/xen/hvmpv.c new file +6
-0+ +return 0 ; +} + +static struct shared_info shared_info_page __page_aligned_bss; +
arch/x86/xen/hvmpv.c line 113
Quoted 10 lines
arch/x86/xen/hvmpv.c new file +10
-0+static void __init init_shared_info (void ) +{ +struct xen_add_to_physmap xatp; + + xatp.domid =DOMID_SELF ; + xatp.idx =0 ; + xatp.space = XENMAPSPACE_shared_info; + xatp.gpfn =__pa (&shared_info_page)>>PAGE_SHIFT ; +if (HYPERVISOR_memory_op (XENMEM_add_to_physmap,&xatp)) +BUG ();
It isn't too late to back out if it fails; just return an error code.
arch/x86/xen/hvmpv.c line 125
Quoted 13 lines
arch/x86/xen/hvmpv.c new file +12
-0+ + HYPERVISOR_shared_info = (struct shared_info *)&shared_info_page; + +per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; +} + +void __init xen_guest_init (void ) +{ +int r; + +/* Ensure the we won't confused with others */ +if (xen_domain_type !=XEN_NATIVE )
Just use xen_domain()?
Quoted 32 lines
arch/x86/xen/hvmpv.c new file +8
-0+return ; + + r =init_hvm_pv_info (); +if (r<0 ) +return ; + +init_shared_info (); +}include/xen/interface/hvm/hvm_op.h +8
-0@@ -69,4 +69,12 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_set_pci_link_route);/* Flushes all VCPU TLBs: @arg must be NULL. */ #define HVMOP_flush_tlbs 5 +#define HVMOP_enable_pv 9 +struct xen_hvm_pv_type { +domid_t domid; +uint32_t flags; +#define HVM_PV_CLOCK (1ull<<0) +#define HVM_PV_EVTCHN (1ull<<1) +}; +#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */ include/xen/xen.h +0
-0@@ -19,6 +19,15 @@ extern enum xen_domain_type xen_domain_type;#define xen_hvm_domain () (xen_domain()&& \ xen_domain_type == XEN_HVM_DOMAIN)
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 30
Quoted 10 lines
include/xen/xen.h +8
-0+#ifdef CONFIG_XEN_HVM_PV + +#define XEN_HVM_PV_EVTCHN_ENABLED (1u<< 1) +extern u32 xen_hvm_pv_features; + +#define xen_hvm_pv_evtchn_enabled () \ + (xen_hvm_pv_features& XEN_HVM_PV_EVTCHN_ENABLED) +#endif /* CONFIG_XEN_HVM_PV */
Did you compile test this with !CONFIG_XEN_HVM_PV? I think it would fail with xen_hvm_pv_evtchn_enabled() undefined. Fortunately making xen_hvm_pv_evtchn_enabled() evaluate to constant 0 in the !configed case will clean things up.
J
PATCH v5 5/7 x86/xen: The entrance for PV extension of HVM
In reply to Sheng Yangxen_guest_init() would setup the environment.
--- arch/x86/include/asm/xen/hypervisor.h | 6 ++++++ arch/x86/kernel/setup.c | 4 ++++ 2 files changed, 10 insertions(+), 0 deletions(-)
arch/x86/include/asm/xen/hypervisor.h +6 -0
@@ -37,4 +37,10 @@extern struct shared_info *HYPERVISOR_shared_info;extern struct start_info *xen_start_info; +#ifdef CONFIG_XEN_HVM_PV +void __init xen_guest_init (void ); +#else +static inline void xen_guest_init (void ) {}; +#endif +#endif /* _ASM_X86_XEN_HYPERVISOR_H */
arch/x86/kernel/setup.c +4 -0
@@ -113,6 +113,8 @@#endif #include <asm/mce.h> +#include <asm/xen/hypervisor.h> +/* * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. * The direct mapping extends to max_pfn_mapped, so that we can directly access
@@ -740,6 +742,8 @@ void __init setup_arch(char **cmdline_p) x86_init.oem.arch_setup (); +xen_guest_init (); +setup_memory_map ();parse_setup_data ();/* update the e820_saved too */
Signature · 1 line
1.5.4.5
PATCH v5 6/7 xen: Enable PV clocksource for HVM
In reply to Sheng YangAnd enable it by default in PV extended HVM guest.
--- arch/x86/xen/hvmpv.c | 20 ++++++++++++++++++++ arch/x86/xen/time.c | 4 +++- 2 files changed, 23 insertions(+), 1 deletions(-)
arch/x86/xen/hvmpv.c +20 -0
@@ -43,6 +43,7 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version >>16 , version &0xffff , extra.extraversion); +printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); }static int __init xen_para_available (void )
@@ -117,6 +118,22 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) = &HYPERVISOR_shared_info->vcpu_info[0 ]; } +extern struct clocksource xen_clocksource; + +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG (); + + pv_time_ops.sched_clock = xen_sched_clock; + + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + +clocksource_register (&xen_clocksource); +} +void __init xen_guest_init (void ) {int r;
@@ -130,4 +147,7 @@ void __init xen_guest_init(void)return ;init_shared_info (); + +/* PV clocksource would be enabled by default */ +init_pv_clocksource (); }
arch/x86/xen/time.c +3 -1
@@ -13,6 +13,7 @@#include <linux/clockchips.h> #include <linux/kernel_stat.h> #include <linux/math64.h> +#include <linux/module.h> #include <asm/pvclock.h> #include <asm/xen/hypervisor.h>
@@ -243,7 +244,7 @@ int xen_set_wallclock(unsigned long now)return -1 ; } -static struct clocksource xen_clocksource__read_mostly = { +struct clocksource xen_clocksource__read_mostly = { .name ="xen" , .rating =400 , .read = xen_clocksource_get_cycles,
@@ -252,6 +253,7 @@ static struct clocksource xen_clocksource __read_mostly = { .shift = XEN_SHIFT, .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +EXPORT_SYMBOL_GPL (xen_clocksource);/* Xen clockevent implementation
Signature · 1 line
1.5.4.5
Re: [PATCH 6/7] xen: Enable PV clocksource for HVM
Quoted 21 lines · Sheng Yang
And enable it by default in PV extended HVM guest.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/hvmpv.c | 20 ++++++++++++++++++++ arch/x86/xen/time.c | 4 +++- 2 files changed, 23 insertions(+), 1 deletions(-)arch/x86/xen/hvmpv.c +1
-0@@ -43,6 +43,7 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); }static int __init xen_para_available (void )
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 122
Quoted 5 lines
arch/x86/xen/hvmpv.c +1
-0@@ -117,6 +118,22 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; } +extern struct clocksource xen_clocksource;
No externs in .c files. But aside from that, I'd prefer you export a function from xen/time.c to do the clocksource registration and call it from both places (even if its a one-liner).
arch/x86/xen/hvmpv.c static void __init init_shared_info(void)
Quoted 6 lines
arch/x86/xen/hvmpv.c +5
-0+ +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG ();
BUG is a bit severe. Will it really never, ever fail? And if it does, the consequence is hardly serious; we just fall back to emulated devices.
Quoted 51 lines
arch/x86/xen/hvmpv.c +10
-0+ + pv_time_ops.sched_clock = xen_sched_clock; + + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + +clocksource_register (&xen_clocksource); +} +void __init xen_guest_init (void ){ int r;arch/x86/xen/hvmpv.c +3
-0@@ -130,4 +147,7 @@ void __init xen_guest_init(void)return ;init_shared_info (); + +/* PV clocksource would be enabled by default */ +init_pv_clocksource (); }arch/x86/xen/time.c +3
-1@@ -13,6 +13,7 @@#include <linux/clockchips.h> #include <linux/kernel_stat.h> #include <linux/math64.h> +#include <linux/module.h> #include <asm/pvclock.h> #include <asm/xen/hypervisor.h> @@ -243,7 +244,7 @@ int xen_set_wallclock(unsigned long now)return -1 ; } -static struct clocksource xen_clocksource__read_mostly = { +struct clocksource xen_clocksource__read_mostly = { .name ="xen" , .rating =400 , .read = xen_clocksource_get_cycles,@@ -252,6 +253,7 @@ static struct clocksource xen_clocksource __read_mostly = { .shift = XEN_SHIFT, .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +EXPORT_SYMBOL_GPL (xen_clocksource);/* Xen clockevent implementation
Re: [PATCH 6/7] xen: Enable PV clocksource for HVM
Quoted 9 lines · Jeremy Fitzhardinge
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 126
Quoted 7 lines
arch/x86/xen/hvmpv.c +5
-0+ +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG ();BUG is a bit severe. Will it really never, ever fail?
When running on a hypervisor without these extension, for example.
Ian.
Signature · 3 lines
Ian Campbell My polyvinyl cowboy wallet was made in Hong Kong by Montgomery Clift!
Re: [PATCH 6/7] xen: Enable PV clocksource for HVM
In reply to Jeremy FitzhardingeQuoted 25 lines · On Friday 05 March 2010 01:40:51 Jeremy Fitzhardinge wrote:
Quoted 24 lines · Sheng Yang
And enable it by default in PV extended HVM guest.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/hvmpv.c | 20 ++++++++++++++++++++ arch/x86/xen/time.c | 4 +++- 2 files changed, 23 insertions(+), 1 deletions(-)arch/x86/xen/hvmpv.c +1
-0@@ -43,6 +43,7 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); }static int __init xen_para_available (void )@@ -117,6 +118,22 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; }
Quoted 5 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 121
Quoted 1 line
arch/x86/xen/hvmpv.c +1
-0+extern struct clocksource xen_clocksource;No externs in .c files. But aside from that, I'd prefer you export a function from
xen/time.cto do the clocksource registration and call it from both places (even if its a one-liner).
OK
Quoted 9 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 126
Quoted 6 lines
arch/x86/xen/hvmpv.c +5
-0+ +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG ();BUG is a bit severe. Will it really never, ever fail? And if it does, the consequence is hardly serious; we just fall back to emulated devices.
It shouldn't fail. If we got a hypervisor without these extension, this one won't be called. Because the feature should be detected by CPUID, and the hypervisor would mask the unsupported features. So the guest won't see that bit if it's not supported, and this one wouldn't be involved.
Signature · 2 lines
regards Yang, Sheng
Quoted 52 lines
Quoted 51 lines
arch/x86/xen/hvmpv.c +13
-0+ + pv_time_ops.sched_clock = xen_sched_clock; + + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + +clocksource_register (&xen_clocksource); +} +void __init xen_guest_init (void ) {int r;@@ -130,4 +147,7 @@ void __init xen_guest_init(void)return ;init_shared_info (); + +/* PV clocksource would be enabled by default */ +init_pv_clocksource (); }arch/x86/xen/time.c +3
-1@@ -13,6 +13,7 @@#include <linux/clockchips.h> #include <linux/kernel_stat.h> #include <linux/math64.h> +#include <linux/module.h> #include <asm/pvclock.h> #include <asm/xen/hypervisor.h> @@ -243,7 +244,7 @@ int xen_set_wallclock(unsigned long now)return -1 ; } -static struct clocksource xen_clocksource__read_mostly = { +struct clocksource xen_clocksource__read_mostly = { .name ="xen" , .rating =400 , .read = xen_clocksource_get_cycles,@@ -252,6 +253,7 @@ static struct clocksource xen_clocksource__read_mostly = { .shift =XEN_SHIFT , .flags =CLOCK_SOURCE_IS_CONTINUOUS , }; +EXPORT_SYMBOL_GPL (xen_clocksource);/* Xen clockevent implementation
Re: [PATCH 6/7] xen: Enable PV clocksource for HVM
Quoted 43 lines · Sheng Yang
Quoted 25 lines · On Friday 05 March 2010 01:40:51 Jeremy Fitzhardinge wrote:
Quoted 24 lines · Sheng Yang
And enable it by default in PV extended HVM guest.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/hvmpv.c | 20 ++++++++++++++++++++ arch/x86/xen/time.c | 4 +++- 2 files changed, 23 insertions(+), 1 deletions(-)arch/x86/xen/hvmpv.c +1
-0@@ -43,6 +43,7 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); }static int __init xen_para_available (void )@@ -117,6 +118,22 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; }Quoted 5 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 121
Quoted 1 line
arch/x86/xen/hvmpv.c +1
-0+extern struct clocksource xen_clocksource;No externs in .c files. But aside from that, I'd prefer you export a function from
xen/time.cto do the clocksource registration and call it from both places (even if its a one-liner).OK
Quoted 9 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 126
Quoted 6 lines
arch/x86/xen/hvmpv.c +5
-0+ +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG ();BUG is a bit severe. Will it really never, ever fail? And if it does, the consequence is hardly serious; we just fall back to emulated devices.
Quoted 4 lines
It shouldn't fail. If we got a hypervisor without these extension, this one won't be called. Because the feature should be detected by CPUID, and the hypervisor would mask the unsupported features. So the guest won't see that bit if it's not supported, and this one wouldn't be involved.
It is still possible that a hypervisor might want to turn on the general extensions but not the PV clock extensions, and we can't predict what reasons we might have for doing that in the future. A BUG() is an awfully big hammer for a failure like this and destroys any hope of making changes in a forward/backward compatible manner in the future. Surely it is possible to simply continue with non-PV clock?
Ian.
Re: [PATCH 6/7] xen: Enable PV clocksource for HVM
Quoted 50 lines · On Friday 05 March 2010 17:31:47 Ian Campbell wrote:
Quoted 44 lines · Sheng Yang
Quoted 25 lines · On Friday 05 March 2010 01:40:51 Jeremy Fitzhardinge wrote:
Quoted 24 lines · Sheng Yang
And enable it by default in PV extended HVM guest.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/hvmpv.c | 20 ++++++++++++++++++++ arch/x86/xen/time.c | 4 +++- 2 files changed, 23 insertions(+), 1 deletions(-)arch/x86/xen/hvmpv.c +1
-0@@ -43,6 +43,7 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); }static int __init xen_para_available (void )@@ -117,6 +118,22 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; }Quoted 5 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 121
Quoted 1 line
arch/x86/xen/hvmpv.c +1
-0+extern struct clocksource xen_clocksource;No externs in .c files. But aside from that, I'd prefer you export a function from
xen/time.cto do the clocksource registration and call it from both places (even if its a one-liner).OK
Quoted 9 lines
arch/x86/xen/hvmpv.c static void __init init_shared_info(void) · line 126
Quoted 5 lines
arch/x86/xen/hvmpv.c +5
-0+ +static void __init init_pv_clocksource (void ) +{ +if (enable_hvm_pv (HVM_PV_CLOCK )) +BUG ();BUG is a bit severe. Will it really never, ever fail? And if it does, the consequence is hardly serious; we just fall back to emulated devices.
Quoted 4 lines
It shouldn't fail. If we got a hypervisor without these extension, this one won't be called. Because the feature should be detected by CPUID, and the hypervisor would mask the unsupported features. So the guest won't see that bit if it's not supported, and this one wouldn't be involved.
Quoted 6 lines
It is still possible that a hypervisor might want to turn on the general extensions but not the PV clock extensions, and we can't predict what reasons we might have for doing that in the future. A
BUG()is an awfully big hammer for a failure like this and destroys any hope of making changes in a forward/backward compatible manner in the future. Surely it is possible to simply continue with non-PV clock?
Evtchn can't work with it.
If the potentially disable pv clock is your concern, I think the better way is to make PV clock as a separate feature, and let PV evtchn depends on it. I've updated the patches to reflect this change.
Signature · 2 lines
regards Yang, Sheng
PATCH v5 7/7 xen: Make event channel work with PV extension of HVM
In reply to Sheng YangWe mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used X86_PLATFORM_IPI_VECTOR as the noficiation vector for hypervisor to notify guest about the event.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Then we don't use IOAPIC/LAPIC, eliminated the overhead brought by unnecessary VMExit caused by LAPIC.
--- arch/x86/xen/enlighten.c | 6 +- arch/x86/xen/hvmpv.c | 68 +++++++++++++++++++++++++++++++- arch/x86/xen/irq.c | 28 +++++++++++++ arch/x86/xen/smp.c | 92 ++++++++++++++++++++++++++++++++++++++++-- arch/x86/xen/xen-ops.h | 14 +++++++ drivers/xen/events.c | 74 +++++++++++++++++++++++++++++++--- include/xen/events.h | 6 +++ include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- include/xen/xen.h | 2 + 10 files changed, 285 insertions(+), 16 deletions(-)
arch/x86/xen/enlighten.c +3 -3
@@ -717,7 +717,7 @@ static u32 xen_safe_apic_wait_icr_idle(void)return 0 ; } -static void set_xen_basic_apic_ops (void ) +void xen_set_basic_apic_ops (void ) { apic->read = xen_apic_read; apic->write = xen_apic_write;
@@ -1026,7 +1026,7 @@ static void xen_crash_shutdown(struct pt_regs *regs)xen_reboot (SHUTDOWN_crash); } -static const struct machine_ops __initdata xen_machine_ops = { +const struct machine_ops __initdata xen_machine_ops = { .restart = xen_restart, .halt = xen_machine_halt, .power_off = xen_machine_halt,
@@ -1116,7 +1116,7 @@ asmlinkage void __init xen_start_kernel(void)/* * set up the basic apic ops. */ -set_xen_basic_apic_ops (); +xen_set_basic_apic_ops ();#endif if (xen_feature (XENFEAT_mmu_pt_update_preserve_ad)) {
arch/x86/xen/hvmpv.c +67 -1
101 lines not on this page · raw message
arch/x86/xen/irq.c +28 -0
@@ -2,6 +2,7 @@#include <asm/x86_init.h> +#include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h>
@@ -131,3 +132,30 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +#ifdef CONFIG_XEN_HVM_PV +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); + +void __init xen_hvm_pv_init_irq_ops (void ) +{ +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops.irq_disable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable); + pv_irq_ops.irq_enable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable); + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + } +} +#endif
arch/x86/xen/smp.c +87 -5
147 lines not on this page · raw message
arch/x86/xen/xen-ops.h +14 -0
@@ -51,6 +51,12 @@ void __init xen_time_init(void);unsigned long xen_get_wallclock (void );int xen_set_wallclock (unsigned long time);unsigned long long xen_sched_clock (void ); +void xen_set_basic_apic_ops (void ); + +#ifdef CONFIG_XEN_HVM_PV +void __init xen_hvm_pv_init_irq_ops (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void ); +#endif /* CONFIG_XEN_HVM_PV */ irqreturn_t xen_debug_interrupt (int irq,void *dev_id);
@@ -61,9 +67,17 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_pv_smp_init (void ); +#endif /* CONFIG_XEN_HVM_PV */ +extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +#ifdef CONFIG_XEN_HVM_PV +static inline void xen_hvm_pv_smp_init (void ) {} +#endif /* CONFIG_XEN_HVM_PV */ +#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS
drivers/xen/events.c +68 -6
112 lines not on this page · raw message
include/xen/events.h +6 -0
@@ -56,4 +56,10 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs); + +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_evtchn_do_upcall (struct pt_regs *regs); +#endif +#endif /* _XEN_EVENTS_H */
include/xen/hvm.h +5 -0
@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx) return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x) (((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ + HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +#endif /* XEN_HVM_H__ */
include/xen/interface/xen.h +5 -1
@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS (VIRQ_EMUL_PIN_START + VIRQ_EMUL_PIN_NUM) +/* * MMU-UPDATE REQUESTS *
include/xen/xen.h +2 -0
@@ -26,6 +26,8 @@ extern u32 xen_hvm_pv_features;#define xen_hvm_pv_evtchn_enabled () \ (xen_hvm_pv_features & XEN_HVM_PV_EVTCHN_ENABLED) +#else +#define xen_hvm_pv_evtchn_enabled () 0#endif /* CONFIG_XEN_HVM_PV */ #ifdef CONFIG_XEN_DOM0
Signature · 1 line
1.5.4.5
Re: [PATCH 7/7] xen: Make event channel work with PV extension of HVM
Quoted 5 lines · Sheng Yang
We mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used
X86_PLATFORM_IPI_VECTORas the noficiation vector for hypervisor
"notification"
Quoted 7 lines
to notify guest about the event.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Then we don't use IOAPIC/LAPIC, eliminated the overhead brought by unnecessary VMExit caused by LAPIC.
I think you need to expand this comment to explain the consequences of this change on SMP bringup and timers. I still don't see why there are (or should be) any differences from PV cpu bringup, aside from the tiny details of syscall setup (and MTRR I guess, but that's all a noop anyway, right?).
Also, I think you can probably do the timer changes in a separate patch, as even with evtchns, an emulated timer device interrupt will still be delivered properly via VIRQ_EMUL_PIN_*.
Quoted 254 lines
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/enlighten.c | 6 +- arch/x86/xen/hvmpv.c | 68 +++++++++++++++++++++++++++++++- arch/x86/xen/irq.c | 28 +++++++++++++ arch/x86/xen/smp.c | 92 ++++++++++++++++++++++++++++++++++++++++-- arch/x86/xen/xen-ops.h | 14 +++++++ drivers/xen/events.c | 74 +++++++++++++++++++++++++++++++--- include/xen/events.h | 6 +++ include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- include/xen/xen.h | 2 + 10 files changed, 285 insertions(+), 16 deletions(-)arch/x86/xen/enlighten.c +3
-3@@ -717,7 +717,7 @@ static u32 xen_safe_apic_wait_icr_idle(void)return 0 ; } -static void set_xen_basic_apic_ops (void ) +void xen_set_basic_apic_ops (void ) { apic->read = xen_apic_read; apic->write = xen_apic_write;@@ -1026,7 +1026,7 @@ static void xen_crash_shutdown(struct pt_regs *regs)xen_reboot (SHUTDOWN_crash); } -static const struct machine_ops __initdata xen_machine_ops = { +const struct machine_ops __initdata xen_machine_ops = { .restart = xen_restart, .halt = xen_machine_halt, .power_off = xen_machine_halt,@@ -1116,7 +1116,7 @@ asmlinkage void __init xen_start_kernel(void)/* * set up the basic apic ops. */ -set_xen_basic_apic_ops (); +xen_set_basic_apic_ops ();#endif if (xen_feature (XENFEAT_mmu_pt_update_preserve_ad)) {arch/x86/xen/hvmpv.c +67
-1@@ -17,6 +17,7 @@#include <xen/interface/version.h> #include <xen/interface/memory.h> +#include <asm/reboot.h> #include <asm/xen/cpuid.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> @@ -44,6 +45,8 @@ static void __init xen_hvm_pv_banner(void)printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion);printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); +if (xen_hvm_pv_evtchn_enabled ()) +printk (KERN_INFO "PV feature: Event channel enabled\n" ); }static int __init xen_para_available (void )@@ -83,6 +86,9 @@ static int __init init_hvm_pv_info(void)if (!(edx&XEN_CPUID_FEAT2_HVM_PV ))return -ENODEV ; +if (edx&XEN_CPUID_FEAT2_HVM_PV_EVTCHN ) + xen_hvm_pv_features |=XEN_HVM_PV_EVTCHN_ENABLED ; +/* We only support 1 page of hypercall for now */ if (pages !=1 )return -ENOMEM ;@@ -131,12 +137,42 @@ static void __init init_pv_clocksource(void) x86_platform.get_wallclock = xen_get_wallclock; x86_platform.set_wallclock = xen_set_wallclock; -clocksource_register (&xen_clocksource); +/* It would be done in xen_time_init() if evtchn enabled */ +if (!xen_hvm_pv_evtchn_enabled ()) +clocksource_register (&xen_clocksource); +} + +static int set_callback_via (uint64_t via) +{ +struct xen_hvm_param a; + + a.domid =DOMID_SELF ; + a.index =HVM_PARAM_CALLBACK_IRQ ; + a.value = via; +return HYPERVISOR_hvm_op (HVMOP_set_param,&a); } +void do_hvm_pv_evtchn_intr (void ) +{ +per_cpu (irq_count,smp_processor_id ())++; +xen_hvm_evtchn_do_upcall (get_irq_regs ()); +per_cpu (irq_count,smp_processor_id ())--; +} + +#ifdef CONFIG_X86_LOCAL_APIC +static void xen_hvm_pv_evtchn_apic_write (u32 reg,u32 val) +{ +/* The only one reached here should be EOI */ +WARN_ON (reg !=APIC_EOI ); +} +#endif + +extern struct machine_ops xen_machine_ops; +void __init xen_guest_init (void ) {int r; +uint64_t callback_via;/* Ensure the we won't confused with others */ if (xen_domain_type !=XEN_NATIVE )@@ -150,4 +186,34 @@ void __init xen_guest_init(void)/* PV clocksource would be enabled by default */ init_pv_clocksource (); + +if (xen_hvm_pv_evtchn_enabled ()) { +if (enable_hvm_pv (HVM_PV_EVTCHN )) +return ; + +xen_hvm_pv_init_irq_ops (); + + x86_init.timers.timer_init = xen_time_init; + x86_init.timers.setup_percpu_clockev = x86_init_noop; + x86_cpuinit.setup_percpu_clockev = x86_init_noop; + + pv_apic_ops.startup_ipi_hook = paravirt_nop; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +xen_set_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write; +#endif + + callback_via =HVM_CALLBACK_VECTOR (X86_PLATFORM_IPI_VECTOR ); +set_callback_via (callback_via); + + x86_platform_ipi_callback = do_hvm_pv_evtchn_intr; + +disable_acpi (); + +xen_hvm_pv_smp_init (); + machine_ops = xen_machine_ops; + } }arch/x86/xen/irq.c +28
-0@@ -2,6 +2,7 @@#include <asm/x86_init.h> +#include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h> @@ -131,3 +132,30 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +#ifdef CONFIG_XEN_HVM_PV +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); + +void __init xen_hvm_pv_init_irq_ops (void ) +{ +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops.irq_disable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable); + pv_irq_ops.irq_enable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable); + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + } +} +#endif arch/x86/xen/smp.c +15
-3@@ -15,18 +15,24 @@#include <linux/sched.h> #include <linux/err.h> #include <linux/smp.h> +#include <linux/nmi.h> #include <asm/paravirt.h> #include <asm/desc.h> #include <asm/pgtable.h> #include <asm/cpu.h> +#include <asm/trampoline.h> +#include <asm/tlbflush.h> +#include <asm/mtrr.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <asm/xen/hypervisor.h> +#include <xen/xen.h> #include <xen/page.h> #include <xen/events.h> @@ -63,8 +69,13 @@ static __cpuinit void cpu_bringup(void)touch_softlockup_watchdog ();preempt_disable (); -xen_enable_sysenter (); -xen_enable_syscall (); +if (xen_pv_domain ()) { +xen_enable_sysenter (); +xen_enable_syscall (); + }else if (xen_hvm_pv_evtchn_enabled ()) +set_mtrr_aps_delayed_init (); +else +BUG (); cpu =smp_processor_id ();smp_store_cpu_info (cpu);@@ -171,7 +182,8 @@ static void __init xen_smp_prepare_boot_cpu(void)/* We've switched to the "real" per-cpu gdt, so make sure the old memory can be recycled */ -make_lowmem_page_readwrite (xen_initial_gdt); +if (xen_feature (XENFEAT_writable_descriptor_tables)) +make_lowmem_page_readwrite (xen_initial_gdt);xen_setup_vcpu_info_placement (); }
arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle) · line 298
Quoted 5 lines
arch/x86/xen/smp.c +1
-0@@ -282,6 +294,47 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)return 0 ; } +#ifdef CONFIG_XEN_HVM_PV
Rather than #ifdeffing...
arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
Quoted 6 lines
arch/x86/xen/smp.c +5
-0+static __cpuinit int +hvm_pv_cpu_initialize_context (unsigned int cpu,struct task_struct *idle) +{ +struct vcpu_guest_context *ctxt; +unsigned long start_ip;
Just put
if (!xen_hvm_pv_evtchn_enabled()) return -EOPNOTSUPP; /* or something */
and make sure xen_hvm_pv_evtchn_enable() returns constant 0 in the !CONFIG_XEN_HVM_PV case.
Quoted 5 lines
arch/x86/xen/smp.c +5
-0+ +if (cpumask_test_and_set_cpu (cpu, xen_cpu_initialized_map)) +return 0 ; + + ctxt =kzalloc (sizeof (*ctxt),GFP_KERNEL );
arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
Quoted 10 lines
arch/x86/xen/smp.c +9
-0+if (ctxt ==NULL ) +return -ENOMEM ; + + early_gdt_descr.address = (unsigned long )get_cpu_gdt_table (cpu); + initial_code = (unsigned long )cpu_bringup_and_idle; + stack_start.sp = (void *) idle->thread.sp; + +/* start_ip had better be page-aligned! */ + start_ip =setup_trampoline ();
Why go via trampoline?
Quoted 25 lines
arch/x86/xen/smp.c +21
-0+ +/* only start_ip is what we want */ + ctxt->flags =VGCF_HVM_GUEST ; + ctxt->user_regs.eip = start_ip; + +printk (KERN_INFO "Booting processor %d ip 0x%lx\n" , cpu, start_ip); + +if (HYPERVISOR_vcpu_op (VCPUOP_initialise, cpu, ctxt)) +BUG (); + +kfree (ctxt); + return 0; +} +#else +static int hvm_pv_cpu_initialize_context (unsigned int cpu, +struct task_struct *idle) +{ +return 0 ; +} +#endif +static int __cpuinit xen_cpu_up(unsigned int cpu) { struct task_struct *idle = idle_task(cpu);
arch/x86/xen/smp.c static int __cpuinit xen_cpu_up(unsigned int cpu) · line 351
Quoted 7 lines
arch/x86/xen/smp.c +3
-0@@ -292,11 +345,17 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)irq_ctx_init (cpu);#else clear_tsk_thread_flag (idle,TIF_FORK ); + +if (xen_hvm_pv_evtchn_enabled ()) + initial_gs =per_cpu_offset (cpu);
Why does this need to be conditional?
arch/x86/xen/smp.c static int __cpuinit xen_cpu_up(unsigned int cpu) · line 361
Quoted 10 lines
arch/x86/xen/smp.c +4
-1+per_cpu (kernel_stack, cpu) = (unsigned long )task_stack_page (idle) -KERNEL_STACK_OFFSET +THREAD_SIZE ;#endif -xen_setup_runstate_info (cpu); +if (xen_pv_domain ()) +xen_setup_runstate_info (cpu); +
Does HVM not support this? If VCPUOP_register_runstate_memory_area doesn't work on HVM, then just change xen_setup_runstate_info() to not BUG on hypercall failure (and presumably nothing tries to use runstate_info).
Quoted 257 lines
arch/x86/xen/smp.c +0
-0xen_setup_timer (cpu);xen_init_lock_cpu(cpu);arch/x86/xen/smp.c +24
-1@@ -305,7 +364,13 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)/* make sure interrupts start blocked */ per_cpu (xen_vcpu, cpu)->evtchn_upcall_mask =1 ; - rc =cpu_initialize_context (cpu, idle); +if (xen_pv_domain ()) + rc =cpu_initialize_context (cpu, idle); +else if (xen_hvm_pv_evtchn_enabled ()) + rc =hvm_pv_cpu_initialize_context (cpu, idle); +else +BUG (); +if (rc)return rc;@@ -480,3 +545,20 @@ void __init xen_smp_init(void)xen_fill_possible_map ();xen_init_spinlocks (); } + +#ifdef CONFIG_XEN_HVM_PV +static void xen_hvm_pv_flush_tlb_others (const struct cpumask *cpumask, +struct mm_struct *mm,unsigned long va) +{ +/* TODO Make it more specific */ +flush_tlb_all (); +} + +void __init xen_hvm_pv_smp_init (void ) +{ +if (xen_hvm_pv_evtchn_enabled ()) { + smp_ops = xen_smp_ops; + pv_mmu_ops.flush_tlb_others = xen_hvm_pv_flush_tlb_others; + } +} +#endif arch/x86/xen/xen-ops.h +14
-0@@ -51,6 +51,12 @@ void __init xen_time_init(void);unsigned long xen_get_wallclock (void );int xen_set_wallclock (unsigned long time);unsigned long long xen_sched_clock (void ); +void xen_set_basic_apic_ops (void ); + +#ifdef CONFIG_XEN_HVM_PV +void __init xen_hvm_pv_init_irq_ops (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void ); +#endif /* CONFIG_XEN_HVM_PV */ irqreturn_t xen_debug_interrupt (int irq,void *dev_id);@@ -61,9 +67,17 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_pv_smp_init (void ); +#endif /* CONFIG_XEN_HVM_PV */ +extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +#ifdef CONFIG_XEN_HVM_PV +static inline void xen_hvm_pv_smp_init (void ) {} +#endif /* CONFIG_XEN_HVM_PV */ +#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS drivers/xen/events.c +68
-6@@ -32,14 +32,17 @@#include <asm/irq.h> #include <asm/idle.h> #include <asm/sync_bitops.h> +#include <asm/desc.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> +#include <xen/xen.h> #include <xen/xen-ops.h> #include <xen/events.h> #include <xen/interface/xen.h> #include <xen/interface/event_channel.h> +/* * This lock protects updates to the following mapping and reference-count * arrays. The lock does not need to be acquired to read the mapping tables. @@ -616,17 +619,13 @@ static DEFINE_PER_CPU(unsigned, xed_nesting_count);* a bitset of words which contain pending event bits. The second * level is a bitset of pending events themselves. */ -void xen_evtchn_do_upcall (struct pt_regs *regs) +void __xen_evtchn_do_upcall (struct pt_regs *regs) {int cpu =get_cpu (); -struct pt_regs *old_regs =set_irq_regs (regs);struct shared_info *s = HYPERVISOR_shared_info;struct vcpu_info *vcpu_info =__get_cpu_var (xen_vcpu);unsigned count; -exit_idle (); -irq_enter (); - do { unsigned long pending_words;@@ -662,10 +661,25 @@ void xen_evtchn_do_upcall(struct pt_regs *regs) }while (count !=1 ); out: +put_cpu (); +} + +void xen_evtchn_do_upcall (struct pt_regs *regs) +{ +struct pt_regs *old_regs =set_irq_regs (regs); + +exit_idle (); +irq_enter (); + +__xen_evtchn_do_upcall (regs); +irq_exit ();set_irq_regs (old_regs); +} -put_cpu (); +void xen_hvm_evtchn_do_upcall (struct pt_regs *regs) +{ +__xen_evtchn_do_upcall (regs); }/* Rebind a new event channel to an existing irq. */ @@ -944,3 +958,51 @@ void __init xen_init_IRQ(void)irq_ctx_init (smp_processor_id ()); } + +void __init xen_hvm_pv_evtchn_init_IRQ (void ) +{ +int i; + +xen_init_IRQ (); +for (i =0 ; i<NR_IRQS_LEGACY ; i++) { +struct evtchn_bind_virq bind_virq; +struct irq_desc *desc =irq_to_desc (i); +int virq, evtchn; + + virq = i +VIRQ_EMUL_PIN_START ; + bind_virq.virq = virq; + bind_virq.vcpu =0 ; + +if (HYPERVISOR_event_channel_op (EVTCHNOP_bind_virq, +&bind_virq) !=0 ) +BUG (); + + evtchn = bind_virq.port; + evtchn_to_irq[evtchn] = i; + irq_info[i] =mk_virq_info (evtchn, virq); + + desc->status =IRQ_DISABLED ; + desc->action =NULL ; + desc->depth =1 ; + +/* +* 16 old-style INTA-cycle interrupts: +*/ +set_irq_chip_and_handler_name (i,&xen_dynamic_chip, + handle_level_irq,"event" ); + } + +/* +* Cover the whole vector space, no vector can escape +* us. (some of these will be overridden and become +* 'special' SMP interrupts) +*/ +for (i =0 ; i< (NR_VECTORS -FIRST_EXTERNAL_VECTOR ); i++) { +int vector =FIRST_EXTERNAL_VECTOR + i; +if (vector !=IA32_SYSCALL_VECTOR ) +set_intr_gate (vector, interrupt[i]); + } + +/* generic IPI for platform specific use, now used for HVM evtchn */ +alloc_intr_gate (X86_PLATFORM_IPI_VECTOR , x86_platform_ipi); +}include/xen/events.h +6
-0@@ -56,4 +56,10 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs); + +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_evtchn_do_upcall (struct pt_regs *regs); +#endif +#endif /* _XEN_EVENTS_H */ include/xen/hvm.h +5
-0@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx) return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x) (((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ + HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +#endif /* XEN_HVM_H__ */ include/xen/interface/xen.h +5
-1@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS (VIRQ_EMUL_PIN_START + VIRQ_EMUL_PIN_NUM) +/* * MMU-UPDATE REQUESTS * include/xen/xen.h +2
-1@@ -26,6 +26,8 @@ extern u32 xen_hvm_pv_features;#define xen_hvm_pv_evtchn_enabled () \ (xen_hvm_pv_features& XEN_HVM_PV_EVTCHN_ENABLED) +#else +#define xen_hvm_pv_evtchn_enabled () 0#endif /* CONFIG_XEN_HVM_PV */ #ifdef CONFIG_XEN_DOM0 --1.5.4.5
Re: [Xen-devel] Re: [PATCH 7/7] xen: Make event channel work with PV extension of HVM
Quoted 11 lines · Jeremy Fitzhardinge
I think you need to expand this comment to explain the consequences of this change on SMP bringup and timers. I still don't see why there are (or should be) any differences from PV cpu bringup, aside from the tiny details of syscall setup (and MTRR I guess, but that's all a noop anyway, right?).
Also, I think you can probably do the timer changes in a separate patch, as even with evtchns, an emulated timer device interrupt will still be delivered properly via VIRQ_EMUL_PIN_*.
One of the problems is that some VCPU initialization hypercalls at the moment are PV specific and don't work with HVM guests, in particular I find that VCPUOP_initialise doesn't work correctly for me.
But the timers shouldn't be an issue because you should be able to receive the VIRQ_TIMER or the emulated timer interrupt anyway.
Re: [PATCH 7/7] xen: Make event channel work with PV extension of HVM
In reply to Jeremy FitzhardingeQuoted 7 lines · On Friday 05 March 2010 02:24:24 Jeremy Fitzhardinge wrote:
Quoted 4 lines · Sheng Yang
We mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used
X86_PLATFORM_IPI_VECTORas the noficiation vector for hypervisor"notification"
Oops...
Quoted 5 lines
Quoted 5 lines
to notify guest about the event.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Quoted 8 lines
Quoted 2 lines
Then we don't use IOAPIC/LAPIC, eliminated the overhead brought by unnecessary VMExit caused by LAPIC.
I think you need to expand this comment to explain the consequences of this change on SMP bringup and timers. I still don't see why there are (or should be) any differences from PV cpu bringup, aside from the tiny details of syscall setup (and MTRR I guess, but that's all a noop anyway, right?).
We follow the HVM vcpu brought up process. We don't need to initialize all kinds of registers. I think leave it to existed hypervisor is the better idea.
Quoted 3 lines
Also, I think you can probably do the timer changes in a separate patch, as even with evtchns, an emulated timer device interrupt will still be delivered properly via VIRQ_EMUL_PIN_*.
I think as long as we have evtchn, PV timer shouldn't be a problem. And for evtchn, because we don't have lapic now, we can't use lapic timer... So seems PV timer is a requirement for us.
Quoted 233 lines
Quoted 233 lines
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/enlighten.c | 6 +- arch/x86/xen/hvmpv.c | 68 +++++++++++++++++++++++++++++++- arch/x86/xen/irq.c | 28 +++++++++++++ arch/x86/xen/smp.c | 92 ++++++++++++++++++++++++++++++++++++++++-- arch/x86/xen/xen-ops.h | 14 +++++++ drivers/xen/events.c | 74 +++++++++++++++++++++++++++++++--- include/xen/events.h | 6 +++ include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- include/xen/xen.h | 2 + 10 files changed, 285 insertions(+), 16 deletions(-)arch/x86/xen/enlighten.c +3
-3@@ -717,7 +717,7 @@ static u32 xen_safe_apic_wait_icr_idle(void)return 0 ; } -static void set_xen_basic_apic_ops (void ) +void xen_set_basic_apic_ops (void ) { apic->read = xen_apic_read; apic->write = xen_apic_write;@@ -1026,7 +1026,7 @@ static void xen_crash_shutdown(struct pt_regs *regs) xen_reboot(SHUTDOWN_crash); } -static const struct machine_ops __initdata xen_machine_ops = { +const struct machine_ops __initdata xen_machine_ops = { .restart = xen_restart, .halt = xen_machine_halt, .power_off = xen_machine_halt,@@ -1116,7 +1116,7 @@ asmlinkage void __init xen_start_kernel(void)/* * set up the basic apic ops. */ -set_xen_basic_apic_ops (); +xen_set_basic_apic_ops ();#endif if (xen_feature (XENFEAT_mmu_pt_update_preserve_ad)) {arch/x86/xen/hvmpv.c +58
-1@@ -17,6 +17,7 @@#include <xen/interface/version.h> #include <xen/interface/memory.h> +#include <asm/reboot.h> #include <asm/xen/cpuid.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> @@ -44,6 +45,8 @@ static void __init xen_hvm_pv_banner(void)printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion);printk (KERN_INFO "PV feature: PV clocksource enabled\n" ); +if (xen_hvm_pv_evtchn_enabled ()) +printk (KERN_INFO "PV feature: Event channel enabled\n" ); }static int __init xen_para_available (void )@@ -83,6 +86,9 @@ static int __init init_hvm_pv_info(void)if (!(edx&XEN_CPUID_FEAT2_HVM_PV ))return -ENODEV ; +if (edx&XEN_CPUID_FEAT2_HVM_PV_EVTCHN ) + xen_hvm_pv_features |=XEN_HVM_PV_EVTCHN_ENABLED ; +/* We only support 1 page of hypercall for now */ if (pages !=1 )return -ENOMEM ;@@ -131,12 +137,42 @@ static void __init init_pv_clocksource(void) x86_platform.get_wallclock = xen_get_wallclock; x86_platform.set_wallclock = xen_set_wallclock; -clocksource_register (&xen_clocksource); +/* It would be done in xen_time_init() if evtchn enabled */ +if (!xen_hvm_pv_evtchn_enabled ()) +clocksource_register (&xen_clocksource); +} + +static int set_callback_via (uint64_t via) +{ +struct xen_hvm_param a; + + a.domid =DOMID_SELF ; + a.index =HVM_PARAM_CALLBACK_IRQ ; + a.value = via; +return HYPERVISOR_hvm_op (HVMOP_set_param,&a); } +void do_hvm_pv_evtchn_intr (void ) +{ +per_cpu (irq_count,smp_processor_id ())++; +xen_hvm_evtchn_do_upcall (get_irq_regs ()); +per_cpu (irq_count,smp_processor_id ())--; +} + +#ifdef CONFIG_X86_LOCAL_APIC +static void xen_hvm_pv_evtchn_apic_write (u32 reg,u32 val) +{ +/* The only one reached here should be EOI */ +WARN_ON (reg !=APIC_EOI ); +} +#endif + +extern struct machine_ops xen_machine_ops; +void __init xen_guest_init (void ) {int r; +uint64_t callback_via;/* Ensure the we won't confused with others */ if (xen_domain_type !=XEN_NATIVE )@@ -150,4 +186,34 @@ void __init xen_guest_init(void)/* PV clocksource would be enabled by default */ init_pv_clocksource (); + +if (xen_hvm_pv_evtchn_enabled ()) { +if (enable_hvm_pv (HVM_PV_EVTCHN )) +return ; + +xen_hvm_pv_init_irq_ops (); + + x86_init.timers.timer_init = xen_time_init; + x86_init.timers.setup_percpu_clockev = x86_init_noop; + x86_cpuinit.setup_percpu_clockev = x86_init_noop; + + pv_apic_ops.startup_ipi_hook = paravirt_nop; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +xen_set_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write; +#endif + + callback_via =
HVM_CALLBACK_VECTOR(X86_PLATFORM_IPI_VECTOR); +set_callback_via(callback_via);arch/x86/xen/hvmpv.c +8
-0+ + x86_platform_ipi_callback = do_hvm_pv_evtchn_intr; + +disable_acpi (); + +xen_hvm_pv_smp_init (); + machine_ops = xen_machine_ops; + } }arch/x86/xen/irq.c +28
-0@@ -2,6 +2,7 @@#include <asm/x86_init.h> +#include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h> @@ -131,3 +132,30 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +#ifdef CONFIG_XEN_HVM_PV +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); + +void __init xen_hvm_pv_init_irq_ops (void ) +{ +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops.irq_disable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable); + pv_irq_ops.irq_enable = +PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable); + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + } +} +#endif arch/x86/xen/smp.c +6
-0@@ -15,18 +15,24 @@#include <linux/sched.h> #include <linux/err.h> #include <linux/smp.h> +#include <linux/nmi.h> #include <asm/paravirt.h> #include <asm/desc.h> #include <asm/pgtable.h> #include <asm/cpu.h> +#include <asm/trampoline.h> +#include <asm/tlbflush.h> +#include <asm/mtrr.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <asm/xen/hypervisor.h> +#include <xen/xen.h> #include <xen/page.h> #include <xen/events.h> @@ -63,8 +69,13 @@ static __cpuinit void cpu_bringup(void)touch_softlockup_watchdog ();preempt_disable ();
Quoted 9 lines
arch/x86/xen/smp.c static __cpuinit void cpu_bringup(void) · line 78
Quoted 9 lines
arch/x86/xen/smp.c +7
-2-xen_enable_sysenter (); -xen_enable_syscall (); +if (xen_pv_domain ()) { +xen_enable_sysenter (); +xen_enable_syscall (); + }else if (xen_hvm_pv_evtchn_enabled ()) +set_mtrr_aps_delayed_init (); +else +BUG ();
Found set_mtrr_aps_delayed_init() is harmless for PV(anyway it wouldn't get the set_mtrr() when calling smp_store_cpu_info()), so make it less specific.
Quoted 29 lines
Quoted 14 lines
arch/x86/xen/smp.c +2
-1cpu =smp_processor_id ();smp_store_cpu_info (cpu);@@ -171,7 +182,8 @@ static void __init xen_smp_prepare_boot_cpu(void)/* We've switched to the "real" per-cpu gdt, so make sure the old memory can be recycled */ -make_lowmem_page_readwrite (xen_initial_gdt); +if (xen_feature (XENFEAT_writable_descriptor_tables)) +make_lowmem_page_readwrite (xen_initial_gdt);xen_setup_vcpu_info_placement (); }arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct · line 297
Quoted 4 lines
arch/x86/xen/smp.c +1
-0@@ -282,6 +294,47 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) return 0; } +#ifdef CONFIG_XEN_HVM_PV Rather than #ifdeffing...
arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle) · line 302
Quoted 5 lines
arch/x86/xen/smp.c +4
-0+static __cpuinit int +hvm_pv_cpu_initialize_context (unsigned int cpu,struct task_struct *idle) +{ +struct vcpu_guest_context *ctxt; +unsigned long start_ip;Just put
Quoted 5 lines
if (!xen_hvm_pv_evtchn_enabled()) return -EOPNOTSUPP; /* or something */and make sure
xen_hvm_pv_evtchn_enable()returns constant 0 in the !CONFIG_XEN_HVM_PV case.
OK. (In fact I've done that, but forgot to merge with the former patch.)
Quoted 8 lines
Quoted 8 lines
arch/x86/xen/smp.c +8
-0+ +if (cpumask_test_and_set_cpu (cpu, xen_cpu_initialized_map)) +return 0 ; + + ctxt =kzalloc (sizeof (*ctxt),GFP_KERNEL ); +if (ctxt ==NULL ) +return -ENOMEM ; +
Quoted 8 lines
arch/x86/xen/smp.c cpu_initialize_context(unsigned int cpu, struct task_struct *idle) · line 316
Quoted 6 lines
arch/x86/xen/smp.c +6
-0+ early_gdt_descr.address = (unsigned long )get_cpu_gdt_table (cpu); + initial_code = (unsigned long )cpu_bringup_and_idle; + stack_start.sp = (void *) idle->thread.sp; + +/* start_ip had better be page-aligned! */ + start_ip =setup_trampoline ();Why go via trampoline?
We are still following the standard HVM brought up process. VMX/SVM would get the registers right.
Quoted 25 lines
Quoted 25 lines
arch/x86/xen/smp.c +6
-0+ +/* only start_ip is what we want */ + ctxt->flags =VGCF_HVM_GUEST ; + ctxt->user_regs.eip = start_ip; + + printk(KERN_INFO "Booting processor %d ip 0x%lx\n", cpu,
start_ip); +arch/x86/xen/smp.c +14
-0+if (HYPERVISOR_vcpu_op (VCPUOP_initialise, cpu, ctxt)) +BUG (); + +kfree (ctxt); + return 0; +} +#else +static int hvm_pv_cpu_initialize_context (unsigned int cpu, +struct task_struct *idle) +{ +return 0 ; +} +#endif +static int __cpuinit xen_cpu_up (unsigned int cpu) {struct task_struct *idle =idle_task (cpu);@@ -292,11 +345,17 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
Quoted 8 lines
arch/x86/xen/smp.c static int __cpuinit xen_cpu_up(unsigned int cpu) · line 350
Quoted 6 lines
arch/x86/xen/smp.c +3
-0irq_ctx_init (cpu);#else clear_tsk_thread_flag (idle,TIF_FORK ); + +if (xen_hvm_pv_evtchn_enabled ()) + initial_gs =per_cpu_offset (cpu);Why does this need to be conditional?
Seems it won't affect PV... Would update it.
Quoted 6 lines
Quoted 6 lines
arch/x86/xen/smp.c +1
-1+per_cpu (kernel_stack, cpu) = (unsigned long )task_stack_page (idle) -KERNEL_STACK_OFFSET +THREAD_SIZE ;#endif -xen_setup_runstate_info (cpu);
Quoted 8 lines
arch/x86/xen/smp.c static int __cpuinit xen_cpu_up(unsigned int cpu) · line 358
Quoted 3 lines
arch/x86/xen/smp.c +3
-0+if (xen_pv_domain ()) +xen_setup_runstate_info (cpu); +Does HVM not support this? If
VCPUOP_register_runstate_memory_areadoesn't work on HVM, then just changexen_setup_runstate_info()to not BUG on hypercall failure (and presumably nothing tries to userunstate_info).
Found HVM support it...
Signature · 2 lines
regards Yang, Sheng
Quoted 256 lines
Quoted 255 lines
arch/x86/xen/smp.c +11
-1xen_setup_timer (cpu);xen_init_lock_cpu (cpu);@@ -305,7 +364,13 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)/* make sure interrupts start blocked */ per_cpu (xen_vcpu, cpu)->evtchn_upcall_mask =1 ; - rc =cpu_initialize_context (cpu, idle); +if (xen_pv_domain ()) + rc =cpu_initialize_context (cpu, idle); +else if (xen_hvm_pv_evtchn_enabled ()) + rc =hvm_pv_cpu_initialize_context (cpu, idle); +else +BUG (); +if (rc)return rc;@@ -480,3 +545,20 @@ void __init xen_smp_init(void)xen_fill_possible_map ();xen_init_spinlocks (); } + +#ifdef CONFIG_XEN_HVM_PV +static void xen_hvm_pv_flush_tlb_others (const struct cpumask *cpumask, + struct mm_struct *mm, unsignedlong va) +{
arch/x86/xen/smp.c +9
-0+/* TODO Make it more specific */ +flush_tlb_all (); +} + +void __init xen_hvm_pv_smp_init (void ) +{ +if (xen_hvm_pv_evtchn_enabled ()) { + smp_ops = xen_smp_ops; + pv_mmu_ops.flush_tlb_others =
xen_hvm_pv_flush_tlb_others; + }arch/x86/xen/smp.c +2
-0+} +#endif arch/x86/xen/xen-ops.h +14
-0@@ -51,6 +51,12 @@ void __init xen_time_init(void);unsigned long xen_get_wallclock (void );int xen_set_wallclock (unsigned long time);unsigned long long xen_sched_clock (void ); +void xen_set_basic_apic_ops (void ); + +#ifdef CONFIG_XEN_HVM_PV +void __init xen_hvm_pv_init_irq_ops (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void ); +#endif /* CONFIG_XEN_HVM_PV */ irqreturn_t xen_debug_interrupt (int irq,void *dev_id);@@ -61,9 +67,17 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_pv_smp_init (void ); +#endif /* CONFIG_XEN_HVM_PV */ +extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +#ifdef CONFIG_XEN_HVM_PV +static inline void xen_hvm_pv_smp_init (void ) {} +#endif /* CONFIG_XEN_HVM_PV */ +#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS drivers/xen/events.c +3
-0@@ -32,14 +32,17 @@#include <asm/irq.h> #include <asm/idle.h> #include <asm/sync_bitops.h> +#include <asm/desc.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> +#include <xen/xen.h> #include <xen/xen-ops.h> #include <xen/events.h> #include <xen/interface/xen.h> #include <xen/interface/event_channel.h> +/* * This lock protects updates to the following mapping and reference-count * arrays. The lock does not need to be acquired to read the mapping tables. @@ -616,17 +619,13 @@ static
DEFINE_PER_CPU(unsigned, xed_nesting_count); * a bitset of words which contain pending event bits.
The second * level is a bitset of pending events themselves.
*/drivers/xen/events.c +63
-6-void xen_evtchn_do_upcall (struct pt_regs *regs) +void __xen_evtchn_do_upcall (struct pt_regs *regs) {int cpu =get_cpu (); -struct pt_regs *old_regs =set_irq_regs (regs);struct shared_info *s = HYPERVISOR_shared_info;struct vcpu_info *vcpu_info =__get_cpu_var (xen_vcpu);unsigned count; -exit_idle (); -irq_enter (); - do { unsigned long pending_words;@@ -662,10 +661,25 @@ void xen_evtchn_do_upcall(struct pt_regs *regs) }while (count !=1 ); out: +put_cpu (); +} + +void xen_evtchn_do_upcall (struct pt_regs *regs) +{ +struct pt_regs *old_regs =set_irq_regs (regs); + +exit_idle (); +irq_enter (); + +__xen_evtchn_do_upcall (regs); +irq_exit ();set_irq_regs (old_regs); +} -put_cpu (); +void xen_hvm_evtchn_do_upcall (struct pt_regs *regs) +{ +__xen_evtchn_do_upcall (regs); }/* Rebind a new event channel to an existing irq. */ @@ -944,3 +958,51 @@ void __init xen_init_IRQ(void)irq_ctx_init (smp_processor_id ()); } + +void __init xen_hvm_pv_evtchn_init_IRQ (void ) +{ +int i; + +xen_init_IRQ (); +for (i =0 ; i<NR_IRQS_LEGACY ; i++) { +struct evtchn_bind_virq bind_virq; +struct irq_desc *desc =irq_to_desc (i); +int virq, evtchn; + + virq = i +VIRQ_EMUL_PIN_START ; + bind_virq.virq = virq; + bind_virq.vcpu =0 ; + +if (HYPERVISOR_event_channel_op (EVTCHNOP_bind_virq, +&bind_virq) !=0 ) +BUG (); + + evtchn = bind_virq.port; + evtchn_to_irq[evtchn] = i; + irq_info[i] =mk_virq_info (evtchn, virq); + + desc->status =IRQ_DISABLED ; + desc->action =NULL ; + desc->depth =1 ; + +/* +* 16 old-style INTA-cycle interrupts: +*/ +set_irq_chip_and_handler_name (i,&xen_dynamic_chip, + handle_level_irq,"event" ); + } + +/* +* Cover the whole vector space, no vector can escape +* us. (some of these will be overridden and become +* 'special' SMP interrupts) +*/ +for (i =0 ; i< (NR_VECTORS -FIRST_EXTERNAL_VECTOR ); i++) { +int vector =FIRST_EXTERNAL_VECTOR + i; +if (vector !=IA32_SYSCALL_VECTOR ) +set_intr_gate (vector, interrupt[i]); + } + +/* generic IPI for platform specific use, now used for HVM evtchn */ +
alloc_intr_gate(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi); +}include/xen/events.h +6
-0@@ -56,4 +56,10 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs); + +#ifdef CONFIG_XEN_HVM_PV +void xen_hvm_evtchn_do_upcall (struct pt_regs *regs); +#endif +#endif /* _XEN_EVENTS_H */ include/xen/hvm.h +3
-0@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx)return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x)(((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ +
HVM_CALLBACK_VIA_TYPE_SHIFT| (x)) +
#endif /*XEN_HVM_H__*/include/xen/interface/xen.h +5
-1@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS (VIRQ_EMUL_PIN_START + VIRQ_EMUL_PIN_NUM) +/* * MMU-UPDATE REQUESTS * include/xen/xen.h +2
-1@@ -26,6 +26,8 @@ extern u32 xen_hvm_pv_features;#define xen_hvm_pv_evtchn_enabled () \ (xen_hvm_pv_features& XEN_HVM_PV_EVTCHN_ENABLED) +#else +#define xen_hvm_pv_evtchn_enabled () 0#endif /* CONFIG_XEN_HVM_PV */ #ifdef CONFIG_XEN_DOM0 --1.5.4.5