PV extension of HVM (Hybrid) for Xen
Subsystems (8)
PATCH v4 1/7 xen: add support for hvm_op
In reply to Sheng YangFrom: Jeremy Fitzhardinge <[email protected]>
Add support for hvm_op hypercall.
--- arch/x86/include/asm/xen/hypercall.h | 6 ++ include/xen/hvm.h | 23 +++++++ include/xen/interface/hvm/hvm_op.h | 72 ++++++++++++++++++++++ include/xen/interface/hvm/params.h | 111 ++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 0 deletions(-) create mode 100644 include/xen/hvm.h create mode 100644 include/xen/interface/hvm/hvm_op.h create mode 100644 include/xen/interface/hvm/params.h
arch/x86/include/asm/xen/hypercall.h +6 -0
@@ -417,6 +417,12 @@ HYPERVISOR_nmi_op(unsigned long op, unsigned long arg)return _hypercall2 (int, nmi_op, op, arg); } +static inline unsigned long __must_check +HYPERVISOR_hvm_op (int op,void *arg) +{ + return _hypercall2(unsigned long, hvm_op, op, arg); +} +static inline void MULTI_fpu_taskswitch (struct multicall_entry *mcl,int set) {
include/xen/hvm.h new file +23 -0
@@ -0,0 +1,23 @@ +/* Simple wrappers around HVM functions */ +#ifndef XEN_HVM_H__ +#define XEN_HVM_H__ + +#include <xen/interface/hvm/params.h> + +static inline unsigned long hvm_get_parameter (int idx) +{ +struct xen_hvm_param xhv; +int r; + + xhv.domid =DOMID_SELF ; + xhv.index = idx; + r =HYPERVISOR_hvm_op (HVMOP_get_param, &xhv); +if (r <0 ) { +printk (KERN_ERR "cannot get hvm parameter %d: %d.\n" , + idx, r); +return 0 ; + } +return xhv.value; +} + +#endif /* XEN_HVM_H__ */
include/xen/interface/hvm/hvm_op.h new file +72 -0
72 lines not on this page · raw message
include/xen/interface/hvm/params.h new file +111 -0
111 lines not on this page · raw message
Signature · 1 line
1.5.4.5
PATCH v4 6/7 xen: Unified checking for Xen of PV drivers to xenbus_register_frontend()
In reply to Sheng Yang--- drivers/block/xen-blkfront.c | 3 --- drivers/input/xen-kbdfront.c | 3 --- drivers/net/xen-netfront.c | 5 ----- drivers/video/xen-fbfront.c | 3 --- include/xen/xenbus.h | 3 +++ 5 files changed, 3 insertions(+), 14 deletions(-)
drivers/block/xen-blkfront.c +0 -3
@@ -1068,9 +1068,6 @@ static struct xenbus_driver blkfront = {static int __init xlblk_init (void ) { -if (!xen_domain ()) -return -ENODEV ; -if (register_blkdev (XENVBD_MAJOR ,DEV_NAME )) {printk (KERN_WARNING "xen_blk: can't get major %d with name %s\n" ,XENVBD_MAJOR ,DEV_NAME );
drivers/input/xen-kbdfront.c +0 -3
@@ -338,9 +338,6 @@ static struct xenbus_driver xenkbd_driver = {static int __init xenkbd_init (void ) { -if (!xen_domain ()) -return -ENODEV ; -/* Nothing to do if running in dom0. */ if (xen_initial_domain ())return -ENODEV ;
drivers/net/xen-netfront.c +0 -5
@@ -1804,14 +1804,9 @@ static struct xenbus_driver netfront_driver = {static int __init netif_init (void ) { -if (!xen_domain ()) -return -ENODEV ; -if (xen_initial_domain ())return 0 ; -printk (KERN_INFO "Initialising Xen virtual ethernet driver.\n" ); -return xenbus_register_frontend (&netfront_driver); }module_init (netif_init);
drivers/video/xen-fbfront.c +0 -3
@@ -683,9 +683,6 @@ static struct xenbus_driver xenfb_driver = {static int __init xenfb_init (void ) { -if (!xen_domain ()) -return -ENODEV ; -/* Nothing to do if running in dom0. */ if (xen_initial_domain ())return -ENODEV ;
include/xen/xenbus.h +3 -0
@@ -43,6 +43,7 @@#include <xen/interface/grant_table.h> #include <xen/interface/io/xenbus.h> #include <xen/interface/io/xs_wire.h> +#include <xen/xen.h> /* Register callback to watch this node. */ struct xenbus_watch
@@ -112,6 +113,8 @@ static inline int __must_check xenbus_register_frontend(struct xenbus_driver *drv) {WARN_ON (drv->owner !=THIS_MODULE ); +if (!xen_domain ()) +return -ENODEV ;return __xenbus_register_frontend (drv,THIS_MODULE ,KBUILD_MODNAME ); }
Signature · 1 line
1.5.4.5
PATCH v4 2/7 xen: Import cpuid.h from Xen
In reply to Sheng YangFrom: Keir Fraser <[email protected]>
Which would be used by CPUID detection later
--- arch/x86/include/asm/xen/cpuid.h | 68 ++++++++++++++++++++++++++++++++++++++ 1 files changed, 68 insertions(+), 0 deletions(-) create mode 100644 arch/x86/include/asm/xen/cpuid.h
arch/x86/include/asm/xen/cpuid.h new file +68 -0
@@ -0,0 +1,68 @@ +/****************************************************************************** +* arch/include/asm/xen/cpuid.h +* +* CPUID interface to Xen. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +* Copyright (c) 2007 Citrix Systems, Inc. +* +* Authors: +* Keir Fraser <[email protected]> +*/ + +#ifndef __ASM_X86_XEN_CPUID_H__ +#define __ASM_X86_XEN_CPUID_H__ + +/* Xen identification leaves start at 0x40000000. */ +#define XEN_CPUID_FIRST_LEAF 0x40000000 +#define XEN_CPUID_LEAF (i) (XEN_CPUID_FIRST_LEAF + (i)) + +/* +* Leaf 1 (0x40000000) +* EAX: Largest Xen-information leaf. All leaves up to an including @EAX +* are supported by the Xen host. +* EBX-EDX: "XenVMMXenVMM" signature, allowing positive identification +* of a Xen host. +*/ +#define XEN_CPUID_SIGNATURE_EBX 0x566e6558/* "XenV" */ +#define XEN_CPUID_SIGNATURE_ECX 0x65584d4d/* "MMXe" */ +#define XEN_CPUID_SIGNATURE_EDX 0x4d4d566e/* "nVMM" */ + +/* +* Leaf 2 (0x40000001) +* EAX[31:16]: Xen major version. +* EAX[15: 0]: Xen minor version. +* EBX-EDX: Reserved (currently all zeroes). +*/ + +/* +* Leaf 3 (0x40000002) +* EAX: Number of hypercall transfer pages. This register is always guaranteed +* to specify one hypercall page. +* EBX: Base address of Xen-specific MSRs. +* ECX: Features 1. Unused bits are set to zero. +* EDX: Features 2. Unused bits are set to zero. +*/ + +/* Does the host support MMU_PT_UPDATE_PRESERVE_AD for this guest? */ +#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0 +#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) + +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
Signature · 1 line
1.5.4.5
PATCH v4 4/7 xen: The entrance for PV extension of HVM
In reply to Sheng Yangxen_guest_init() would setup the environment.
--- arch/x86/kernel/setup.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-)
arch/x86/kernel/setup.c +8 -0
@@ -113,6 +113,10 @@#endif #include <asm/mce.h> +#ifdef CONFIG_XEN +#include <xen/xen.h> +#endif +/* * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. * The direct mapping extends to max_pfn_mapped, so that we can directly access
@@ -740,6 +744,10 @@ void __init setup_arch(char **cmdline_p) x86_init.oem.arch_setup (); +#ifdef CONFIG_XEN +xen_guest_init (); +#endif +setup_memory_map ();parse_setup_data ();/* update the e820_saved too */
Signature · 1 line
1.5.4.5
PATCH v4 7/7 xen: Enable grant table and xenbus for PV extension of HVM
In reply to Sheng YangNow pv drivers(vnif, vbd) can work.
--- arch/x86/xen/enlighten.c | 1 + drivers/input/xen-kbdfront.c | 3 ++ drivers/video/xen-fbfront.c | 3 ++ drivers/xen/grant-table.c | 59 +++++++++++++++++++++++++++++++++++-- drivers/xen/xenbus/xenbus_probe.c | 22 ++++++++++--- include/xen/xen.h | 2 + include/xen/xenbus.h | 2 +- 7 files changed, 83 insertions(+), 9 deletions(-)
arch/x86/xen/enlighten.c +1 -0
@@ -1277,6 +1277,7 @@ static int init_hvm_pv_info(void) pv_info = xen_info; pv_info.kernel_rpl =0 ; + xen_domain_type =XEN_HVM_DOMAIN ;return 0 ; }
drivers/input/xen-kbdfront.c +3 -0
@@ -342,6 +342,9 @@ static int __init xenkbd_init(void)if (xen_initial_domain ())return -ENODEV ; +if (xen_hvm_domain ()) +return -ENODEV ; +return xenbus_register_frontend (&xenkbd_driver); }
drivers/video/xen-fbfront.c +3 -0
@@ -687,6 +687,9 @@ static int __init xenfb_init(void)if (xen_initial_domain ())return -ENODEV ; +if (xen_hvm_domain ()) +return -ENODEV ; +return xenbus_register_frontend (&xenfb_driver); }
drivers/xen/grant-table.c +56 -3
92 lines not on this page · raw message
drivers/xen/xenbus/xenbus_probe.c +17 -5
@@ -55,6 +55,8 @@#include <xen/events.h> #include <xen/page.h> +#include <xen/hvm.h> +#include "xenbus_comms.h" #include "xenbus_probe.h"
@@ -786,7 +788,8 @@ static int __init xenbus_probe_init(void)DPRINTK ("" ); err = -ENODEV ; -if (!xen_domain ()) + +if (!xen_evtchn_enabled ()) goto out_error;/* Register ourselves with the kernel bus subsystem */
@@ -805,10 +808,19 @@ static int __init xenbus_probe_init(void)/* dom0 not yet supported */ }else { xenstored_ready =1 ; - xen_store_evtchn = xen_start_info->store_evtchn; - xen_store_mfn = xen_start_info->store_mfn; +if (xen_hvm_pv_evtchn_enabled ()) { + xen_store_evtchn = +hvm_get_parameter (HVM_PARAM_STORE_EVTCHN ); + xen_store_mfn = +hvm_get_parameter (HVM_PARAM_STORE_PFN ); + xen_store_interface = +ioremap (xen_store_mfn <<PAGE_SHIFT ,PAGE_SIZE ); + }else { + xen_store_evtchn = xen_start_info->store_evtchn; + xen_store_mfn = xen_start_info->store_mfn; + xen_store_interface =mfn_to_virt (xen_store_mfn); + } } - xen_store_interface =mfn_to_virt (xen_store_mfn);/* Initialize the interface to xenstore. */ err =xs_init ();
@@ -922,7 +934,7 @@ static void wait_for_devices(struct xenbus_driver *xendrv)struct device_driver *drv = xendrv ? &xendrv->driver :NULL ;unsigned int seconds_waited =0 ; -if (!ready_to_wait_for_devices || !xen_domain ()) +if (!ready_to_wait_for_devices || !xen_evtchn_enabled ())return ;while (exists_connecting_device (drv)) {
include/xen/xen.h +2 -0
@@ -28,6 +28,8 @@ extern u32 xen_hvm_pv_status;#define xen_hvm_pv_evtchn_enabled () (xen_hvm_pv_enabled() && \ (xen_hvm_pv_status & XEN_HVM_PV_EVTCHN_ENABLED)) +#define xen_evtchn_enabled () (xen_pv_domain() || xen_hvm_pv_evtchn_enabled()) +#ifdef CONFIG_XEN_DOM0 #include <xen/interface/xen.h> #include <asm/xen/hypervisor.h>
include/xen/xenbus.h +1 -1
@@ -113,7 +113,7 @@ static inline int __must_check xenbus_register_frontend(struct xenbus_driver *drv) {WARN_ON (drv->owner !=THIS_MODULE ); -if (!xen_domain ()) +if (!xen_evtchn_enabled ())return -ENODEV ;return __xenbus_register_frontend (drv,THIS_MODULE ,KBUILD_MODNAME ); }
Signature · 1 line
1.5.4.5
v4 cover PV extension of HVM (Hybrid) for Xen
Hi, Jeremy & Ian
Here is the forth version of patchset to enable PV extension of HVM support in Linux kernel of Xen.
I'd like to have your comments on this patch series. We think the patches are prepared for check in pv_ops domU tree.
The PV extension of HVM is started from real mode like HVM guest, but also with a a range of PV features(e.g. PV halt, PV timer, event channel, as well as PV drivers). So guest with this feature can takes the advantages of both H/W virtualization and Para-Virtualization.
The first two of the patchset imported several header file from Jeremy's tree and Xen tree, respect to Jeremy and Keir's works.
The whole patchset based on Linux upstream.
You need a line like:
cpuid = [ '0x40000002:edx=0x3' ]
in HVM configuration file to expose hybrid feature to guest, and
CONFIG_XEN
in the guest kernel configuration file to enable the hybrid support.
And the compiled image can be used as native/pv domU/hvm guest/pv feature hvm kernel.
Current the patchset support x86_64 only.
I've discussed the vector-evtchn mapping with Stefano, and we agreed on that it's more complex than we thought, and can't brought much benefit for pvops domU. So there is no major change in this update.
Change from v3:
1. Rebase to Linux 2.6.33 release.
2. change the name to "PV extension of HVM"
3. Some minor coding polishing.
Change from v2:
1. change the name "hybrid" to "PV featured HVM".
2. Unified the PV driver's judgement of xen_domain() to xen_evtchn_enabled().
3. Move the function(evtchn) initialize hypercall near the real enabling place, rather than a unified place before function enabled.
4. Remove the reserved E820 region for grant table. Use QEmu Xen platform device's MMIO instead.
The major change from v1:
1. SMP support.
2. Modify the entrance point to avoid most of genernic kernel modification.
3. Binding PV timer with event channel mechanism.
--
regards
Yang, Sheng
arch/x86/include/asm/xen/cpuid.h | 73 +++++++++++++ arch/x86/include/asm/xen/hypercall.h | 6 + arch/x86/kernel/setup.c | 8 ++ arch/x86/xen/enlighten.c | 188 ++++++++++++++++++++++++++++++++++ arch/x86/xen/irq.c | 54 ++++++++++ arch/x86/xen/smp.c | 144 ++++++++++++++++++++++++++- arch/x86/xen/xen-head.S | 6 + arch/x86/xen/xen-ops.h | 4 + drivers/block/xen-blkfront.c | 3 - drivers/input/xen-kbdfront.c | 6 +- drivers/net/xen-netfront.c | 5 - drivers/video/xen-fbfront.c | 6 +- drivers/xen/events.c | 66 +++++++++++- drivers/xen/grant-table.c | 59 ++++++++++- drivers/xen/xenbus/xenbus_probe.c | 22 +++- include/xen/events.h | 1 + include/xen/hvm.h | 28 +++++ include/xen/interface/hvm/hvm_op.h | 79 ++++++++++++++ include/xen/interface/hvm/params.h | 111 ++++++++++++++++++++ include/xen/interface/xen.h | 6 +- include/xen/xen.h | 11 ++ include/xen/xenbus.h | 3 + 22 files changed, 861 insertions(+), 28 deletions(-)
PATCH v4 3/7 xen/hvm: Xen PV extension of HVM initialization
The PV extension of HVM(once known as Hybrid) is started from real mode like HVM guest, but also with a component based PV feature selection(e.g. PV halt, PV timer, event channel, then PV drivers). So guest can takes the advantages of both H/W virtualization and Para-Virtualization.
This patch introduced the PV extension of HVM guest initialization.
Guest would detect the capability using CPUID 0x40000002.edx, then call HVMOP_enable_pv hypercall to enable pv support in hypervisor.
--- arch/x86/include/asm/xen/cpuid.h | 5 ++ arch/x86/xen/enlighten.c | 115 ++++++++++++++++++++++++++++++++++++ arch/x86/xen/irq.c | 21 +++++++ arch/x86/xen/xen-head.S | 6 ++ arch/x86/xen/xen-ops.h | 1 + include/xen/interface/hvm/hvm_op.h | 7 ++ include/xen/xen.h | 9 +++ 7 files changed, 164 insertions(+), 0 deletions(-)
arch/x86/include/asm/xen/cpuid.h +5 -0
@@ -65,4 +65,9 @@#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV 0 +#define XEN_CPUID_FEAT2_HVM_PV (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV_EVTCHN 1 +#define XEN_CPUID_FEAT2_HVM_PV_EVTCHN (1u<<1) +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
arch/x86/xen/enlighten.c +115 -0
130 lines not on this page · raw message
arch/x86/xen/irq.c +21 -0
@@ -131,3 +131,24 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +static void xen_hvm_pv_safe_halt (void ) +{ +/* Do local_irq_enable() explicitly in hvm_pv guest */ +local_irq_enable (); +xen_safe_halt (); +} + +static void xen_hvm_pv_halt (void ) +{ +if (irqs_disabled ()) +HYPERVISOR_vcpu_op (VCPUOP_down,smp_processor_id (),NULL ); +else +xen_hvm_pv_safe_halt (); +} + +void __init xen_hvm_pv_init_irq_ops (void ) +{ + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; +}
arch/x86/xen/xen-head.S +6 -0
@@ -33,6 +33,12 @@ ENTRY(hypercall_page) .skip PAGE_SIZE_asm .popsection +.pushsection .data + .align PAGE_SIZE_asm +ENTRY(shared_info_page) + .skip PAGE_SIZE_asm +.popsection + ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz "2.6") ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz "xen-3.0")
arch/x86/xen/xen-ops.h +1 -0
@@ -41,6 +41,7 @@ void xen_vcpu_restore(void);void __init xen_build_dynamic_phys_to_machine (void );void xen_init_irq_ops (void ); +void xen_hvm_pv_init_irq_ops (void );void xen_setup_timer (int cpu);void xen_setup_runstate_info (int cpu);void xen_teardown_timer (int cpu);
include/xen/interface/hvm/hvm_op.h +7 -0
@@ -69,4 +69,11 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_set_pci_link_route);/* Flushes all VCPU TLBs: @arg must be NULL. */ #define HVMOP_flush_tlbs 5 +#define HVMOP_enable_pv 9 +struct xen_hvm_pv_type { +domid_t domid; +uint32_t flags; +#define HVM_PV_EVTCHN (1ull<<1) +}; +#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */
include/xen/xen.h +9 -0
@@ -9,6 +9,7 @@ enum xen_domain_type {#ifdef CONFIG_XEN extern enum xen_domain_type xen_domain_type; +extern void xen_guest_init (void );#else #define xen_domain_type XEN_NATIVE#endif
@@ -19,6 +20,14 @@ extern enum xen_domain_type xen_domain_type;#define xen_hvm_domain () (xen_domain() && \ xen_domain_type == XEN_HVM_DOMAIN) +#define XEN_HVM_PV_ENABLED (1u << 0) +#define XEN_HVM_PV_EVTCHN_ENABLED (1u << 1) +extern u32 xen_hvm_pv_status; + +#define xen_hvm_pv_enabled () (xen_hvm_pv_status & XEN_HVM_PV_ENABLED) +#define xen_hvm_pv_evtchn_enabled () (xen_hvm_pv_enabled() && \ + (xen_hvm_pv_status & XEN_HVM_PV_EVTCHN_ENABLED)) +#ifdef CONFIG_XEN_DOM0 #include <xen/interface/xen.h> #include <asm/xen/hypervisor.h>
Signature · 1 line
1.5.4.5
PATCH v4 5/7 xen: Make event channel work with PV extension of HVM
In reply to Sheng YangWe mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used X86_PLATFORM_IPI_VECTOR as the noficiation vector for hypervisor to notify guest about the event.
The Xen PV timer is used to provide guest a reliable timer.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Then we don't use IOAPIC/LAPIC.
--- arch/x86/xen/enlighten.c | 72 +++++++++++++++++++++ arch/x86/xen/irq.c | 37 ++++++++++- arch/x86/xen/smp.c | 144 ++++++++++++++++++++++++++++++++++++++++++- arch/x86/xen/xen-ops.h | 3 + drivers/xen/events.c | 66 ++++++++++++++++++- include/xen/events.h | 1 + include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- 8 files changed, 326 insertions(+), 8 deletions(-)
arch/x86/xen/enlighten.c +72 -0
@@ -58,6 +58,9 @@#include <asm/reboot.h> #include <asm/stackprotector.h> +#include <xen/hvm.h> +#include <xen/events.h> +#include "xen-ops.h" #include "mmu.h" #include "multicalls.h"
@@ -1212,6 +1215,8 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version >>16 , version &0xffff , extra.extraversion); +if (xen_hvm_pv_evtchn_enabled ()) +printk (KERN_INFO "PV feature: Event channel enabled\n" ); }static int xen_para_available (void )
@@ -1256,6 +1261,9 @@ static int init_hvm_pv_info(void) xen_hvm_pv_status =XEN_HVM_PV_ENABLED ; +if (edx &XEN_CPUID_FEAT2_HVM_PV_EVTCHN ) + xen_hvm_pv_status |=XEN_HVM_PV_EVTCHN_ENABLED ; +/* We only support 1 page of hypercall for now */ if (pages !=1 )return -ENOMEM ;
@@ -1292,12 +1300,42 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) = &HYPERVISOR_shared_info->vcpu_info[0 ]; } +static int set_callback_via (uint64_t via) +{ +struct xen_hvm_param a; + + a.domid =DOMID_SELF ; + a.index =HVM_PARAM_CALLBACK_IRQ ; + a.value = via; +return HYPERVISOR_hvm_op (HVMOP_set_param, &a); +} + +void do_hvm_pv_evtchn_intr (void ) +{ +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())++; +#endif +xen_evtchn_do_upcall (get_irq_regs ()); +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())--; +#endif +} + +#ifdef CONFIG_X86_LOCAL_APIC +static void xen_hvm_pv_evtchn_apic_write (u32 reg,u32 val) +{ +/* The only one reached here should be EOI */ +WARN_ON (reg !=APIC_EOI ); +} +#endif +void __init xen_guest_init (void ) {#ifdef CONFIG_X86_32 return ;#else int r; +uint64_t callback_via;/* Ensure the we won't confused with PV */ if (xen_domain_type ==XEN_PV_DOMAIN )
@@ -1310,6 +1348,40 @@ void __init xen_guest_init(void)init_shared_info ();xen_hvm_pv_init_irq_ops (); + +if (xen_hvm_pv_evtchn_enabled ()) { +if (enable_hvm_pv (HVM_PV_EVTCHN )) +return ; + + pv_time_ops = xen_time_ops; + + x86_init.timers.timer_init = xen_time_init; + x86_init.timers.setup_percpu_clockev = x86_init_noop; + x86_cpuinit.setup_percpu_clockev = x86_init_noop; + + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + + pv_apic_ops = xen_apic_ops; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +set_xen_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write; +#endif + + callback_via =HVM_CALLBACK_VECTOR (X86_PLATFORM_IPI_VECTOR ); +set_callback_via (callback_via); + + x86_platform_ipi_callback = do_hvm_pv_evtchn_intr; + +disable_acpi (); + +xen_hvm_pv_smp_init (); + machine_ops = xen_machine_ops; + }#endif }
arch/x86/xen/irq.c +35 -2
@@ -5,6 +5,7 @@#include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h> +#include <xen/xen.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h>
@@ -132,6 +133,20 @@ void __init xen_init_irq_ops() x86_init.irqs.intr_init = xen_init_IRQ; } +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); +static void xen_hvm_pv_safe_halt (void ) {/* Do local_irq_enable() explicitly in hvm_pv guest */
@@ -147,8 +162,26 @@ static void xen_hvm_pv_halt(void)xen_hvm_pv_safe_halt (); } +static const struct pv_irq_ops xen_hvm_pv_irq_ops__initdata = { + .save_fl =__PV_IS_CALLEE_SAVE (native_save_fl), + .restore_fl =__PV_IS_CALLEE_SAVE (native_restore_fl), + .irq_disable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable), + .irq_enable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable), + + .safe_halt = xen_hvm_pv_safe_halt, + .halt = xen_hvm_pv_halt, +#ifdef CONFIG_X86_64 + .adjust_exception_frame = paravirt_nop, +#endif +}; +void __init xen_hvm_pv_init_irq_ops (void ) { - pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; - pv_irq_ops.halt = xen_hvm_pv_halt; +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops = xen_hvm_pv_irq_ops; + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + }else { + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; + } }
arch/x86/xen/smp.c +143 -1
173 lines not on this page · raw message
arch/x86/xen/xen-ops.h +3 -0
@@ -34,6 +34,7 @@ void xen_reserve_top(void);char *__init xen_memory_setup (void );void __init xen_arch_setup (void );void __init xen_init_IRQ (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void );void xen_enable_sysenter (void );void xen_enable_syscall (void );void xen_vcpu_restore (void );
@@ -61,10 +62,12 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +void xen_hvm_pv_smp_init (void );extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +static inline void xen_hvm_pv_smp_init (void ) {}#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS
drivers/xen/events.c +62 -4
90 lines not on this page · raw message
include/xen/events.h +1 -0
@@ -56,4 +56,5 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs);#endif /* _XEN_EVENTS_H */
include/xen/hvm.h +5 -0
@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx) return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x) (((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ + HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +#endif /* XEN_HVM_H__ */
include/xen/interface/xen.h +5 -1
@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS 40 +/* * MMU-UPDATE REQUESTS *
Signature · 1 line
1.5.4.5
Re: [LKML] [PATCH 7/7] xen: Enable grant table and xenbus for PV extension of HVM
In reply to Sheng Yangdrivers/xen/grant-table.c static int gnttab_map(unsigned int start_idx, unsigned int end_idx) · line 501
Quoted 3 lines
drivers/xen/grant-table.c +3
-0+/* The region reserved by QEmu for Xen platform device */ +#define GNTTAB_START 0xf2000000ul +#define GNTTAB_SIZE 0x20000ul
I thought that in the earlier review you said:
"> > +#define GNTTAB_START 0xfbfe0000ul
Quoted 6 lines
Quoted 1 line
+#define
GNTTAB_SIZE0x20000ulIs it possible that there would be a PCI device that would be passed in the guest that would conflict with the above mentioned E820 region?
I would change them to a dedicated PCI MMIO address in the next version. Thanks.
"
?
Quoted 8 lines · Sheng Yang
I'd like to have your comments on this patch series. We think the patches are prepared for check in
pv_opsdomU tree.The PV extension of HVM is started from real mode like HVM guest, but also with a a range of PV features(e.g. PV halt, PV timer, event channel, as well as PV drivers). So guest with this feature can takes the advantages of both H/W virtualization and Para-Virtualization.
The PV device side of this has a lot of overlap with the older PV driver for HVM domains code. Aside perhaps for the details of how interrupts/events are actually delivered, that seems like a completely distinct piece of work which could be done separately.
Also, I'd like to see a clearer statement of what you're specifically trying to optimise for here. What is inefficient in an unmodified HVM domain, and how do your patches address these things. What measurements have you made?
Quoted 11 lines
The first two of the patchset imported several header file from Jeremy's tree and Xen tree, respect to Jeremy and Keir's works.
The whole patchset based on Linux upstream.
You need a line like:
cpuid = [ '0x40000002:edx=0x3' ]
in HVM configuration file to expose hybrid feature to guest, and
Quoted 8 lines
CONFIG_XENin the guest kernel configuration file to enable the hybrid support.
And the compiled image can be used as native/pv domU/hvm guest/pv feature hvm kernel.
Current the patchset support
x86_64only.
Please add a new config option for the new code, and make it dependent on x86-64, rather than scattering random #ifdef CONFIG_X86_64s around the code. Make sure all the code is controlled by this CONFIG variable so it disappears in a 32-bit build (ideally by putting it into a new file where possible, and only building it when enabled).
Quoted 7 lines
I've discussed the vector-evtchn mapping with Stefano, and we agreed on that it's more complex than we thought, and can't brought much benefit for pvops domU. So there is no major change in this update.
Change from v3:
1. Rebase to Linux 2.6.33 release.
Why 2.6.33? I'm currently working on 2.6.32; I intend to move to .33 shortly, but .32 is a much more convenient base (particularly if you're interested in getting these changes into any distro kernels).
I rebased it to .32 without much effort.
I'll make specific comments in the patches.
J
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Sheng YangQuoted 106 lines · Sheng Yang
The PV extension of HVM(once known as Hybrid) is started from real mode like HVM guest, but also with a component based PV feature selection(e.g. PV halt, PV timer, event channel, then PV drivers). So guest can takes the advantages of both H/W virtualization and Para-Virtualization.
This patch introduced the PV extension of HVM guest initialization.
Guest would detect the capability using CPUID 0x40000002.edx, then call
HVMOP_enable_pvhypercall to enable pv support in hypervisor.Signed-off-by Sheng Yang<[email protected]>Signed-off-by Yaozu (Eddie) Dong<[email protected]>--- arch/x86/include/asm/xen/cpuid.h | 5 ++ arch/x86/xen/enlighten.c | 115 ++++++++++++++++++++++++++++++++++++ arch/x86/xen/irq.c | 21 +++++++ arch/x86/xen/xen-head.S | 6 ++ arch/x86/xen/xen-ops.h | 1 + include/xen/interface/hvm/hvm_op.h | 7 ++ include/xen/xen.h | 9 +++ 7 files changed, 164 insertions(+), 0 deletions(-)arch/x86/include/asm/xen/cpuid.h +5
-0@@ -65,4 +65,9 @@#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV 0 +#define XEN_CPUID_FEAT2_HVM_PV (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV_EVTCHN 1 +#define XEN_CPUID_FEAT2_HVM_PV_EVTCHN (1u<<1) +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */ arch/x86/xen/enlighten.c +47
-0@@ -34,6 +34,8 @@#include <xen/interface/version.h> #include <xen/interface/physdev.h> #include <xen/interface/vcpu.h> +#include <xen/interface/memory.h> +#include <xen/interface/hvm/hvm_op.h> #include <xen/features.h> #include <xen/page.h> #include <xen/hvc-console.h> @@ -43,6 +45,7 @@#include <asm/page.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> +#include <asm/xen/cpuid.h> #include <asm/fixmap.h> #include <asm/processor.h> #include <asm/proto.h> @@ -1198,3 +1201,115 @@ asmlinkage void __init xen_start_kernel(void)x86_64_start_reservations ((char *)__pa_symbol (&boot_params));#endif } + +static void __init xen_hvm_pv_banner (void ) +{ +unsigned version =HYPERVISOR_xen_version (XENVER_version,NULL ); +struct xen_extraversion extra; +HYPERVISOR_xen_version (XENVER_extraversion,&extra); + +printk (KERN_INFO "Booting PV featured HVM kernel on %s\n" , + pv_info.name); +printk (KERN_INFO "Xen version: %d.%d%s\n" , + version>>16 , version&0xffff , extra.extraversion); +} + +static int xen_para_available (void ) +{ +uint32_t eax, ebx, ecx, edx; +cpuid (XEN_CPUID_LEAF (0 ),&eax,&ebx,&ecx,&edx); + +if (ebx ==XEN_CPUID_SIGNATURE_EBX && + ecx ==XEN_CPUID_SIGNATURE_ECX && + edx ==XEN_CPUID_SIGNATURE_EDX && + ((eax -XEN_CPUID_LEAF (0 ))>=2 )) +return 1 ; + +return 0 ; +} + +u32 xen_hvm_pv_status; +EXPORT_SYMBOL_GPL (xen_hvm_pv_status); + +static int enable_hvm_pv (u64 flags) +{ +struct xen_hvm_pv_type a; + + a.domid =DOMID_SELF ; + a.flags = flags; +return HYPERVISOR_hvm_op (HVMOP_enable_pv,&a); +} + +static int init_hvm_pv_info (void ) +{ +uint32_t ecx, edx, pages, msr; +u64 pfn; +
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1257
Quoted 10 lines
arch/x86/xen/enlighten.c +10
-0+if (!xen_para_available ()) +return -EINVAL ; + +cpuid (XEN_CPUID_LEAF (2 ),&pages,&msr,&ecx,&edx); + +/* Check if hvm_pv mode is supported */ +if (!(edx&XEN_CPUID_FEAT2_HVM_PV )) +return -ENODEV ; + + xen_hvm_pv_status =XEN_HVM_PV_ENABLED ;
Why use this? Why not just set the domain type to HVM, and leave the "status" field as a bitset of available paravirtualizations?
Quoted 25 lines
arch/x86/xen/enlighten.c +24
-0+ +/* We only support 1 page of hypercall for now */ +if (pages !=1 ) +return -ENOMEM ; + + pfn =__pa (hypercall_page); +wrmsrl (msr, pfn); + +xen_setup_features (); + + x86_init.oem.banner = xen_hvm_pv_banner; + pv_info = xen_info; + pv_info.kernel_rpl =0 ; + +return 0 ; +} + +extern struct shared_info shared_info_page; + +static void __init init_shared_info (void ) +{ +struct xen_add_to_physmap xatp; + + xatp.domid =DOMID_SELF ;
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1291
Quoted 10 lines
arch/x86/xen/enlighten.c +10
-0+ xatp.idx =0 ; + xatp.space = XENMAPSPACE_shared_info; + xatp.gpfn =__pa (&shared_info_page)>>PAGE_SHIFT ; +if (HYPERVISOR_memory_op (XENMEM_add_to_physmap,&xatp)) +BUG (); + + HYPERVISOR_shared_info = (struct shared_info *)&shared_info_page; + +/* Don't do the full vcpu_info placement stuff until we have a +possible map and a non-dummy shared_info. */
Is this comment meaningful here? This is the real shared info at this point, no? Are you going to support vcpu_info placement?
Quoted 4 lines
arch/x86/xen/enlighten.c +3
-0+per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; +} +
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1304
Quoted 10 lines
arch/x86/xen/enlighten.c +10
-0+void __init xen_guest_init (void ) +{ +#ifdef CONFIG_X86_32 +return ; +#else +int r; + +/* Ensure the we won't confused with PV */ +if (xen_domain_type ==XEN_PV_DOMAIN ) +return ;
Aren't you specifically testing for xen_domain_type == NATIVE here? If its anything else, then it means we're either PV, or have become an HVM domain some other way (like probing for the Xen platform PCI device).
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1314
Quoted 11 lines
arch/x86/xen/enlighten.c +10
-0+ + r =init_hvm_pv_info (); +if (r<0 ) +return ; + +init_shared_info (); + +xen_hvm_pv_init_irq_ops (); +#endif +}
Can you split all this off into a new file. It doesn't seem to have any dependencies on the rest of enlighten.c, and I've been trying to disaggregate it anyway.
Quoted 21 lines
arch/x86/xen/enlighten.c +1
-0+arch/x86/xen/irq.c +11
-0@@ -131,3 +131,24 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +static void xen_hvm_pv_safe_halt (void ) +{ +/* Do local_irq_enable() explicitly in hvm_pv guest */ +local_irq_enable (); +xen_safe_halt (); +} + +static void xen_hvm_pv_halt (void ) +{ +if (irqs_disabled ())
arch/x86/xen/irq.c void __init xen_init_irq_ops() · line 154
Quoted 10 lines
arch/x86/xen/irq.c +10
-0+HYPERVISOR_vcpu_op (VCPUOP_down,smp_processor_id (),NULL ); + else +xen_hvm_pv_safe_halt (); +} + +void __init xen_hvm_pv_init_irq_ops (void ) +{ + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; +}
It would be better to make this patch purely common infrastructure, and make specific features (like hvm+pv halt) separate patches (one per patch).
Quoted 5 lines
arch/x86/xen/xen-head.S +0
-0
arch/x86/xen/xen-head.S ENTRY(hypercall_page) · line 41
Quoted 9 lines
arch/x86/xen/xen-head.S +5
-0@@ -33,6 +33,12 @@ ENTRY(hypercall_page) .skip PAGE_SIZE_asm .popsection +.pushsection .data + .align PAGE_SIZE_asm +ENTRY(shared_info_page) + .skip PAGE_SIZE_asm +.popsection
Why does this need to be defined in asm? Can't it be either allocated or defined in C?
Quoted 45 lines
arch/x86/xen/xen-head.S +1
-0+ ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz "2.6")
ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz "xen-3.0")arch/x86/xen/xen-ops.h +1
-0@@ -41,6 +41,7 @@ void xen_vcpu_restore(void);void __init xen_build_dynamic_phys_to_machine (void );void xen_init_irq_ops (void ); +void xen_hvm_pv_init_irq_ops (void );void xen_setup_timer (int cpu);void xen_setup_runstate_info (int cpu);void xen_teardown_timer (int cpu);include/xen/interface/hvm/hvm_op.h +7
-0@@ -69,4 +69,11 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_set_pci_link_route);/* Flushes all VCPU TLBs: @arg must be NULL. */ #define HVMOP_flush_tlbs 5 +#define HVMOP_enable_pv 9 +struct xen_hvm_pv_type { +domid_t domid; +uint32_t flags; +#define HVM_PV_EVTCHN (1ull<<1) +}; +#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */ include/xen/xen.h +1
-0@@ -9,6 +9,7 @@ enum xen_domain_type {#ifdef CONFIG_XEN extern enum xen_domain_type xen_domain_type; +extern void xen_guest_init (void );#else #define xen_domain_type XEN_NATIVE#endif
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 24
Quoted 5 lines
include/xen/xen.h +1
-0@@ -19,6 +20,14 @@ extern enum xen_domain_type xen_domain_type;#define xen_hvm_domain () (xen_domain()&& \ xen_domain_type == XEN_HVM_DOMAIN) +#define XEN_HVM_PV_ENABLED (1u<< 0)
Why have this? We already have xen_domain_type which will either be XEN_NATIVE (ie, either real native, or on some fully emulated environment we have no specific optimisations for), or XEN_HVM_DOMAIN (we know we're running under Xen as an HVM domain).
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 27
Quoted 3 lines
include/xen/xen.h +2
-0+#define XEN_HVM_PV_EVTCHN_ENABLED (1u<< 1) +extern u32 xen_hvm_pv_status;
I think "status" is a misnomer here. Isn't it specifically a set of PV features which are active?
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 32
Quoted 5 lines
include/xen/xen.h +4
-0+ +#define xen_hvm_pv_enabled () (xen_hvm_pv_status& XEN_HVM_PV_ENABLED) +#define xen_hvm_pv_evtchn_enabled () (xen_hvm_pv_enabled()&& \ + (xen_hvm_pv_status& XEN_HVM_PV_EVTCHN_ENABLED))
Testing for xen_hvm_pv_enabled() should be redundant; surely the status/feature flag won't be set unless the environment supports the feature, and if it does it doesn't really matter what the domain type is.
Quoted 5 lines
include/xen/xen.h +1
-0+#ifdef CONFIG_XEN_DOM0 #include<xen/interface/xen.h> #include<asm/xen/hypervisor.h>
J
Re: [Xen-devel] [PATCH 4/7] xen: The entrance for PV extension of HVM
In reply to Sheng YangQuoted 13 lines · Sheng Yang
xen_guest_init()would setup the environment.Cc Ingo Molnar<[email protected]>Signed-off-by Sheng Yang<[email protected]>--- arch/x86/kernel/setup.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-)arch/x86/kernel/setup.c +0
-0
arch/x86/kernel/setup.c line 119
Quoted 7 lines
arch/x86/kernel/setup.c +3
-0@@ -113,6 +113,10 @@#endif #include <asm/mce.h> +#ifdef CONFIG_XEN +#include <xen/xen.h> +#endif
No #ifdefs; put them in xen.h if necessary (which they aren't). This probably isn't the right header anyway; this is specifically for testing the presence of Xen and/or the current domain type. It shouldn't have any other prototypes in it, or include anything else (it probably already includes too much). Given this is already x86-specific code, include asm/xen/something.h.
arch/x86/kernel/setup.c void __init setup_arch(char **cmdline_p) · line 750
Quoted 12 lines
arch/x86/kernel/setup.c +1
-0+/* * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. * The direct mapping extends to
max_pfn_mapped, so that we can directly accessarch/x86/kernel/setup.c +3
-0@@ -740,6 +744,10 @@ void __init setup_arch(char **cmdline_p) x86_init.oem.arch_setup (); +#ifdef CONFIG_XEN +xen_guest_init (); +#endif
Again, no #ifdefs here. Put an #ifdeffed stub in an appropriate header.
J
Re: [LKML] [PATCH 7/7] xen: Enable grant table and xenbus for PV extension of HVM
In reply to Konrad Rzeszutek WilkQuoted 8 lines · On Tuesday 02 March 2010 01:38:58 Konrad Rzeszutek Wilk wrote:
drivers/xen/grant-table.c static int gnttab_map(unsigned int start_idx, unsigned int end_idx) · line 501
Quoted 3 lines
drivers/xen/grant-table.c +3
-0+/* The region reserved by QEmu for Xen platform device */ +#define GNTTAB_START 0xf2000000ul +#define GNTTAB_SIZE 0x20000ulI thought that in the earlier review you said:
"> > +#define
GNTTAB_START0xfbfe0000ul
Quoted 9 lines
Quoted 5 lines
Quoted 1 line
+#define
GNTTAB_SIZE0x20000ulIs it possible that there would be a PCI device that would be passed in the guest that would conflict with the above mentioned E820 region?
I would change them to a dedicated PCI MMIO address in the next version. Thanks.
Oops... Thanks for reminder. :)
Signature · 2 lines
regards Yang, Sheng
Re: [LKML] [PATCH 7/7] xen: Enable grant table and xenbus for PV extension of HVM
In reply to Konrad Rzeszutek WilkQuoted 14 lines · On Tuesday 02 March 2010 01:38:58 Konrad Rzeszutek Wilk wrote:
drivers/xen/grant-table.c static int gnttab_map(unsigned int start_idx, unsigned int end_idx) · line 501
Quoted 3 lines
drivers/xen/grant-table.c +3
-0+/* The region reserved by QEmu for Xen platform device */ +#define GNTTAB_START 0xf2000000ul +#define GNTTAB_SIZE 0x20000ulI thought that in the earlier review you said:
"> > +#define
GNTTAB_START0xfbfe0000ulQuoted 5 lines
Quoted 1 line
+#define
GNTTAB_SIZE0x20000ulIs it possible that there would be a PCI device that would be passed in the guest that would conflict with the above mentioned E820 region?
Quoted 5 lines
I would change them to a dedicated PCI MMIO address in the next version. Thanks.
"
?
And yes, this is the dedicated PCI MMIO address I mentioned.. I would update the comments to get it more clear.
I don't think it's very clear solution, because the real good way to do this is probe pci device and find out with one is the platform pci device then use it. But the grant table initialization is quite earlier compared to the possible probing now... I hardcode the position now, and hunting for a better idea.
Comments?
Signature · 2 lines
regards Yang, Sheng
Quoted 8 lines · On Tuesday 02 March 2010 08:42:01 Jeremy Fitzhardinge wrote:
Quoted 7 lines · Sheng Yang
I'd like to have your comments on this patch series. We think the patches are prepared for check in
pv_opsdomU tree.The PV extension of HVM is started from real mode like HVM guest, but also with a a range of PV features(e.g. PV halt, PV timer, event channel, as well as PV drivers). So guest with this feature can takes the advantages of both H/W virtualization and Para-Virtualization.
Thanks for the comments!
Quoted 5 lines
The PV device side of this has a lot of overlap with the older PV driver for HVM domains code. Aside perhaps for the details of how interrupts/events are actually delivered, that seems like a completely distinct piece of work which could be done separately.
Yes. PV driver is not our core target. So leave it a little late is fine.
Quoted 4 lines
Also, I'd like to see a clearer statement of what you're specifically trying to optimise for here. What is inefficient in an unmodified HVM domain, and how do your patches address these things. What measurements have you made?
The key issue we want to address is the overhead of LAPIC, especially EOI and other actions resulted in explicit vmexit. That is the overhead we want to eliminate. And evtchn seems like a perfect choice to us.
Quoted 19 lines
Quoted 19 lines
The first two of the patchset imported several header file from Jeremy's tree and Xen tree, respect to Jeremy and Keir's works.
The whole patchset based on Linux upstream.
You need a line like:
cpuid = [ '0x40000002:edx=0x3' ]
in HVM configuration file to expose hybrid feature to guest, and
CONFIG_XENin the guest kernel configuration file to enable the hybrid support.
And the compiled image can be used as native/pv domU/hvm guest/pv feature hvm kernel.
Quoted 7 lines
Quoted 1 line
Current the patchset support
x86_64only.Please add a new config option for the new code, and make it dependent on x86-64, rather than scattering random #ifdef
CONFIG_X86_64saround the code. Make sure all the code is controlled by this CONFIG variable so it disappears in a 32-bit build (ideally by putting it into a new file where possible, and only building it when enabled).
Sure.
Quoted 5 lines
Quoted 5 lines
I've discussed the vector-evtchn mapping with Stefano, and we agreed on that it's more complex than we thought, and can't brought much benefit for pvops domU. So there is no major change in this update.
Quoted 8 lines
Quoted 2 lines
Change from v3:
1. Rebase to Linux 2.6.33 release.Why 2.6.33? I'm currently working on 2.6.32; I intend to move to .33 shortly, but .32 is a much more convenient base (particularly if you're interested in getting these changes into any distro kernels).
I rebased it to .32 without much effort.
OK.
Quoted 3 lines
I'll make specific comments in the patches.
Thanks
Signature · 2 lines
regards Yang, Sheng
Quoted 3 lines
J
Quoted 8 lines · Sheng Yang
Quoted 6 lines
Why 2.6.33? I'm currently working on 2.6.32; I intend to move to .33 shortly, but .32 is a much more convenient base (particularly if you're interested in getting these changes into any distro kernels).
I rebased it to .32 without much effort.
OK.
Hm, looks like the x86_platform_ipi stuff is missing in .32.
J
Quoted 11 lines · On Tuesday 02 March 2010 09:32:20 Jeremy Fitzhardinge wrote:
Quoted 7 lines · Sheng Yang
Quoted 5 lines
Why 2.6.33? I'm currently working on 2.6.32; I intend to move to .33 shortly, but .32 is a much more convenient base (particularly if you're interested in getting these changes into any distro kernels).
I rebased it to .32 without much effort.
OK.
Hm, looks like the
x86_platform_ipistuff is missing in .32.
I thought that one should still called smp_generic_interrupt in .32?
Signature · 2 lines
regards Yang, Sheng
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Jeremy FitzhardingeQuoted 112 lines · On Tuesday 02 March 2010 09:02:40 Jeremy Fitzhardinge wrote:
Quoted 111 lines · Sheng Yang
The PV extension of HVM(once known as Hybrid) is started from real mode like HVM guest, but also with a component based PV feature selection(e.g. PV halt, PV timer, event channel, then PV drivers). So guest can takes the advantages of both H/W virtualization and Para-Virtualization.
This patch introduced the PV extension of HVM guest initialization.
Guest would detect the capability using CPUID 0x40000002.edx, then call
HVMOP_enable_pvhypercall to enable pv support in hypervisor.Signed-off-by Sheng Yang<[email protected]>Signed-off-by Yaozu (Eddie) Dong<[email protected]>--- arch/x86/include/asm/xen/cpuid.h | 5 ++ arch/x86/xen/enlighten.c | 115 ++++++++++++++++++++++++++++++++++++ arch/x86/xen/irq.c | 21 +++++++ arch/x86/xen/xen-head.S | 6 ++ arch/x86/xen/xen-ops.h | 1 + include/xen/interface/hvm/hvm_op.h | 7 ++ include/xen/xen.h | 9 +++ 7 files changed, 164 insertions(+), 0 deletions(-)(quoted diff) +0
-0
b/arch/x86/include/asm/xen/cpuid.hindex 8787f03..a93c851 100644arch/x86/include/asm/xen/cpuid.h +5
-0@@ -65,4 +65,9 @@#define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0#define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV 0 +#define XEN_CPUID_FEAT2_HVM_PV (1u<<0) +#define _XEN_CPUID_FEAT2_HVM_PV_EVTCHN 1 +#define XEN_CPUID_FEAT2_HVM_PV_EVTCHN (1u<<1) +#endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */ arch/x86/xen/enlighten.c +52
-0@@ -34,6 +34,8 @@#include <xen/interface/version.h> #include <xen/interface/physdev.h> #include <xen/interface/vcpu.h> +#include <xen/interface/memory.h> +#include <xen/interface/hvm/hvm_op.h> #include <xen/features.h> #include <xen/page.h> #include <xen/hvc-console.h> @@ -43,6 +45,7 @@#include <asm/page.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> +#include <asm/xen/cpuid.h> #include <asm/fixmap.h> #include <asm/processor.h> #include <asm/proto.h> @@ -1198,3 +1201,115 @@ asmlinkage void __init xen_start_kernel(void)x86_64_start_reservations ((char *)__pa_symbol (&boot_params));#endif } + +static void __init xen_hvm_pv_banner (void ) +{ +unsigned version =HYPERVISOR_xen_version (XENVER_version,NULL ); +struct xen_extraversion extra; +HYPERVISOR_xen_version (XENVER_extraversion,&extra); + +printk (KERN_INFO "Booting PV featured HVM kernel on %s\n" , + pv_info.name); +printk (KERN_INFO "Xen version: %d.%d%s\n" , + version>>16 , version&0xffff , extra.extraversion); +} + +static int xen_para_available (void ) +{ +uint32_t eax, ebx, ecx, edx; +cpuid (XEN_CPUID_LEAF (0 ),&eax,&ebx,&ecx,&edx); + +if (ebx ==XEN_CPUID_SIGNATURE_EBX && + ecx ==XEN_CPUID_SIGNATURE_ECX && + edx ==XEN_CPUID_SIGNATURE_EDX && + ((eax -XEN_CPUID_LEAF (0 ))>=2 )) +return 1 ; + +return 0 ; +} + +u32 xen_hvm_pv_status; +EXPORT_SYMBOL_GPL (xen_hvm_pv_status); + +static int enable_hvm_pv (u64 flags) +{ +struct xen_hvm_pv_type a; + + a.domid =DOMID_SELF ; + a.flags = flags; +return HYPERVISOR_hvm_op (HVMOP_enable_pv,&a); +} + +static int init_hvm_pv_info (void ) +{ +uint32_t ecx, edx, pages, msr; +u64 pfn; + +if (!xen_para_available ()) +return -EINVAL ; + +cpuid (XEN_CPUID_LEAF (2 ),&pages,&msr,&ecx,&edx); +
Quoted 8 lines
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1257
Quoted 5 lines
arch/x86/xen/enlighten.c +5
-0+/* Check if hvm_pv mode is supported */ +if (!(edx&XEN_CPUID_FEAT2_HVM_PV )) +return -ENODEV ; + + xen_hvm_pv_status =XEN_HVM_PV_ENABLED ;Why use this? Why not just set the domain type to HVM, and leave the "status" field as a bitset of available paravirtualizations?
A annoy thing in pv drivers is that it would test if the domain type is _NOT_ XEN_NATIVE. So set the domain to XEN_HVM_DOMAIN would result in PV driver initialization then probably panic. Maybe we can do something to PV drivers, as patch 6 and a part of patch 7.
Quoted 30 lines
Quoted 30 lines
arch/x86/xen/enlighten.c +29
-0+ +/* We only support 1 page of hypercall for now */ +if (pages !=1 ) +return -ENOMEM ; + + pfn =__pa (hypercall_page); +wrmsrl (msr, pfn); + +xen_setup_features (); + + x86_init.oem.banner = xen_hvm_pv_banner; + pv_info = xen_info; + pv_info.kernel_rpl =0 ; + +return 0 ; +} + +extern struct shared_info shared_info_page; + +static void __init init_shared_info (void ) +{ +struct xen_add_to_physmap xatp; + + xatp.domid =DOMID_SELF ; + xatp.idx =0 ; + xatp.space = XENMAPSPACE_shared_info; + xatp.gpfn =__pa (&shared_info_page)>>PAGE_SHIFT ; +if (HYPERVISOR_memory_op (XENMEM_add_to_physmap,&xatp)) +BUG ();
Quoted 8 lines
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1291
Quoted 5 lines
arch/x86/xen/enlighten.c +5
-0+ + HYPERVISOR_shared_info = (struct shared_info *)&shared_info_page; + +/* Don't do the full vcpu_info placement stuff until we have a +possible map and a non-dummy shared_info. */ Is this comment meaningful here? This is the real shared info at this point, no? Are you going to support
vcpu_infoplacement?
Would discard this..
Quoted 9 lines
Quoted 9 lines
arch/x86/xen/enlighten.c +9
-0+per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; +} + +void __init xen_guest_init (void ) +{ +#ifdef CONFIG_X86_32 +return ; +#else +int r;
Quoted 8 lines
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1304
Quoted 4 lines
arch/x86/xen/enlighten.c +4
-0+ +/* Ensure the we won't confused with PV */ +if (xen_domain_type ==XEN_PV_DOMAIN ) +return ;Aren't you specifically testing for
xen_domain_type== NATIVE here? If its anything else, then it means we're either PV, or have become an HVM domain some other way (like probing for the Xen platform PCI device).
Yes, that's better.
Quoted 6 lines
Quoted 6 lines
arch/x86/xen/enlighten.c +6
-0+ + r =init_hvm_pv_info (); +if (r<0 ) +return ; + +init_shared_info ();
Quoted 8 lines
arch/x86/xen/enlighten.c asmlinkage void __init xen_start_kernel(void) · line 1314
Quoted 4 lines
arch/x86/xen/enlighten.c +4
-0+ +xen_hvm_pv_init_irq_ops (); +#endif +}Can you split all this off into a new file. It doesn't seem to have any dependencies on the rest of
enlighten.c, and I've been trying to disaggregate it anyway.
Part of pv_ops are overlapped. I would try if a new file would bring much duplicate.
Quoted 25 lines
Quoted 25 lines
arch/x86/xen/enlighten.c +1
-0+arch/x86/xen/irq.c +16
-0@@ -131,3 +131,24 @@ void __init xen_init_irq_ops() pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } + +static void xen_hvm_pv_safe_halt (void ) +{ +/* Do local_irq_enable() explicitly in hvm_pv guest */ +local_irq_enable (); +xen_safe_halt (); +} + +static void xen_hvm_pv_halt (void ) +{ +if (irqs_disabled ()) +HYPERVISOR_vcpu_op (VCPUOP_down,smp_processor_id (),NULL ); +else +xen_hvm_pv_safe_halt (); +} +
Quoted 8 lines
arch/x86/xen/irq.c void __init xen_init_irq_ops() · line 154
Quoted 5 lines
arch/x86/xen/irq.c +5
-0+void __init xen_hvm_pv_init_irq_ops (void ) +{ + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; +}It would be better to make this patch purely common infrastructure, and make specific features (like hvm+pv halt) separate patches (one per patch).
OK. (In fact I merged them in second edition...)
Quoted 9 lines
Quoted 9 lines
arch/x86/xen/xen-head.S +0
-0@@ -33,6 +33,12 @@ ENTRY(hypercall_page) .skip PAGE_SIZE_asm .popsection
Quoted 8 lines
arch/x86/xen/xen-head.S ENTRY(hypercall_page) · line 40
Quoted 5 lines
arch/x86/xen/xen-head.S +5
-0+.pushsection .data + .align PAGE_SIZE_asm +ENTRY(shared_info_page) + .skip PAGE_SIZE_asm +.popsectionWhy does this need to be defined in asm? Can't it be either allocated or defined in C?
I think we need a aligned page, as hypercall page.
Quoted 50 lines
Quoted 50 lines
arch/x86/xen/xen-head.S +1
-0+ ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz "2.6") ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz "xen-3.0")arch/x86/xen/xen-ops.h +1
-0@@ -41,6 +41,7 @@ void xen_vcpu_restore(void);void __init xen_build_dynamic_phys_to_machine (void );void xen_init_irq_ops (void ); +void xen_hvm_pv_init_irq_ops (void );void xen_setup_timer (int cpu);void xen_setup_runstate_info (int cpu);void xen_teardown_timer (int cpu);(quoted diff) +0
-0
b/include/xen/interface/hvm/hvm_op.hindex 7c74ba4..0ce8a26 100644include/xen/interface/hvm/hvm_op.h +0
-0@@ -69,4 +69,11 @@
DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_set_pci_link_route); /* Flushes all VCPU TLBs: @arg must be NULL. */
#defineHVMOP_flush_tlbs5include/xen/interface/hvm/hvm_op.h +7
-0+#define HVMOP_enable_pv 9 +struct xen_hvm_pv_type { +domid_t domid; +uint32_t flags; +#define HVM_PV_EVTCHN (1ull<<1) +}; +#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */ include/xen/xen.h +1
-0@@ -9,6 +9,7 @@ enum xen_domain_type {#ifdef CONFIG_XEN extern enum xen_domain_type xen_domain_type; +extern void xen_guest_init (void );#else #define xen_domain_type XEN_NATIVE#endif @@ -19,6 +20,14 @@ extern enum xen_domain_type xen_domain_type;#define xen_hvm_domain () (xen_domain()&& \ xen_domain_type == XEN_HVM_DOMAIN)
Quoted 6 lines
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 23
Quoted 1 line
include/xen/xen.h +1
-0+#define XEN_HVM_PV_ENABLED (1u<< 0)Why have this? We already have
xen_domain_typewhich will either beXEN_NATIVE(ie, either real native, or on some fully emulated environment we have no specific optimisations for), orXEN_HVM_DOMAIN(we know we're running under Xen as an HVM domain).
Something may need to change in pv driver, as I said above.
Quoted 6 lines
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 25
Quoted 3 lines
include/xen/xen.h +2
-0+#define XEN_HVM_PV_EVTCHN_ENABLED (1u<< 1) +extern u32 xen_hvm_pv_status;I think "status" is a misnomer here. Isn't it specifically a set of PV features which are active?
Could you give a suggestion of the name? I am not a native English speaker...
Quoted 9 lines
include/xen/xen.h extern enum xen_domain_type xen_domain_type; · line 29
Quoted 5 lines
include/xen/xen.h +4
-0+ +#define xen_hvm_pv_enabled () (xen_hvm_pv_status& XEN_HVM_PV_ENABLED) +#define xen_hvm_pv_evtchn_enabled () (xen_hvm_pv_enabled()&& \ + (xen_hvm_pv_status& XEN_HVM_PV_EVTCHN_ENABLED))Testing for
xen_hvm_pv_enabled()should be redundant; surely the status/feature flag won't be set unless the environment supports the feature, and if it does it doesn't really matter what the domain type is.
Sure
Signature · 2 lines
regards Yang, Sheng
Quoted 7 lines
include/xen/xen.h extern enum xen_domain_type xen_domain_type;
Quoted 4 lines
include/xen/xen.h +1
-0+#ifdef CONFIG_XEN_DOM0 #include <xen/interface/xen.h> #include <asm/xen/hypervisor.h> J
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
In reply to Sheng YangQuoted 6 lines · Sheng Yang
We mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used
X86_PLATFORM_IPI_VECTORas the noficiation vector for hypervisor to notify guest about the event.
I don't understand this aspect of the patch. Can you explain the interrupt delivery path when HVM event channels are enabled?
Quoted 97 lines
The Xen PV timer is used to provide guest a reliable timer.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Then we don't use IOAPIC/LAPIC.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/enlighten.c | 72 +++++++++++++++++++++ arch/x86/xen/irq.c | 37 ++++++++++- arch/x86/xen/smp.c | 144 ++++++++++++++++++++++++++++++++++++++++++- arch/x86/xen/xen-ops.h | 3 + drivers/xen/events.c | 66 ++++++++++++++++++- include/xen/events.h | 1 + include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- 8 files changed, 326 insertions(+), 8 deletions(-)arch/x86/xen/enlighten.c +38
-0@@ -58,6 +58,9 @@#include <asm/reboot.h> #include <asm/stackprotector.h> +#include <xen/hvm.h> +#include <xen/events.h> +#include "xen-ops.h" #include "mmu.h" #include "multicalls.h" @@ -1212,6 +1215,8 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +if (xen_hvm_pv_evtchn_enabled ()) +printk (KERN_INFO "PV feature: Event channel enabled\n" ); }static int xen_para_available (void )@@ -1256,6 +1261,9 @@ static int init_hvm_pv_info(void) xen_hvm_pv_status =XEN_HVM_PV_ENABLED ; +if (edx&XEN_CPUID_FEAT2_HVM_PV_EVTCHN ) + xen_hvm_pv_status |=XEN_HVM_PV_EVTCHN_ENABLED ; +/* We only support 1 page of hypercall for now */ if (pages !=1 )return -ENOMEM ;@@ -1292,12 +1300,42 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; } +static int set_callback_via (uint64_t via) +{ +struct xen_hvm_param a; + + a.domid =DOMID_SELF ; + a.index =HVM_PARAM_CALLBACK_IRQ ; + a.value = via; +return HYPERVISOR_hvm_op (HVMOP_set_param,&a); +} + +void do_hvm_pv_evtchn_intr (void ) +{ +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())++; +#endif +xen_evtchn_do_upcall (get_irq_regs ()); +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())--; +#endif +} + +#ifdef CONFIG_X86_LOCAL_APIC +static void xen_hvm_pv_evtchn_apic_write (u32 reg,u32 val) +{ +/* The only one reached here should be EOI */ +WARN_ON (reg !=APIC_EOI ); +} +#endif +void __init xen_guest_init (void ) {#ifdef CONFIG_X86_32 return ;#else int r; +uint64_t callback_via;/* Ensure the we won't confused with PV */ if (
xen_domain_type==XEN_PV_DOMAIN)
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1354
Quoted 7 lines
arch/x86/xen/enlighten.c +3
-0@@ -1310,6 +1348,40 @@ void __init xen_guest_init(void)init_shared_info ();xen_hvm_pv_init_irq_ops (); + +if (xen_hvm_pv_evtchn_enabled ()) { +if (enable_hvm_pv (HVM_PV_EVTCHN ))
If this fails, shouldn't it also clear XEN_HVM_PV_EVTCHN_ENABLED?
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1387
Quoted 8 lines
arch/x86/xen/enlighten.c +7
-0+return ; + + pv_time_ops = xen_time_ops; + + x86_init.timers.timer_init = xen_time_init; + x86_init.timers.setup_percpu_clockev = x86_init_noop; + x86_cpuinit.setup_percpu_clockev = x86_init_noop;
Presumably even if we don't have PV_EVTCHN available/enabled, the Xen clocksource would be available for getting time?
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1387
Quoted 13 lines
arch/x86/xen/enlighten.c +12
-0+ + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + + pv_apic_ops = xen_apic_ops; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +set_xen_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write;
I'd just change the xen_apic_write to remove the WARN_ON, since you don't seem to care about it either.
Quoted 109 lines
arch/x86/xen/enlighten.c +12
-0+#endif + + callback_via =HVM_CALLBACK_VECTOR (X86_PLATFORM_IPI_VECTOR ); +set_callback_via (callback_via); + + x86_platform_ipi_callback = do_hvm_pv_evtchn_intr; + +disable_acpi (); + +xen_hvm_pv_smp_init (); + machine_ops = xen_machine_ops; + }#endif }arch/x86/xen/irq.c +35
-2@@ -5,6 +5,7 @@#include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h> +#include <xen/xen.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> @@ -132,6 +133,20 @@ void __init xen_init_irq_ops() x86_init.irqs.intr_init = xen_init_IRQ; } +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); +static void xen_hvm_pv_safe_halt (void ) {/* Do local_irq_enable() explicitly in hvm_pv guest */ @@ -147,8 +162,26 @@ static void xen_hvm_pv_halt(void)xen_hvm_pv_safe_halt (); } +static const struct pv_irq_ops xen_hvm_pv_irq_ops__initdata = { + .save_fl =__PV_IS_CALLEE_SAVE (native_save_fl), + .restore_fl =__PV_IS_CALLEE_SAVE (native_restore_fl), + .irq_disable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable), + .irq_enable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable), + + .safe_halt = xen_hvm_pv_safe_halt, + .halt = xen_hvm_pv_halt, +#ifdef CONFIG_X86_64 + .adjust_exception_frame = paravirt_nop, +#endif +}; +void __init xen_hvm_pv_init_irq_ops (void ) { - pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; - pv_irq_ops.halt = xen_hvm_pv_halt; +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops = xen_hvm_pv_irq_ops; + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + }else { + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; + } }arch/x86/xen/smp.c +6
-0@@ -15,20 +15,26 @@#include <linux/sched.h> #include <linux/err.h> #include <linux/smp.h> +#include <linux/nmi.h> #include <asm/paravirt.h> #include <asm/desc.h> #include <asm/pgtable.h> #include <asm/cpu.h> +#include <asm/trampoline.h> +#include <asm/tlbflush.h> +#include <asm/mtrr.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <asm/xen/hypervisor.h> #include <xen/page.h> #include <xen/events.h> +#include <xen/xen.h> #include "xen-ops.h" #include "mmu.h"
arch/x86/xen/smp.c static void __init xen_smp_prepare_boot_cpu(void) · line 182
Quoted 7 lines
arch/x86/xen/smp.c +2
-1@@ -171,7 +177,8 @@ static void __init xen_smp_prepare_boot_cpu(void)/* We've switched to the "real" per-cpu gdt, so make sure the old memory can be recycled */ -make_lowmem_page_readwrite (xen_initial_gdt); +if (xen_pv_domain ()) +make_lowmem_page_readwrite (xen_initial_gdt);
Test for xen_feature(XENFEAT_writable_descriptor_tables).
Quoted 29 lines
arch/x86/xen/smp.c +21
-0xen_setup_vcpu_info_placement (); }@@ -480,3 +487,138 @@ void __init xen_smp_init(void)xen_fill_possible_map ();xen_init_spinlocks (); } + +static __cpuinit void xen_hvm_pv_start_secondary (void ) +{ +int cpu =smp_processor_id (); + +cpu_init (); +touch_nmi_watchdog (); +preempt_disable (); + +/* otherwise gcc will move up smp_processor_id before the cpu_init */ +barrier (); +/* +* Check TSC synchronization with the BSP: +*/ +check_tsc_sync_target (); + +/* Done in smp_callin(), move it here */ +set_mtrr_aps_delayed_init (); +smp_store_cpu_info (cpu); + +/* This must be done before setting cpu_online_mask */
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 520
Quoted 10 lines
arch/x86/xen/smp.c +10
-0+set_cpu_sibling_map (cpu); +wmb (); + +set_cpu_online (smp_processor_id (), true); +per_cpu (cpu_state,smp_processor_id ()) =CPU_ONLINE ; + +/* enable local interrupts */ +local_irq_enable (); + +xen_setup_cpu_clockevents ();
How much of this is necessary? At this point, isn't CPU bringup the same as PV?
Quoted 51 lines
arch/x86/xen/smp.c +50
-0+ +wmb (); +cpu_idle (); +} + +static __cpuinit int +hvm_pv_cpu_initialize_context (unsigned int cpu,struct task_struct *idle) +{ +struct vcpu_guest_context *ctxt; +unsigned long start_ip; + +if (cpumask_test_and_set_cpu (cpu, xen_cpu_initialized_map)) +return 0 ; + + ctxt =kzalloc (sizeof (*ctxt),GFP_KERNEL ); +if (ctxt ==NULL ) +return -ENOMEM ; + + early_gdt_descr.address = (unsigned long )get_cpu_gdt_table (cpu); + initial_code = (unsigned long )xen_hvm_pv_start_secondary; + stack_start.sp = (void *) idle->thread.sp; + +/* start_ip had better be page-aligned! */ + start_ip =setup_trampoline (); + +/* only start_ip is what we want */ + ctxt->flags =VGCF_HVM_GUEST ; + ctxt->user_regs.eip = start_ip; + +printk (KERN_INFO "Booting processor %d ip 0x%lx\n" , cpu, start_ip); + +if (HYPERVISOR_vcpu_op (VCPUOP_initialise, cpu, ctxt)) +BUG (); + +kfree (ctxt); +return 0 ; +} + +static int __init xen_hvm_pv_cpu_up (unsigned int cpu) +{ +struct task_struct *idle =idle_task (cpu); +int rc; +unsigned long flags; + +per_cpu (current_task, cpu) = idle; + +#ifdef CONFIG_X86_32 +irq_ctx_init (cpu); +#else +clear_tsk_thread_flag (idle,TIF_FORK );
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 580
Quoted 10 lines
arch/x86/xen/smp.c +10
-0+ initial_gs =per_cpu_offset (cpu); +per_cpu (kernel_stack, cpu) = + (unsigned long )task_stack_page (idle) - +KERNEL_STACK_OFFSET +THREAD_SIZE ; +#endif + +xen_setup_timer (cpu); +xen_init_lock_cpu (cpu); + +per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ;
Can you put all this duplicated code into a common function?
Quoted 27 lines
arch/x86/xen/smp.c +26
-0+ + rc =hvm_pv_cpu_initialize_context (cpu, idle); +if (rc) +return rc; + +if (num_online_cpus () ==1 ) +alternatives_smp_switch (1 ); + + rc =xen_smp_intr_init (cpu); +if (rc) +return rc; + + rc =HYPERVISOR_vcpu_op (VCPUOP_up, cpu,NULL ); +BUG_ON (rc); + +/* +* Check TSC synchronization with the AP (keep irqs disabled +* while doing so): +*/ +local_irq_save (flags); +check_tsc_sync_source (cpu); +local_irq_restore (flags); + +while (!cpu_online (cpu)) { +cpu_relax (); +touch_nmi_watchdog ();
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 616
Quoted 10 lines
arch/x86/xen/smp.c +10
-0+ } + +return 0 ; +} + +static void xen_hvm_pv_flush_tlb_others (const struct cpumask *cpumask, +struct mm_struct *mm,unsigned long va) +{ +/* TODO Make it more specific */ +flush_tlb_all ();
Is the MMUEXT_TLB_FLUSH/INVLPG_MULTI hypercall not currently available to HVM?
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 620
Quoted 5 lines
arch/x86/xen/smp.c +4
-0+} + +void __init xen_hvm_pv_smp_init (void ) +{
It's probably safest to have this do its own test for xen_hvm_pv_evtchn_enabled(), rather than assuming the caller is getting it right.
Quoted 50 lines
arch/x86/xen/smp.c +4
-0+ smp_ops = xen_smp_ops; + smp_ops.cpu_up = xen_hvm_pv_cpu_up; + pv_mmu_ops.flush_tlb_others = xen_hvm_pv_flush_tlb_others; +}arch/x86/xen/xen-ops.h +3
-0@@ -34,6 +34,7 @@ void xen_reserve_top(void);char *__init xen_memory_setup (void );void __init xen_arch_setup (void );void __init xen_init_IRQ (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void );void xen_enable_sysenter (void );void xen_enable_syscall (void );void xen_vcpu_restore (void );@@ -61,10 +62,12 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +void xen_hvm_pv_smp_init (void );extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +static inline void xen_hvm_pv_smp_init (void ) {}#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS drivers/xen/events.c +3
-0@@ -37,9 +37,12 @@#include <xen/xen-ops.h> #include <xen/events.h> +#include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/interface/event_channel.h> +#include <asm/desc.h> +/* * This lock protects updates to the following mapping and reference-count * arrays. The lock does not need to be acquired to read the mapping tables. @@ -624,8 +627,13 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)struct vcpu_info *vcpu_info =__get_cpu_var (xen_vcpu);unsigned count;
drivers/xen/events.c void xen_evtchn_do_upcall(struct pt_regs *regs) · line 637
Quoted 10 lines
drivers/xen/events.c +7
-2-exit_idle (); -irq_enter (); +/* +* If is PV featured HVM, these have already been done +*/ +if (likely (!xen_hvm_pv_evtchn_enabled ())) { +exit_idle (); +irq_enter (); + }
In that case, rather than putting this conditional in the hot path, make an inner __xen_evtchn_do_upcall which is wrapped by the PV and HVM variants which do the appropriate things. (And drop the pt_regs arg, I think.)
Quoted 97 lines
drivers/xen/events.c +52
-2do { unsigned long pending_words;@@ -662,8 +670,10 @@ void xen_evtchn_do_upcall(struct pt_regs *regs) }while (count !=1 ); out: -irq_exit (); -set_irq_regs (old_regs); +if (likely (!xen_hvm_pv_evtchn_enabled ())) { +irq_exit (); +set_irq_regs (old_regs); + }put_cpu (); }@@ -944,3 +954,51 @@ void __init xen_init_IRQ(void)irq_ctx_init (smp_processor_id ()); } + +void __init xen_hvm_pv_evtchn_init_IRQ (void ) +{ +int i; + +xen_init_IRQ (); +for (i =0 ; i<NR_IRQS_LEGACY ; i++) { +struct evtchn_bind_virq bind_virq; +struct irq_desc *desc =irq_to_desc (i); +int virq, evtchn; + + virq = i +VIRQ_EMUL_PIN_START ; + bind_virq.virq = virq; + bind_virq.vcpu =0 ; + +if (HYPERVISOR_event_channel_op (EVTCHNOP_bind_virq, + &bind_virq) !=0 ) +BUG (); + + evtchn = bind_virq.port; + evtchn_to_irq[evtchn] = i; + irq_info[i] =mk_virq_info (evtchn, virq); + + desc->status =IRQ_DISABLED ; + desc->action =NULL ; + desc->depth =1 ; + +/* +* 16 old-style INTA-cycle interrupts: +*/ +set_irq_chip_and_handler_name (i,&xen_dynamic_chip, + handle_level_irq,"event" ); + } + +/* +* Cover the whole vector space, no vector can escape +* us. (some of these will be overridden and become +* 'special' SMP interrupts) +*/ +for (i =0 ; i< (NR_VECTORS -FIRST_EXTERNAL_VECTOR ); i++) { +int vector =FIRST_EXTERNAL_VECTOR + i; +if (vector !=IA32_SYSCALL_VECTOR ) +set_intr_gate (vector, interrupt[i]); + } + +/* generic IPI for platform specific use, now used for HVM evtchn */ +alloc_intr_gate (X86_PLATFORM_IPI_VECTOR , x86_platform_ipi); +}include/xen/events.h +1
-0@@ -56,4 +56,5 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs);#endif /* _XEN_EVENTS_H */ include/xen/hvm.h +5
-0@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx) return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x) (((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ + HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +#endif /* XEN_HVM_H__ */ include/xen/interface/xen.h +0
-0
include/xen/interface/xen.h line 97
Quoted 9 lines
include/xen/interface/xen.h +4
-1@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS 40
(VIRQ_EMUL_PIN_START + VIRQ_EMUL_PIN_NUM)?
J
Re: [Xen-devel] [PATCH 4/7] xen: The entrance for PV extension of HVM
In reply to Jeremy FitzhardingeQuoted 17 lines · On Tuesday 02 March 2010 09:05:36 Jeremy Fitzhardinge wrote:
Quoted 16 lines · Sheng Yang
xen_guest_init()would setup the environment.Cc Ingo Molnar<[email protected]>Signed-off-by Sheng Yang<[email protected]>--- arch/x86/kernel/setup.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-)arch/x86/kernel/setup.c +0
-0@@ -113,6 +113,10 @@#endif #include <asm/mce.h>
Quoted 10 lines
arch/x86/kernel/setup.c line 118
Quoted 3 lines
arch/x86/kernel/setup.c +3
-0+#ifdef CONFIG_XEN +#include <xen/xen.h> +#endif No #ifdefs; put them in
xen.hif necessary (which they aren't). This probably isn't the right header anyway; this is specifically for testing the presence of Xen and/or the current domain type. It shouldn't have any other prototypes in it, or include anything else (it probably already includes too much). Given this is already x86-specific code, includeasm/xen/something.h.
Would update it. Thanks.
Signature · 2 lines
regards Yang, Sheng
Quoted 18 lines
arch/x86/kernel/setup.c void __init setup_arch(char **cmdline_p) · line 749
Quoted 13 lines
arch/x86/kernel/setup.c +1
-0+/* * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. * The direct mapping extends to
max_pfn_mapped, so that we can directly access @@ -740,6 +744,10 @@ void__initsetup_arch(char **cmdline_p)x86_init.oem.arch_setup();arch/x86/kernel/setup.c +3
-0+#ifdef CONFIG_XEN +xen_guest_init (); +#endif Again, no #ifdefs here. Put an #ifdeffed stub in an appropriate header.
J
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Sheng YangQuoted 8 lines · Sheng Yang
Quoted 3 lines
Why use this? Why not just set the domain type to HVM, and leave the "status" field as a bitset of available paravirtualizations?
A annoy thing in pv drivers is that it would test if the domain type is _NOT_
XEN_NATIVE. So set the domain toXEN_HVM_DOMAINwould result in PV driver initialization then probably panic. Maybe we can do something to PV drivers, as patch 6 and a part of patch 7.
In that case, change them to specifically test for PV until they're ready to deal with an HVM environment.
Quoted 4 lines
Part of
pv_opsare overlapped. I would try if a new file would bring much duplicate.
It shouldn't do; un-static things if necessary.
Quoted 10 lines
Quoted 8 lines
arch/x86/xen/xen-head.S ENTRY(hypercall_page) · line 40
Quoted 5 lines
arch/x86/xen/xen-head.S +4
-0+ .align PAGE_SIZE_asm +ENTRY(shared_info_page) + .skip PAGE_SIZE_asm +.popsectionWhy does this need to be defined in asm? Can't it be either allocated or defined in C?
I think we need a aligned page, as hypercall page.
That can be declared in C with the __page_aligned_bss annotation. And couldn't you dynamically allocate it with the bootmem allocator?
Quoted 5 lines
Quoted 3 lines
I think "status" is a misnomer here. Isn't it specifically a set of PV features which are active?
Could you give a suggestion of the name? I am not a native English speaker...
xen_hvm_pv_features.
J
Re: [Xen-devel] [PATCH 6/7] xen: Unified checking for Xen of PV drivers to xenbus_register_frontend()
In reply to Sheng YangQuoted 2 lines · Sheng Yang
Signed-off-by Sheng Yang<[email protected]>
We covered this before; just change the tests so they run in PV domains so they don't crash on HVM+PV, and we can address these later.
J
Quoted 11 lines
Quoted 5 lines
Also, I'd like to see a clearer statement of what you're specifically trying to optimise for here. What is inefficient in an unmodified HVM domain, and how do your patches address these things. What measurements have you made?
The key issue we want to address is the overhead of LAPIC, especially EOI and other actions resulted in explicit vmexit. That is the overhead we want to eliminate. And evtchn seems like a perfect choice to us.
That is not complete. Linux timer in HVM is always annoying, and time to time make surprise to endusers. PV timer does help here. Eventchannel is nice to HVM as well.
Thx, Eddie
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
In reply to Jeremy FitzhardingeQuoted 9 lines · On Tuesday 02 March 2010 09:38:21 Jeremy Fitzhardinge wrote:
Quoted 5 lines · Sheng Yang
We mapped each IOAPIC pin to a VIRQ, so that we can deliver interrupt through these VIRQs.
We used
X86_PLATFORM_IPI_VECTORas the noficiation vector for hypervisor to notify guest about the event.I don't understand this aspect of the patch. Can you explain the interrupt delivery path when HVM event channels are enabled?
The goal is to eliminate LAPIC overhead(especially EOI and other operation which caused VMExit when using LAPIC) in the current interrupt delivery system, so that we use evtchn to delivery interrupt.
For the emulated device, we allocate one VIRQ for each IOAPIC pin. When IOAPIC pin should be assert, instead we inject a VIRQ.
The X86_PLATFORM_IPI_VECTOR is the entry point for event handling. When we want to inject event(VIRQ or others) to guest, we inject a X86_PLATFORM_IPI_VECTOR, so that the handler of vector in the guest can handle the events well.
Quoted 101 lines
Quoted 101 lines
The Xen PV timer is used to provide guest a reliable timer.
The patch also enabled SMP support, then we can support IPI through evtchn as well.
Then we don't use IOAPIC/LAPIC.
Signed-off-by Sheng Yang<[email protected]>--- arch/x86/xen/enlighten.c | 72 +++++++++++++++++++++ arch/x86/xen/irq.c | 37 ++++++++++- arch/x86/xen/smp.c | 144 ++++++++++++++++++++++++++++++++++++++++++- arch/x86/xen/xen-ops.h | 3 + drivers/xen/events.c | 66 ++++++++++++++++++- include/xen/events.h | 1 + include/xen/hvm.h | 5 ++ include/xen/interface/xen.h | 6 ++- 8 files changed, 326 insertions(+), 8 deletions(-)arch/x86/xen/enlighten.c +38
-0@@ -58,6 +58,9 @@#include <asm/reboot.h> #include <asm/stackprotector.h> +#include <xen/hvm.h> +#include <xen/events.h> +#include "xen-ops.h" #include "mmu.h" #include "multicalls.h" @@ -1212,6 +1215,8 @@ static void __init xen_hvm_pv_banner(void) pv_info.name);printk (KERN_INFO "Xen version: %d.%d%s\n" , version>>16 , version&0xffff , extra.extraversion); +if (xen_hvm_pv_evtchn_enabled ()) +printk (KERN_INFO "PV feature: Event channel enabled\n" ); }static int xen_para_available (void )@@ -1256,6 +1261,9 @@ static int init_hvm_pv_info(void) xen_hvm_pv_status =XEN_HVM_PV_ENABLED ; +if (edx&XEN_CPUID_FEAT2_HVM_PV_EVTCHN ) + xen_hvm_pv_status |=XEN_HVM_PV_EVTCHN_ENABLED ; +/* We only support 1 page of hypercall for now */ if (pages !=1 )return -ENOMEM ;@@ -1292,12 +1300,42 @@ static void __init init_shared_info(void)per_cpu (xen_vcpu,0 ) =&HYPERVISOR_shared_info->vcpu_info[0 ]; } +static int set_callback_via (uint64_t via) +{ +struct xen_hvm_param a; + + a.domid =DOMID_SELF ; + a.index =HVM_PARAM_CALLBACK_IRQ ; + a.value = via; +return HYPERVISOR_hvm_op (HVMOP_set_param,&a); +} + +void do_hvm_pv_evtchn_intr (void ) +{ +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())++; +#endif +xen_evtchn_do_upcall (get_irq_regs ()); +#ifdef CONFIG_X86_64 +per_cpu (irq_count,smp_processor_id ())--; +#endif +} + +#ifdef CONFIG_X86_LOCAL_APIC +static void xen_hvm_pv_evtchn_apic_write (u32 reg,u32 val) +{ +/* The only one reached here should be EOI */ +WARN_ON (reg !=APIC_EOI ); +} +#endif +void __init xen_guest_init (void ) {#ifdef CONFIG_X86_32 return ;#else int r; +uint64_t callback_via;/* Ensure the we won't confused with PV */ if (
xen_domain_type==XEN_PV_DOMAIN)arch/x86/xen/enlighten.c +0
-0@@ -1310,6 +1348,40 @@ void __init xen_guest_init(void)init_shared_info ();
Quoted 6 lines
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1353
Quoted 4 lines
arch/x86/xen/enlighten.c +3
-0xen_hvm_pv_init_irq_ops (); + +if (xen_hvm_pv_evtchn_enabled ()) { +if (enable_hvm_pv (HVM_PV_EVTCHN ))If this fails, shouldn't it also clear
XEN_HVM_PV_EVTCHN_ENABLED?
Yes...
Quoted 10 lines
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1360
Quoted 7 lines
arch/x86/xen/enlighten.c +7
-0+return ; + + pv_time_ops = xen_time_ops; + + x86_init.timers.timer_init = xen_time_init; + x86_init.timers.setup_percpu_clockev = x86_init_noop; + x86_cpuinit.setup_percpu_clockev = x86_init_noop;Presumably even if we don't have
PV_EVTCHNavailable/enabled, the Xen clocksource would be available for getting time?
I think currently Xen pv clocksource and clockevent are binding... Not sure if a single line "clocksource_register(&xen_clocksource)" can work. I would give it a try, maybe add a new PV feature.
Quoted 7 lines
Quoted 7 lines
arch/x86/xen/enlighten.c +7
-0+ + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + + pv_apic_ops = xen_apic_ops; +#ifdef CONFIG_X86_LOCAL_APIC
Quoted 8 lines
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1372
Quoted 5 lines
arch/x86/xen/enlighten.c +5
-0+/* +* set up the basic apic ops. +*/ +set_xen_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write;I'd just change the
xen_apic_writeto remove theWARN_ON, since you don't seem to care about it either.
:)
Quoted 111 lines
Quoted 111 lines
arch/x86/xen/enlighten.c +12
-0+#endif + + callback_via =HVM_CALLBACK_VECTOR (X86_PLATFORM_IPI_VECTOR ); +set_callback_via (callback_via); + + x86_platform_ipi_callback = do_hvm_pv_evtchn_intr; + +disable_acpi (); + +xen_hvm_pv_smp_init (); + machine_ops = xen_machine_ops; + }#endif }arch/x86/xen/irq.c +35
-2@@ -5,6 +5,7 @@#include <xen/interface/xen.h> #include <xen/interface/sched.h> #include <xen/interface/vcpu.h> +#include <xen/xen.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> @@ -132,6 +133,20 @@ void __init xen_init_irq_ops() x86_init.irqs.intr_init = xen_init_IRQ; } +static void xen_hvm_pv_evtchn_disable (void ) +{ +native_irq_disable (); +xen_irq_disable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_disable); + +static void xen_hvm_pv_evtchn_enable (void ) +{ +native_irq_enable (); +xen_irq_enable (); +} +PV_CALLEE_SAVE_REGS_THUNK (xen_hvm_pv_evtchn_enable); +static void xen_hvm_pv_safe_halt (void ) {/* Do local_irq_enable() explicitly in hvm_pv guest */ @@ -147,8 +162,26 @@ static void xen_hvm_pv_halt(void)xen_hvm_pv_safe_halt (); } +static const struct pv_irq_ops xen_hvm_pv_irq_ops__initdata = { + .save_fl =__PV_IS_CALLEE_SAVE (native_save_fl), + .restore_fl =__PV_IS_CALLEE_SAVE (native_restore_fl), + .irq_disable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_disable), + .irq_enable =PV_CALLEE_SAVE (xen_hvm_pv_evtchn_enable), + + .safe_halt = xen_hvm_pv_safe_halt, + .halt = xen_hvm_pv_halt, +#ifdef CONFIG_X86_64 + .adjust_exception_frame = paravirt_nop, +#endif +}; +void __init xen_hvm_pv_init_irq_ops (void ) { - pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; - pv_irq_ops.halt = xen_hvm_pv_halt; +if (xen_hvm_pv_evtchn_enabled ()) { + pv_irq_ops = xen_hvm_pv_irq_ops; + x86_init.irqs.intr_init = xen_hvm_pv_evtchn_init_IRQ; + }else { + pv_irq_ops.safe_halt = xen_hvm_pv_safe_halt; + pv_irq_ops.halt = xen_hvm_pv_halt; + } }arch/x86/xen/smp.c +6
-0@@ -15,20 +15,26 @@#include <linux/sched.h> #include <linux/err.h> #include <linux/smp.h> +#include <linux/nmi.h> #include <asm/paravirt.h> #include <asm/desc.h> #include <asm/pgtable.h> #include <asm/cpu.h> +#include <asm/trampoline.h> +#include <asm/tlbflush.h> +#include <asm/mtrr.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <asm/xen/hypervisor.h> #include <xen/page.h> #include <xen/events.h> +#include <xen/xen.h> #include "xen-ops.h" #include "mmu.h" @@ -171,7 +177,8 @@ static void __init xen_smp_prepare_boot_cpu(void)
Quoted 7 lines
arch/x86/xen/smp.c static void __init xen_smp_prepare_boot_cpu(void) · line 181
Quoted 5 lines
arch/x86/xen/smp.c +2
-1/* We've switched to the "real" per-cpu gdt, so make sure the old memory can be recycled */ -make_lowmem_page_readwrite (xen_initial_gdt); +if (xen_pv_domain ()) +make_lowmem_page_readwrite (xen_initial_gdt);Test for
xen_feature(XENFEAT_writable_descriptor_tables).
Sure.
Quoted 33 lines
Quoted 33 lines
arch/x86/xen/smp.c +26
-0xen_setup_vcpu_info_placement (); }@@ -480,3 +487,138 @@ void __init xen_smp_init(void)xen_fill_possible_map ();xen_init_spinlocks (); } + +static __cpuinit void xen_hvm_pv_start_secondary (void ) +{ +int cpu =smp_processor_id (); + +cpu_init (); +touch_nmi_watchdog (); +preempt_disable (); + +/* otherwise gcc will move up smp_processor_id before the cpu_init */ +barrier (); +/* +* Check TSC synchronization with the BSP: +*/ +check_tsc_sync_target (); + +/* Done in smp_callin(), move it here */ +set_mtrr_aps_delayed_init (); +smp_store_cpu_info (cpu); + +/* This must be done before setting cpu_online_mask */ +set_cpu_sibling_map (cpu); +wmb (); + +set_cpu_online (smp_processor_id (), true); +per_cpu (cpu_state,smp_processor_id ()) =CPU_ONLINE ;
Quoted 8 lines
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 520
Quoted 5 lines
arch/x86/xen/smp.c +5
-0+ +/* enable local interrupts */ +local_irq_enable (); + +xen_setup_cpu_clockevents ();How much of this is necessary? At this point, isn't CPU bringup the same as PV?
Xen_enable_sysenter/syscall is not needed for this. And we have a TSC sync here - but it seems unnecessary for PV. But set_mtrr_aps_delayed_init() is needed. Reuse the cpu_bring_up() is fine?
Quoted 54 lines
Quoted 54 lines
arch/x86/xen/smp.c +7
-0+ +wmb (); +cpu_idle (); +} + +static __cpuinit int +hvm_pv_cpu_initialize_context(unsigned int cpu, struct task_struct*idle) +{
arch/x86/xen/smp.c +46
-0+struct vcpu_guest_context *ctxt; +unsigned long start_ip; + +if (cpumask_test_and_set_cpu (cpu, xen_cpu_initialized_map)) +return 0 ; + + ctxt =kzalloc (sizeof (*ctxt),GFP_KERNEL ); +if (ctxt ==NULL ) +return -ENOMEM ; + + early_gdt_descr.address = (unsigned long )get_cpu_gdt_table (cpu); + initial_code = (unsigned long )xen_hvm_pv_start_secondary; + stack_start.sp = (void *) idle->thread.sp; + +/* start_ip had better be page-aligned! */ + start_ip =setup_trampoline (); + +/* only start_ip is what we want */ + ctxt->flags =VGCF_HVM_GUEST ; + ctxt->user_regs.eip = start_ip; + +printk (KERN_INFO "Booting processor %d ip 0x%lx\n" , cpu, start_ip); + +if (HYPERVISOR_vcpu_op (VCPUOP_initialise, cpu, ctxt)) +BUG (); + +kfree (ctxt); +return 0 ; +} + +static int __init xen_hvm_pv_cpu_up (unsigned int cpu) +{ +struct task_struct *idle =idle_task (cpu); +int rc; +unsigned long flags; + +per_cpu (current_task, cpu) = idle; + +#ifdef CONFIG_X86_32 +irq_ctx_init (cpu); +#else +clear_tsk_thread_flag (idle,TIF_FORK ); + initial_gs =per_cpu_offset (cpu); +per_cpu (kernel_stack, cpu) = + (unsigned long )task_stack_page (idle) - +KERNEL_STACK_OFFSET +THREAD_SIZE ;
Quoted 8 lines
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 580
Quoted 6 lines
arch/x86/xen/smp.c +6
-0+#endif + +xen_setup_timer (cpu); +xen_init_lock_cpu (cpu); + +per_cpu (cpu_state, cpu) =CPU_UP_PREPARE ;Can you put all this duplicated code into a common function?
Sure.
Quoted 32 lines
Quoted 32 lines
arch/x86/xen/smp.c +31
-0+ + rc =hvm_pv_cpu_initialize_context (cpu, idle); +if (rc) +return rc; + +if (num_online_cpus () ==1 ) +alternatives_smp_switch (1 ); + + rc =xen_smp_intr_init (cpu); +if (rc) +return rc; + + rc =HYPERVISOR_vcpu_op (VCPUOP_up, cpu,NULL ); +BUG_ON (rc); + +/* +* Check TSC synchronization with the AP (keep irqs disabled +* while doing so): +*/ +local_irq_save (flags); +check_tsc_sync_source (cpu); +local_irq_restore (flags); + +while (!cpu_online (cpu)) { +cpu_relax (); +touch_nmi_watchdog (); + } + +return 0 ; +} +
Quoted 8 lines
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 616
Quoted 5 lines
arch/x86/xen/smp.c +5
-0+static void xen_hvm_pv_flush_tlb_others (const struct cpumask *cpumask, +struct mm_struct *mm,unsigned long va) +{ +/* TODO Make it more specific */ +flush_tlb_all ();Is the MMUEXT_TLB_FLUSH/INVLPG_MULTI hypercall not currently available to HVM?
I think they are different. These hypercalls flushed native's TLB, but HVM want to flush guest one, especially when using shadow, HVM need do something for it.
Quoted 8 lines
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 620
Quoted 4 lines
arch/x86/xen/smp.c +4
-0+} + +void __init xen_hvm_pv_smp_init (void ) +{It's probably safest to have this do its own test for
xen_hvm_pv_evtchn_enabled(), rather than assuming the caller is getting it right.
Yes.
Quoted 51 lines
Quoted 51 lines
arch/x86/xen/smp.c +4
-0+ smp_ops = xen_smp_ops; + smp_ops.cpu_up = xen_hvm_pv_cpu_up; + pv_mmu_ops.flush_tlb_others = xen_hvm_pv_flush_tlb_others; +}arch/x86/xen/xen-ops.h +3
-0@@ -34,6 +34,7 @@ void xen_reserve_top(void);char *__init xen_memory_setup (void );void __init xen_arch_setup (void );void __init xen_init_IRQ (void ); +void __init xen_hvm_pv_evtchn_init_IRQ (void );void xen_enable_sysenter (void );void xen_enable_syscall (void );void xen_vcpu_restore (void );@@ -61,10 +62,12 @@ void xen_setup_vcpu_info_placement(void);#ifdef CONFIG_SMP void xen_smp_init (void ); +void xen_hvm_pv_smp_init (void );extern cpumask_var_t xen_cpu_initialized_map;#else static inline void xen_smp_init (void ) {} +static inline void xen_hvm_pv_smp_init (void ) {}#endif #ifdef CONFIG_PARAVIRT_SPINLOCKS drivers/xen/events.c +3
-0@@ -37,9 +37,12 @@#include <xen/xen-ops.h> #include <xen/events.h> +#include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/interface/event_channel.h> +#include <asm/desc.h> +/* * This lock protects updates to the following mapping and reference-count * arrays. The lock does not need to be acquired to read the mapping tables. @@ -624,8 +627,13 @@ void xen_evtchn_do_upcall(struct
pt_regs *regs) struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu); unsigned count;
Quoted 14 lines
drivers/xen/events.c void xen_evtchn_do_upcall(struct pt_regs *regs) · line 636
Quoted 9 lines
drivers/xen/events.c +7
-2-exit_idle (); -irq_enter (); +/* +* If is PV featured HVM, these have already been done +*/ +if (likely (!xen_hvm_pv_evtchn_enabled ())) { +exit_idle (); +irq_enter (); + }In that case, rather than putting this conditional in the hot path, make an inner
__xen_evtchn_do_upcallwhich is wrapped by the PV and HVM variants which do the appropriate things. (And drop thept_regsarg, I think.)
Sure.
Signature · 2 lines
regards Yang, Sheng
Quoted 110 lines
Quoted 97 lines
do { unsigned long pending_words;drivers/xen/events.c +52
-2@@ -662,8 +670,10 @@ void xen_evtchn_do_upcall(struct pt_regs *regs) }while (count !=1 ); out: -irq_exit (); -set_irq_regs (old_regs); +if (likely (!xen_hvm_pv_evtchn_enabled ())) { +irq_exit (); +set_irq_regs (old_regs); + }put_cpu (); }@@ -944,3 +954,51 @@ void __init xen_init_IRQ(void)irq_ctx_init (smp_processor_id ()); } + +void __init xen_hvm_pv_evtchn_init_IRQ (void ) +{ +int i; + +xen_init_IRQ (); +for (i =0 ; i<NR_IRQS_LEGACY ; i++) { +struct evtchn_bind_virq bind_virq; +struct irq_desc *desc =irq_to_desc (i); +int virq, evtchn; + + virq = i +VIRQ_EMUL_PIN_START ; + bind_virq.virq = virq; + bind_virq.vcpu =0 ; + +if (HYPERVISOR_event_channel_op (EVTCHNOP_bind_virq, + &bind_virq) !=0 ) +BUG (); + + evtchn = bind_virq.port; + evtchn_to_irq[evtchn] = i; + irq_info[i] =mk_virq_info (evtchn, virq); + + desc->status =IRQ_DISABLED ; + desc->action =NULL ; + desc->depth =1 ; + +/* +* 16 old-style INTA-cycle interrupts: +*/ +set_irq_chip_and_handler_name (i,&xen_dynamic_chip, + handle_level_irq,"event" ); + } + +/* +* Cover the whole vector space, no vector can escape +* us. (some of these will be overridden and become +* 'special' SMP interrupts) +*/ +for (i =0 ; i< (NR_VECTORS -FIRST_EXTERNAL_VECTOR ); i++) { +int vector =FIRST_EXTERNAL_VECTOR + i; +if (vector !=IA32_SYSCALL_VECTOR ) +set_intr_gate (vector, interrupt[i]); + } + +/* generic IPI for platform specific use, now used for HVM evtchn */ +alloc_intr_gate (X86_PLATFORM_IPI_VECTOR , x86_platform_ipi); +}include/xen/events.h +1
-0@@ -56,4 +56,5 @@ void xen_poll_irq(int irq);/* Determine the IRQ which is bound to an event channel */ unsigned irq_from_evtchn (unsigned int evtchn); +void xen_evtchn_do_upcall (struct pt_regs *regs);#endif /* _XEN_EVENTS_H */ include/xen/hvm.h +5
-0@@ -20,4 +20,9 @@ static inline unsigned long hvm_get_parameter(int idx)return xhv.value; } +#define HVM_CALLBACK_VIA_TYPE_VECTOR 0x2 +#define HVM_CALLBACK_VIA_TYPE_SHIFT 56 +#define HVM_CALLBACK_VECTOR (x) (((uint64_t )HVM_CALLBACK_VIA_TYPE_VECTOR )<<\ +HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +#endif /* XEN_HVM_H__ */include/xen/interface/xen.h +0
-0include/xen/interface/xen.h line 96
Quoted 8 lines
include/xen/interface/xen.h +4
-1@@ -90,7 +90,11 @@#define VIRQ_ARCH_6 22#define VIRQ_ARCH_7 23 -#define NR_VIRQS 24 +#define VIRQ_EMUL_PIN_START 24 +#define VIRQ_EMUL_PIN_NUM 16 + +#define NR_VIRQS 40(
VIRQ_EMUL_PIN_START+VIRQ_EMUL_PIN_NUM)?J
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Sheng YangQuoted 4 lines · Sheng Yang
A annoy thing in pv drivers is that it would test if the domain type is _NOT_
XEN_NATIVE. So set the domain toXEN_HVM_DOMAINwould result in PV driver initialization then probably panic.
What _actually_ panics?
Registration of the frontend devices should be completely harmless (apart from a little wasted RAM) unless a xenbus driver manages to come up and enumerate the xen bus and cause the ->probe function run.
You should be gating the xenbus startup on the availability of PV functionality not the individual driver registrations. This keeps the test in a single easy to maintain place.
Compare with pci_register_driver and all of the callers of that function -- not a single one of them has an "is_pci_available" test anywhere.
Ian.
Re: [LKML] [PATCH 7/7] xen: Enable grant table and xenbus for PV extension of HVM
In reply to Sheng YangQuoted 21 lines · Sheng Yang
Quoted 14 lines · On Tuesday 02 March 2010 01:38:58 Konrad Rzeszutek Wilk wrote:
drivers/xen/grant-table.c static int gnttab_map(unsigned int start_idx, unsigned int end_idx) · line 501
Quoted 3 lines
drivers/xen/grant-table.c +3
-0+/* The region reserved by QEmu for Xen platform device */ +#define GNTTAB_START 0xf2000000ul +#define GNTTAB_SIZE 0x20000ulI thought that in the earlier review you said:
"> > +#define
GNTTAB_START0xfbfe0000ulQuoted 5 lines
Quoted 1 line
+#define
GNTTAB_SIZE0x20000ulIs it possible that there would be a PCI device that would be passed in the guest that would conflict with the above mentioned E820 region?
Quoted 5 lines
I would change them to a dedicated PCI MMIO address in the next version. Thanks.
"
?
Quoted 7 lines
And yes, this is the dedicated PCI MMIO address I mentioned.. I would update the comments to get it more clear.
I don't think it's very clear solution, because the real good way to do this is probe pci device and find out with one is the platform pci device then use it. But the grant table initialization is quite earlier compared to the possible probing now... I hardcode the position now, and hunting for a better
Would it be possible to move the grant table initialization to later phase? Past the PCI loading/initialization?
Quoted 8 lines
idea.
Comments?
Signature · 2 lines
regards Yang, Sheng
Re: [LKML] [PATCH 7/7] xen: Enable grant table and xenbus for PV extension of HVM
Quoted 25 lines · Konrad Rzeszutek Wilk
Quoted 24 lines · Sheng Yang
Quoted 14 lines · On Tuesday 02 March 2010 01:38:58 Konrad Rzeszutek Wilk wrote:
drivers/xen/grant-table.c static int gnttab_map(unsigned int start_idx, unsigned int end_idx) · line 501
Quoted 3 lines
drivers/xen/grant-table.c +3
-0+/* The region reserved by QEmu for Xen platform device */ +#define GNTTAB_START 0xf2000000ul +#define GNTTAB_SIZE 0x20000ulI thought that in the earlier review you said:
"> > +#define
GNTTAB_START0xfbfe0000ulQuoted 5 lines
Quoted 1 line
+#define
GNTTAB_SIZE0x20000ulIs it possible that there would be a PCI device that would be passed in the guest that would conflict with the above mentioned E820 region?
Quoted 5 lines
I would change them to a dedicated PCI MMIO address in the next version. Thanks.
"
?And yes, this is the dedicated PCI MMIO address I mentioned.. I would update the comments to get it more clear.
Quoted 7 lines
Quoted 4 lines
I don't think it's very clear solution, because the real good way to do this is probe pci device and find out with one is the platform pci device then use it. But the grant table initialization is quite earlier compared to the possible probing now... I hardcode the position now, and hunting for a better
Would it be possible to move the grant table initialization to later phase? Past the PCI loading/initialization?
Or provide the address via an MSR, hypervisor specific CPUID leaf or IO port or by using the early_pci infrastructure in the kernel. I don't think we are short of options ;-)
Ian.
Quoted 9 lines
Quoted 9 lines
idea.
Comments?
Signature · 2 lines
regards Yang, Sheng
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Ian CampbellQuoted 12 lines · Ian Campbell
Quoted 4 lines · Sheng Yang
A annoy thing in pv drivers is that it would test if the domain type is _NOT_
XEN_NATIVE. So set the domain toXEN_HVM_DOMAINwould result in PV driver initialization then probably panic.What _actually_ panics?
Registration of the frontend devices should be completely harmless (apart from a little wasted RAM) unless a xenbus driver manages to come up and enumerate the xen bus and cause the ->probe function run.
Quoted 7 lines
You should be gating the xenbus startup on the availability of PV functionality not the individual driver registrations. This keeps the test in a single easy to maintain place.
Compare with
pci_register_driverand all of the callers of that function -- not a single one of them has an "is_pci_available" test anywhere.
The problem is that it currently assumes xenbus is initialized by the time the PV drivers init, because in a PV boot xenbus gets initted very early. We need to change it so that it can cope with drivers being initialized and registering with xenbus before it has been initialized.
We have the same problem with plain PV-on-HVM drivers as xenbus only comes up as a result of probing the Xen PCI platform device, which may be after the PV drivers' init functions have been called.
J
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
Quoted 21 lines · Jeremy Fitzhardinge
Quoted 12 lines · Ian Campbell
Quoted 4 lines · Sheng Yang
A annoy thing in pv drivers is that it would test if the domain type is _NOT_
XEN_NATIVE. So set the domain toXEN_HVM_DOMAINwould result in PV driver initialization then probably panic.What _actually_ panics?
Registration of the frontend devices should be completely harmless (apart from a little wasted RAM) unless a xenbus driver manages to come up and enumerate the xen bus and cause the ->probe function run.
Quoted 7 lines
You should be gating the xenbus startup on the availability of PV functionality not the individual driver registrations. This keeps the test in a single easy to maintain place.
Compare with
pci_register_driverand all of the callers of that function -- not a single one of them has an "is_pci_available" test anywhere.
Quoted 8 lines
The problem is that it currently assumes xenbus is initialized by the time the PV drivers init, because in a PV boot xenbus gets initted very early. We need to change it so that it can cope with drivers being initialized and registering with xenbus before it has been initialized.
We have the same problem with plain PV-on-HVM drivers as xenbus only comes up as a result of probing the Xen PCI platform device, which may be after the PV drivers' init functions have been called.
Give a look at the fourth patch of the series I just posted: it introduces a simple driver for the Xen PCI platform device to initialize xenbus and gran table later on when running in a HVM domain, at the same time leaving the PV case as it is.
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
Quoted 5 lines · Stefano Stabellini
Give a look at the fourth patch of the series I just posted: it introduces a simple driver for the Xen PCI platform device to initialize xenbus and gran table later on when running in a HVM domain, at the same time leaving the PV case as it is.
Yes, I noticed that. Do the PV drivers cope with having the PCI probe/xenbus setup after their init functions have been called, or do you make sure the PCI probe happens early?
J
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
Quoted 11 lines · Jeremy Fitzhardinge
Quoted 5 lines · Stefano Stabellini
Give a look at the fourth patch of the series I just posted: it introduces a simple driver for the Xen PCI platform device to initialize xenbus and gran table later on when running in a HVM domain, at the same time leaving the PV case as it is.
Yes, I noticed that. Do the PV drivers cope with having the PCI probe/xenbus setup after their init functions have been called, or do you make sure the PCI probe happens early?
Nope, they seem to cope fine so far.
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
In reply to Sheng YangQuoted 7 lines · Sheng Yang
Quoted 3 lines
Presumably even if we don't have
PV_EVTCHNavailable/enabled, the Xen clocksource would be available for getting time?I think currently Xen pv clocksource and clockevent are binding... Not sure if a single line "
clocksource_register(&xen_clocksource)" can work. I would give it a try, maybe add a new PV feature.
There should be no strong binding between them, but there may be some sloppy assumptions in xen/time.c which should be fixed. Linux itself treats clocksources and eventsources as completely distinct entities, and doesn't assume they're running on the same timebase (for example).
Having a PV clocksource even if the timer interrupts are emulated would make sense and be useful.
Quoted 41 lines
Quoted 32 lines
Quoted 32 lines
xen_setup_vcpu_info_placement(); }arch/x86/xen/smp.c +26
-0@@ -480,3 +487,138 @@ void __init xen_smp_init(void)xen_fill_possible_map ();xen_init_spinlocks (); } + +static __cpuinit void xen_hvm_pv_start_secondary (void ) +{ +int cpu =smp_processor_id (); + +cpu_init (); +touch_nmi_watchdog (); +preempt_disable (); + +/* otherwise gcc will move up smp_processor_id before the cpu_init */ +barrier (); +/* +* Check TSC synchronization with the BSP: +*/ +check_tsc_sync_target (); + +/* Done in smp_callin(), move it here */ +set_mtrr_aps_delayed_init (); +smp_store_cpu_info (cpu); + +/* This must be done before setting cpu_online_mask */ +set_cpu_sibling_map (cpu); +wmb (); + +set_cpu_online (smp_processor_id (), true); +per_cpu (cpu_state,smp_processor_id ()) =CPU_ONLINE ;Quoted 9 lines
arch/x86/xen/smp.c void __init xen_smp_init(void) · line 520
Quoted 6 lines
arch/x86/xen/smp.c +5
-0+ +/* enable local interrupts */ +local_irq_enable (); + +xen_setup_cpu_clockevents ();How much of this is necessary? At this point, isn't CPU bringup the same as PV?
Quoted 4 lines
Xen_enable_sysenter/syscall is not needed for this. And we have a TSC sync here - but it seems unnecessary for PV. But
set_mtrr_aps_delayed_init()is needed. Reuse thecpu_bring_up()is fine?
Doesn't Xen arrange for the tscs to be synced anyway?
Quoted 7 lines
Quoted 3 lines
Is the MMUEXT_TLB_FLUSH/INVLPG_MULTI hypercall not currently available to HVM?
I think they are different. These hypercalls flushed native's TLB, but HVM want to flush guest one, especially when using shadow, HVM need do something for it.
I see.
J
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
In reply to Jeremy FitzhardingeQuoted 16 lines · On Tuesday 02 March 2010 09:38:21 Jeremy Fitzhardinge wrote:
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1372
Quoted 12 lines · Sheng Yang
arch/x86/xen/enlighten.c +12
-0+ + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + + pv_apic_ops = xen_apic_ops; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +set_xen_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write;I'd just change the
xen_apic_writeto remove theWARN_ON, since you don't seem to care about it either.
So which code base I should make these patches against? We expect the patchset can be accepted in the Linux upstream soon after you pick it up.
Quoted 16 lines
drivers/xen/events.c void xen_evtchn_do_upcall(struct pt_regs *regs) · line 636
Quoted 11 lines
drivers/xen/events.c +7
-2-exit_idle (); -irq_enter (); +/* +* If is PV featured HVM, these have already been done +*/ +if (likely (!xen_hvm_pv_evtchn_enabled ())) { +exit_idle (); +irq_enter (); + }In that case, rather than putting this conditional in the hot path, make an inner
__xen_evtchn_do_upcallwhich is wrapped by the PV and HVM variants which do the appropriate things. (And drop thept_regsarg, I think.)
Seems we still need pt_regs to for handle_irq()?
Signature · 2 lines
regards Yang, Sheng
Re: [Xen-devel] [PATCH 3/7] xen/hvm: Xen PV extension of HVM initialization
In reply to Stefano StabelliniQuoted 8 lines · Stefano Stabellini
Quoted 7 lines · Jeremy Fitzhardinge
Quoted 5 lines · Stefano Stabellini
Give a look at the fourth patch of the series I just posted: it introduces a simple driver for the Xen PCI platform device to initialize xenbus and gran table later on when running in a HVM domain, at the same time leaving the PV case as it is.
Quoted 6 lines
Quoted 4 lines
Yes, I noticed that. Do the PV drivers cope with having the PCI probe/xenbus setup after their init functions have been called, or do you make sure the PCI probe happens early?
Nope, they seem to cope fine so far.
xenbus_frontend_register is basically just driver_register so it should be safe to call even before the register_bus(xenbusfoo) call. Things shouldn't kick off until xen_probe() is called which either happens explicitly in xenbus_probe_init() or via probe_work which contains a xenstored_ready check.
Ian.
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
In reply to Sheng YangQuoted 18 lines · Sheng Yang
Quoted 16 lines · On Tuesday 02 March 2010 09:38:21 Jeremy Fitzhardinge wrote:
arch/x86/xen/enlighten.c void __init xen_guest_init(void) · line 1372
Quoted 12 lines · Sheng Yang
arch/x86/xen/enlighten.c +12
-0+ + x86_platform.calibrate_tsc = xen_tsc_khz; + x86_platform.get_wallclock = xen_get_wallclock; + x86_platform.set_wallclock = xen_set_wallclock; + + pv_apic_ops = xen_apic_ops; +#ifdef CONFIG_X86_LOCAL_APIC +/* +* set up the basic apic ops. +*/ +set_xen_basic_apic_ops (); + apic->write = xen_hvm_pv_evtchn_apic_write;I'd just change the
xen_apic_writeto remove theWARN_ON, since you don't seem to care about it either.
Quoted 2 lines
So which code base I should make these patches against? We expect the patchset can be accepted in the Linux upstream soon after you pick it up.
I think that once we agree on which approach we take we should work together on this.
Jeremy, can we have our own temporary branch so that we can work together on a common code base to finish this patch series?
Re: [Xen-devel] [PATCH 5/7] xen: Make event channel work with PV extension of HVM
Quoted 3 lines · Stefano Stabellini
Jeremy, can we have our own temporary branch so that we can work together on a common code base to finish this patch series?
I'm not sure quite what you mean, but I've committed Sheng's work to xen/pvhvm-sheng for now, and I'll commit your patches to a similar branch once I know where to apply them.
J