Supporting Enlightened Windows 2008 Server
RFC PATCH 1/1 Supporting Enlightened Windows 2008 Server
During the spring Xen Summit in New York, I presented our work on hosting Enlightened windows 2008 server on Xen based platforms. Now that Microsoft has published their Hypervisor specification, we can release our code to the community. The goal of this effort has been to host Enlightened windows 2008 server as a guest on a Xen based platform. We have addressed this requirement by :
A) Leveraging the I/O framework in the Xen based platform. Appropriate PV drivers will be loaded up in the guest to improve the I/O performance.
B) Implement an adapter that implements the required Hyper-V functionality.
We have implemented only a subset of Hyper-V functionality that is required for enlightened windows 2008 guest today. However, we have the framework in place to implement any additional functionality that the windows guests may leverage going forward. The framework is extensible and one can easily implement OS specific enlightenments.
I am enclosing three patches that implement our adapter for your consideration:
1) ns_tools.patch: Modifications to xen tools to support the the adapter
2) ns_xen_base.patch: Modifications to the base Xen code to support the adapter
3) ns_xen_extension.patch: New code implementing the adapter
These patches have been tested on a xen 3.2 based system (SLES10 SP2).
Signed-off by K. Y. Srinivasan ([email protected])
Regards,
K. Y
tools/python/xen/lowlevel/xc/xc.c +5 -4
@@ -632,14 +632,14 @@ static PyObject *pyxc_hvm_build(XcObjectint i;#endif char *image; -int memsize, vcpus =1 , acpi =0 , apic =1 ; +int memsize, vcpus =1 , acpi =0 , apic =1 , extid =0 ;static char *kwd_list[] = {"domid" , -"memsize" ,"image" ,"vcpus" ,"acpi" , +"memsize" ,"image" ,"vcpus" ,"extid" ,"acpi" ,"apic" ,NULL }; -if ( !PyArg_ParseTupleAndKeywords (args, kwds,"iis|iii" , kwd_list, +if ( !PyArg_ParseTupleAndKeywords (args, kwds,"iis|iiii" , kwd_list, &dom, &memsize, - &image, &vcpus, &acpi, &apic) ) + &image, &vcpus, &extid, &acpi, &apic) )return NULL ;if (xc_hvm_build (self->xc_handle, dom, memsize, image) !=0 )
@@ -664,6 +664,7 @@ static PyObject *pyxc_hvm_build(XcObject va_hvm->checksum = -sum;munmap (va_map,XC_PAGE_SIZE );#endif +xc_set_hvm_param (self->xc_handle, dom,HVM_PARAM_EXTEND_HYPERVISOR , extid);return Py_BuildValue ("{}" ); }
tools/python/xen/xend/XendConfig.py +1 -1
@@ -126,7 +126,7 @@ LEGACY_CFG_TO_XENAPI_CFG = reverse_dict( # Platform configuration keys. XENAPI_PLATFORM_CFG = [ 'acpi', 'apic', 'boot', 'device_model', 'display', 'fda', 'fdb', 'keymap', 'isa', 'localtime', 'monitor', - 'nographic', 'pae', 'rtc_timeoffset', 'serial', 'sdl', + 'nographic', 'pae', 'extid', 'rtc_timeoffset', 'serial', 'sdl', 'soundhw','stdvga', 'usb', 'usbdevice', 'vnc', 'vncconsole', 'vncdisplay', 'vnclisten', 'timer_mode', 'vncpasswd', 'vncunused', 'xauthority', 'pci', 'vhpt',
tools/python/xen/xend/image.py +3 -0
@@ -426,6 +426,7 @@ class HVMImageHandler(ImageHandler): self.apic = int(vmConfig['platform'].get('apic', 0)) self.acpi = int(vmConfig['platform'].get('acpi', 0)) + self.extid = int(vmConfig['platform'].get('extid', 0)) self.guest_os_type = vmConfig['platform'].get('guest_os_type') # Return a list of cmd line args to the device models based on the
@@ -516,6 +517,7 @@ class HVMImageHandler(ImageHandler): log.debug("store_evtchn = %d", store_evtchn) log.debug("memsize = %d", mem_mb) log.debug("vcpus = %d", self.vm.getVCpuCount()) + log.debug("extid = %d", self.extid) log.debug("acpi = %d", self.acpi) log.debug("apic = %d", self.apic)
@@ -523,6 +525,7 @@ class HVMImageHandler(ImageHandler): image = self.kernel, memsize = mem_mb, vcpus = self.vm.getVCpuCount(), + extid = self.extid, acpi = self.acpi, apic = self.apic) rc['notes'] = { 'SUSPEND_CANCEL': 1 }
tools/python/xen/xm/create.py +5 -1
@@ -199,6 +199,10 @@ gopts.var('timer_mode', val='TIMER_MODE' use="""Timer mode (0=delay virtual time when ticks are missed; 1=virtual time is always wallclock time.""") +gopts.var('extid', val='EXTID', + fn=set_int, default=0, + use="Specify extention ID for a HVM domain.") + gopts.var('acpi', val='ACPI', fn=set_int, default=1, use="Disable or enable ACPI of HVM domain.")
@@ -719,7 +723,7 @@ def configure_vifs(config_devs, vals): def configure_hvm(config_image, vals): """Create the config for HVM devices. """ - args = [ 'device_model', 'pae', 'vcpus', 'boot', 'fda', 'fdb', 'timer_mode', + args = [ 'device_model', 'pae', 'extid', 'vcpus', 'boot', 'fda', 'fdb', 'timer_mode', 'localtime', 'serial', 'stdvga', 'isa', 'nographic', 'soundhw', 'vnc', 'vncdisplay', 'vncunused', 'vncconsole', 'vnclisten', 'sdl', 'display', 'xauthority', 'rtc_timeoffset', 'monitor',
%patch
xen/arch/x86/hvm/svm/svm.c +15 -1
@@ -50,6 +50,7 @@#include <asm/hvm/vpt.h> #include <asm/hvm/trace.h> #include <asm/hap.h> +#include <asm/hvm/hvm_extensions.h> u32 svm_feature_flags;
@@ -73,6 +74,7 @@/* vmcb used for extended host state */ static void *root_vmcb[NR_CPUS ]__read_mostly ; +static void inline __update_guest_eip (struct cpu_user_regs *regs,unsigned int inst_len) {
@@ -882,7 +884,7 @@ .set_tsc_offset = svm_set_tsc_offset, .inject_exception = svm_inject_exception, .init_hypercall_page = svm_init_hypercall_page, - .event_pending = svm_event_pending + .event_pending = svm_event_pending, };int start_svm (struct cpuinfo_x86 *c)
@@ -1044,6 +1046,7 @@HVMTRACE_3D (CPUID , v, input, ((uint64_t )eax <<32 ) | ebx, ((uint64_t )ecx <<32 ) | edx); +ext_intercept_do_cpuid (input, regs); inst_len =__get_instruction_length (v,INSTR_CPUID ,NULL );__update_guest_eip (regs, inst_len); }
@@ -1739,6 +1742,11 @@/* is it a read? */ if (vmcb->exitinfo1 ==0 ) { +if (ext_intercept_do_msr_read (ecx, regs)) + { + goto done; + } +switch (ecx) {case MSR_IA32_TSC : msr_content =hvm_get_guest_time (v);
@@ -1829,6 +1837,11 @@ }else { +if (ext_intercept_do_msr_write (ecx, regs)) + { + goto done_1; + } + msr_content = (u32 )regs->eax | ((u64 )regs->edx <<32 );hvmtrace_msr_write (v, ecx, msr_content);
@@ -1889,6 +1902,7 @@ }break ; } +done_1: inst_len =__get_instruction_length (v,INSTR_WRMSR ,NULL ); }
xen/arch/x86/hvm/vmx/vmx.c +13 -2
@@ -50,6 +50,7 @@#include <asm/hvm/vpt.h> #include <public/hvm/save.h> #include <asm/hvm/trace.h> +#include <asm/hvm/hvm_extensions.h> enum handler_return { HNDL_done, HNDL_unhandled, HNDL_exception_raised };
@@ -62,6 +63,7 @@static void vmx_update_guest_cr (struct vcpu *v,unsigned int cr);static void vmx_update_guest_efer (struct vcpu *v); +static int vmx_domain_initialise (struct domain *d) {return vmx_alloc_vlapic_mapping (d);
@@ -1238,7 +1240,8 @@unsigned int count = *ecx;#ifdef VMXASSIST -if ( input ==0x40000003 ) +if (( input ==0x40000003 ) && + (vmx_guest_x86_mode (current) ==0 )) {/* * NB. Unsupported interface for private use of VMXASSIST only.
@@ -1319,12 +1322,13 @@static void vmx_do_cpuid (struct cpu_user_regs *regs) { -unsigned int eax, ebx, ecx, edx; +unsigned int eax, ebx, ecx, edx, input; eax = regs->eax; ebx = regs->ebx; ecx = regs->ecx; edx = regs->edx; + input = eax;vmx_cpuid_intercept (&eax, &ebx, &ecx, &edx);
@@ -1332,6 +1336,7 @@ regs->ebx = ebx; regs->ecx = ecx; regs->edx = edx; +ext_intercept_do_cpuid (input, regs); }#define CASE_GET_REG_P (REG , reg) \
@@ -2316,6 +2321,9 @@HVM_DBG_LOG (DBG_LEVEL_1 ,"ecx=%x" , ecx); +if (ext_intercept_do_msr_read (ecx, regs)) + goto done; +switch ( ecx ) {case MSR_IA32_TSC :
@@ -2499,6 +2507,9 @@HVM_DBG_LOG (DBG_LEVEL_1 ,"ecx=%x, eax=%x, edx=%x" , ecx, (u32 )regs->eax, (u32 )regs->edx); +if (ext_intercept_do_msr_write (ecx, regs)) +return 1 ; + msr_content = (u32 )regs->eax | ((u64 )regs->edx <<32 );hvmtrace_msr_write (v, ecx, msr_content);
xen/include/asm-x86/hvm/domain.h +4 -0
@@ -73,6 +73,10 @@/* Pass-through */ struct hvm_iommu hvm_iommu; +/* Hvm extension handle */ +void *ext_handle;/* will be NULL on creation (memset)*/ + struct extension_intercept_vector *ext_vector; + };#endif /* __ASM_X86_HVM_DOMAIN_H__ */
xen/include/public/hvm/params.h +8 -5
@@ -50,10 +50,12 @@#define HVM_PARAM_BUFIOREQ_PFN 6 +#define HVM_PARAM_EXTEND_HYPERVISOR 7 +#ifdef __ia64__ -#define HVM_PARAM_NVRAM_FD 7 -#define HVM_PARAM_VHPT_SIZE 8 -#define HVM_PARAM_BUFPIOREQ_PFN 9 +#define HVM_PARAM_NVRAM_FD 8 +#define HVM_PARAM_VHPT_SIZE 9 +#define HVM_PARAM_BUFPIOREQ_PFN 10#endif /*
@@ -75,12 +77,13 @@* Missed interrupts are collapsed together and delivered as one 'late tick'. * Guest time always tracks wallclock (i.e., real) time. */ -#define HVM_PARAM_TIMER_MODE 10 +//KYS Check the modifications done to this file +#define HVM_PARAM_TIMER_MODE 11#define HVMPTM_delay_for_missed_ticks 0#define HVMPTM_no_delay_for_missed_ticks 1#define HVMPTM_no_missed_ticks_pending 2#define HVMPTM_one_missed_tick_pending 3 -#define HVM_NR_PARAMS 11 +#define HVM_NR_PARAMS 12#endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
xen/arch/x86/hvm/Makefile +1 -0
@@ -1,5 +1,6 @@ subdir-y += svm subdir-y += vmx +subdir-y += hvm_ext obj-y += hvm.o obj-y += i8254.o
xen/arch/x86/hvm/hvm.c +18 -0
@@ -42,6 +42,7 @@#include <asm/mc146818rtc.h> #include <asm/spinlock.h> #include <asm/hvm/hvm.h> +#include <asm/hvm/hvm_extensions.h> #include <asm/hvm/vpt.h> #include <asm/hvm/support.h> #include <asm/hvm/cacheattr.h>
@@ -118,6 +119,7 @@rtc_migrate_timers (v);hpet_migrate_timers (v);pt_migrate (v); +ext_intercept_do_migrate_timers (v); }void hvm_do_resume (struct vcpu *v)
@@ -279,6 +281,7 @@void hvm_domain_destroy (struct domain *d) { +ext_intercept_domain_destroy (d); hvm_funcs.domain_destroy (d);vioapic_deinit (d);hvm_destroy_cacheattr_region_list (d);
@@ -447,8 +450,14 @@ {int rc; +if ((rc =ext_intercept_vcpu_initialize (v)) !=0 ) + goto fail1; +if ( (rc =vlapic_init (v)) !=0 ) + { +ext_intercept_vcpu_destroy (v); goto fail1; + }if ( (rc = hvm_funcs.vcpu_initialise (v)) !=0 ) goto fail2;
@@ -496,12 +505,14 @@ hvm_funcs.vcpu_destroy (v); fail2:vlapic_destroy (v); +ext_intercept_vcpu_destroy (v); fail1:return rc; }void hvm_vcpu_destroy (struct vcpu *v) { +ext_intercept_vcpu_destroy (v);vlapic_destroy (v); hvm_funcs.vcpu_destroy (v);
@@ -1573,6 +1584,10 @@case 0 :break ; } +if (ext_intercept_do_hypercall (regs)) + { +return HVM_HCALL_completed; + }if ( (eax >= NR_hypercalls) || !hvm_hypercall32_table[eax] ) {
@@ -1964,6 +1979,9 @@if ( a.value > HVMPTM_one_missed_tick_pending ) goto param_fail;break ; +case HVM_PARAM_EXTEND_HYPERVISOR : +if (hvm_ext_bind (d, (int )a.value)) + goto param_fail; } d->arch.hvm_domain.params[a.index] = a.value; rc =0 ;
xen/arch/x86/x86_64/asm-offsets.c +3 -0
@@ -146,4 +146,7 @@BLANK (); OFFSET(CPUINFO_ext_features, struct cpuinfo_x86, x86_capability[1]); +BLANK (); + + OFFSET(DOM_ext_vector, struct domain, arch.hvm_domain.ext_vector); }
xen/arch/x86/hvm/vmx/x86_64/exits.S +8 -0
@@ -112,6 +112,14 @@ ALIGN ENTRY(vmx_asm_do_vmentry) GET_CURRENT(%rbx) + mov VCPU_domain(%rbx),%rax + mov DOM_ext_vector(%rax),%rdx + test %rdx,%rdx + je vmx_no_ext_vector + sti + callq *(%rdx) +vmx_no_ext_vector: + cli # tests must not race interrupts movl VCPU_processor(%rbx),%eax
xen/arch/x86/hvm/svm/x86_64/exits.S +8 -0
@@ -37,6 +37,14 @@ ENTRY(svm_asm_do_resume) GET_CURRENT(%rbx) + mov VCPU_domain(%rbx),%rax + mov DOM_ext_vector(%rax),%rdx + test %rdx,%rdx + je svm_no_ext_vector + sti + callq *(%rdx) +svm_no_ext_vector: + CLGI movl VCPU_processor(%rbx),%eax
xen/arch/x86/hvm/save.c +6 -2
@@ -23,6 +23,8 @@#include <asm/hvm/support.h> #include <public/hvm/save.h> +#include <public/hvm/params.h> +#include <asm/hvm/hvm_extensions.h> void arch_hvm_save (struct domain *d,struct hvm_save_header *hdr) {
@@ -31,8 +33,7 @@/* Save some CPUID bits */ cpuid (1 , &eax, &ebx, &ecx, &edx); hdr->cpuid = eax; - - hdr->pad0 =0 ; + hdr->ext_id = d->arch.hvm_domain.params[HVM_PARAM_EXTEND_HYPERVISOR ]; }int arch_hvm_load (struct domain *d,struct hvm_save_header *hdr)
@@ -61,6 +62,9 @@/* VGA state is not saved/restored, so we nobble the cache. */ d->arch.hvm_domain.stdvga.cache =0 ; + d->arch.hvm_domain.params[HVM_PARAM_EXTEND_HYPERVISOR ] = hdr->ext_id; +if (hvm_ext_bind (d, hdr->ext_id)) +return -1 ;return 0 ; }
xen/include/public/arch-x86/hvm/save.h +23 -2
@@ -38,7 +38,7 @@uint32_t version;/* File format version */ uint64_t changeset;/* Version of Xen that saved this file */ uint32_t cpuid;/* CPUID[0x01][%eax] on the saving machine */ -uint32_t pad0; +uint32_t ext_id;/* extension ID */ }; DECLARE_HVM_SAVE_TYPE(HEADER, 1, struct hvm_save_header);
@@ -422,9 +422,30 @@ DECLARE_HVM_SAVE_TYPE(MTRR, 14, struct hvm_hw_mtrr); +struct hvm_ns_veridian_dom { +uint64_t guestid_msr; +uint64_t hypercall_msr; +uint32_t long_mode; +uint32_t pad0; +}; +DECLARE_HVM_SAVE_TYPE(NS_VERIDIAN_DOM, 15, struct hvm_ns_veridian_dom); + +struct hvm_ns_veridian_cpu { +uint64_t control_msr; +uint64_t version_msr; +uint64_t sief_msr; +uint64_t simp_msr; +uint64_t eom_msr; +uint64_t int_msr[16 ]; +struct { +uint64_t config; +uint64_t count; + } timers[4 ]; +}; +DECLARE_HVM_SAVE_TYPE(NS_VERIDIAN_CPU, 16, struct hvm_ns_veridian_cpu);/* * Largest type-code in use */ -#define HVM_SAVE_CODE_MAX 14 +#define HVM_SAVE_CODE_MAX 16#endif /* __XEN_PUBLIC_HVM_SAVE_X86_H__ */
%patch
xen/arch/x86/hvm/hvm_ext/Makefile new file +3 -0
@@ -0,0 +1,3 @@ +subdir-y += novell + +obj-y += hvm_ext.o
xen/include/asm-x86/hvm/hvm_extensions.h new file +239 -0
239 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/hvm_ext.c new file +350 -0
350 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/novell/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +obj-y += nsintercept.o +obj-y += nshypercall.o
xen/arch/x86/hvm/hvm_ext/novell/ns_errno.h new file +62 -0
62 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/novell/ns_shim.h new file +480 -0
480 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/novell/nshypercall.c new file +1220 -0
1,220 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/novell/nshypercall.h new file +125 -0
125 lines not on this page · raw message
xen/arch/x86/hvm/hvm_ext/novell/nsintercept.c new file +2077 -0
113 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Part of the diff is not shown here. View the raw message.
Quoted 7 lines · Ky Srinivasan
B) Implement an adapter that implements the required Hyper-V functionality.
We have implemented only a subset of Hyper-V functionality that is required for enlightened windows 2008 guest today. However, we have the framework in place to implement any additional functionality that the windows guests may leverage going forward. The framework is extensible and one can easily implement OS specific enlightenments.
The appearance is that you hook on every significant emulation point in the HVM code and implement a shadow hypervisor. Is that an accurate description? What performance wins do you get from Hyper-V emulation, ignoring the obvious wins you will get from installing PV I/O drivers? I think this is a particularly interesting question since it was our impression that the PV memory-management interfaces, which you appear to jump through hoops to support, were not of much benefit with Xen's more sophisticated pagetable algorithm (or with Intel EPT or AMD NPT).
-- Keir
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
On Wed, Feb 20, 2008 at 4:44 AM, in message
<C3E1A703.1CB24%[email protected]>, Keir Fraser
Quoted 4 lines · <[email protected]> wrote:
Quoted 2 lines · Ky Srinivasan
B) Implement an adapter that implements the required Hyper- V functionality.
Quoted 8 lines
Quoted 5 lines
We have implemented only a subset of Hyper- V functionality that is required for enlightened windows 2008 guest today. However, we have the framework in place to implement any additional functionality that the windows guests may leverage going forward. The framework is extensible and one can easily implement OS specific enlightenments.
The appearance is that you hook on every significant emulation point in the HVM code and implement a shadow hypervisor. Is that an accurate description?
My initial goal was to get a framework in place where one could implement any OS specific enhancements without cluttering up the rest of the Xen code. To that end, this code does hook into various emulation points of interest in the hvm code. A secondary goal was to minimize the changes to the base Xen code. Furthermore, I wanted to reuse as much of the base Xen code as possible. So, I would not characterize this as a shadow hypervisor, but rather as an adapter that bridges the semantic gap between the APIs of interest in Hyper-V and Xen.
Quoted 6 lines
What performance wins do you get from Hyper- V emulation, ignoring the obvious wins you will get from installing PV I/O drivers? I think this is a particularly interesting question since it was our impression that the PV memory- management interfaces, which you appear to jump through hoops to support, were not of much benefit with Xen's more sophisticated pagetable algorithm (or with Intel EPT or AMD NPT).
We have done very minimal performance evaluation to date. On NetBench I have seen a 10% improvement. You are right, TLB flush enlightenments did not make much of a difference. These results are on a uniprocessor configuration. Of the enlightenments, MSR based APIC access made the most difference. Going forward, we will be doing a more comprehensive performance evaluation with the goal of having parity with the MS platform.
Regards,
K. Y
Quoted 9 lines
-- Keir
_______________________________________________
Xen- devel mailing list
Xen- [email protected]
http://lists.xensource.com/xen- devel
Quoted 8 lines · Ky Srinivasan
During the spring Xen Summit in New York, I presented our work on hosting Enlightened windows 2008 server on Xen based platforms. Now that Microsoft has published their Hypervisor specification, we can release our code to the community. The goal of this effort has been to host Enlightened windows 2008 server as a guest on a Xen based platform. We have addressed this requirement by :
A) Leveraging the I/O framework in the Xen based platform. Appropriate PV drivers will be loaded up in the guest to improve the I/O performance.
B) Implement an adapter that implements the required Hyper-V functionality.
We have implemented only a subset of Hyper-V functionality that is required for enlightened windows 2008 guest today. However, we have the framework in place to implement any additional functionality that the windows guests may leverage going forward. The framework is extensible and one can easily implement OS specific enlightenments.
Quoted 7 lines
I am enclosing three patches that implement our adapter for your consideration:
1) ns_tools.patch: Modifications to xen tools to support the the adapter
2) ns_xen_base.patch: Modifications to the base Xen code to support the adapter
3) ns_xen_extension.patch: New code implementing the adapterThese patches have been tested on a xen 3.2 based system (SLES10 SP2).
Right now there is not an implementation to support an enlightened windows guest, just a shim.
First question: Do you plan on implementing the full enlightened guest support as GPL source (as you have done with the elementary shim), or are you planning on loading a blob and initializing the hypercall vector, a la the vmware VMI? Has Microsoft agreed to a source-based GPL implementation of the hypercall layer?
Second question: what do you believe the performance will be of an enlightened guest running in the novell shim vs running on hyper-v? Have you implemented enough to do any profiling? Have you done any profiling of a standard hvm guest with and without the intercepts in place? It looks like no measurable impact, just curious.
It doesn't seem as though you can batch page table updates, or did I miss something?
Mike
Signature · 5 lines
Mike D. Day IBM LTC Cell: 919 412-3900 Sametime: [email protected] AIM: ncmikeday Yahoo: ultra.runner PGP key: http://www.ncultra.org/ncmike/pubkey.asc