Xen PV-on-HVM guest support (v2)
PATCH 1/1 Xen PV-on-HVM guest support (v2)
Support for Xen PV-on-HVM guests can be implemented almost entirely in userspace, except for handling one annoying MSR that maps a Xen hypercall blob into guest address space.
A generic mechanism to delegate MSR writes to userspace seems overkill and risks encouraging similar MSR abuse in the future. Thus this patch adds special support for the Xen HVM MSR.
I implemented a new ioctl, KVM_XEN_HVM_CONFIG, that lets userspace tell KVM which MSR the guest will write to, as well as the starting address and size of the hypercall blobs (one each for 32-bit and 64-bit) that userspace has loaded from files. When the guest writes to the MSR, KVM copies one page of the blob from userspace to the guest.
I've tested this patch with a hacked-up version of Gerd's userspace code, booting a number of guests (CentOS 5.3 i386 and x86_64, and FreeBSD 8.0-RC1 amd64) and exercising PV network and block devices.
v2: fix ioctl struct padding; renumber CAP and ioctl constants; check kvm_write_guest() return value; change printks to KERN_DEBUG (I think they're worth keeping for debugging userspace)
---
include/asm-x86/kvm.h +1 -0
@@ -59,6 +59,7 @@#define __KVM_HAVE_MSIX#define __KVM_HAVE_MCE#define __KVM_HAVE_PIT_STATE2 +#define __KVM_HAVE_XEN_HVM/* Architectural interrupt line count. */ #define KVM_NR_INTERRUPTS 256
include/linux/kvm.h +13 -0
@@ -476,6 +476,9 @@ struct kvm_ioeventfd {#endif #define KVM_CAP_IOEVENTFD 36#define KVM_CAP_SET_IDENTITY_MAP_ADDR 37 +#ifdef __KVM_HAVE_XEN_HVM +#define KVM_CAP_XEN_HVM 38 +#endif #ifdef KVM_CAP_IRQ_ROUTING
@@ -528,6 +531,15 @@ struct kvm_x86_mce { };#endif +#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config { +__u32 msr; +__u8 pad[2 ]; +__u8 blob_size[2 ]; +__u64 blob_addr[2 ]; +}; +#endif +#define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)struct kvm_irqfd {
@@ -586,6 +598,7 @@ struct kvm_irqfd {#define KVM_CREATE_PIT2 _IOW(KVMIO, 0x77, struct kvm_pit_config)#define KVM_SET_BOOT_CPU_ID _IO(KVMIO, 0x78)#define KVM_IOEVENTFD _IOW(KVMIO, 0x79, struct kvm_ioeventfd) +#define KVM_XEN_HVM_CONFIG _IOW(KVMIO, 0x7a, struct kvm_xen_hvm_config)/* * ioctls for vcpu fds
include/linux/kvm_host.h +4 -0
@@ -236,6 +236,10 @@ struct kvm {unsigned long mmu_notifier_seq;long mmu_notifier_count;#endif + +#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config xen_hvm_config; +#endif };/* The guest did something we don't support. */
x86/x86.c +44 -0
62 lines not on this page · raw message
Quoted 25 lines · Ed Swierk
Support for Xen PV-on-HVM guests can be implemented almost entirely in userspace, except for handling one annoying MSR that maps a Xen hypercall blob into guest address space.
A generic mechanism to delegate MSR writes to userspace seems overkill and risks encouraging similar MSR abuse in the future. Thus this patch adds special support for the Xen HVM MSR.
I implemented a new ioctl,
KVM_XEN_HVM_CONFIG, that lets userspace tell KVM which MSR the guest will write to, as well as the starting address and size of the hypercall blobs (one each for 32-bit and 64-bit) that userspace has loaded from files. When the guest writes to the MSR, KVM copies one page of the blob from userspace to the guest.I've tested this patch with a hacked-up version of Gerd's userspace code, booting a number of guests (CentOS 5.3 i386 and
x86_64, and FreeBSD 8.0-RC1 amd64) and exercising PV network and block devices.v2: fix ioctl struct padding; renumber CAP and ioctl constants; check
kvm_write_guest()return value; change printks toKERN_DEBUG(I think they're worth keeping for debugging userspace)
include/linux/kvm.h struct kvm_x86_mce { · line 541
Quoted 8 lines
include/linux/kvm.h +8
-0+#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config { +__u32 msr; +__u8 pad[2 ]; +__u8 blob_size[2 ]; +__u64 blob_addr[2 ]; +}; +#endif
Please change the arrays to separate variables (e.g. blob_size_32, blob_size_64), so readers don't have to guess the meaning.
Also, reserve a bunch of space at the end in case we need more hackery.
Is the msr number really variable? Isn't it an ABI?
Quoted 6 lines
* ioctls for vcpu fds
Index:kvm-kmod/include/linux/kvm_host.h
===================================================================
include/linux/kvm_host.h struct kvm { · line 244
Quoted 9 lines
include/linux/kvm_host.h +4
-0@@ -236,6 +236,10 @@ struct kvm {unsigned long mmu_notifier_seq;long mmu_notifier_count;#endif + +#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config xen_hvm_config; +#endif };
struct kvm_arch is a better place for this.
x86/x86.c static int set_msr_mce(struct kvm_vcpu * · line 879
Quoted 11 lines
include/linux/kvm_host.h +0
-0/* The guest did something we don't support. */ Index:
kvm-kmod/x86/x86.c
===================================================================x86/x86.c +1
-0@@ -875,6 +875,35 @@ static int set_msr_mce(struct kvm_vcpu *return 0 ; } +#ifdef KVM_CAP_XEN_HVM
No need for the ifdef - it will always be defined for x86.
x86/x86.c static int set_msr_mce(struct kvm_vcpu *
Quoted 4 lines
x86/x86.c +3
-0+static int xen_hvm_config (struct kvm_vcpu *vcpu,u64 data) +{ +int blob = !!(vcpu->arch.shadow_efer&EFER_LME );
Can use is_long_mode() for this.
Quoted 4 lines
x86/x86.c +4
-0+u32 pnum = data& ~PAGE_MASK ; +u64 paddr = data&PAGE_MASK ; +u8 *page; +int r =1 ;
x86/x86.c static int set_msr_mce(struct kvm_vcpu *
Quoted 10 lines
x86/x86.c +9
-0+ +if (pnum>= vcpu->kvm->xen_hvm_config.blob_size[blob]) + goto out; + page =kzalloc (PAGE_SIZE ,GFP_KERNEL ); +if (!page) + goto out; +if (copy_from_user (page, (u8 *)vcpu->kvm->xen_hvm_config.blob_addr[blob] + + pnum *PAGE_SIZE ,PAGE_SIZE )) + goto out_free;
We want to return -EFAULT here (but make sure the entire code path allows this).
Quoted 16 lines
x86/x86.c +12
-0+if (kvm_write_guest (vcpu->kvm, paddr, page,PAGE_SIZE )) + goto out_free; +printk (KERN_DEBUG "kvm: copied xen hvm blob %d page %d to 0x%llx\n" , + blob, pnum, paddr); + r =0 ; +out_free: +kfree (page); +out: + return r; +} +#endif +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) { switch (msr) {
x86/x86.c int kvm_set_msr_common(struct kvm_vcpu * · line 1026
Quoted 8 lines
x86/x86.c +4
-0@@ -990,6 +1019,10 @@ int kvm_set_msr_common(struct kvm_vcpu * "0x%x data 0x%llx\n", msr, data);break ;default : +#ifdef KVM_CAP_XEN_HVM +if (msr&& (msr == vcpu->kvm->xen_hvm_config.msr)) +return xen_hvm_config (vcpu, data); +#endif
Again, can skip the ifdef.
Quoted 12 lines
x86/x86.c +0
-0if (!ignore_msrs) { pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",msr, data);
x86/x86.c +4
-0@@ -2453,6 +2486,17 @@ long kvm_arch_vm_ioctl(struct file *filp r =0 ;break ; } +#ifdef KVM_CAP_XEN_HVM +case KVM_XEN_HVM_CONFIG : { + r = -EFAULT ; + if (copy_from_user(&kvm->xen_hvm_config, argp,
x86/x86.c long kvm_arch_vm_ioctl(struct file *filp · line 2502
Quoted 10 lines
x86/x86.c +7
-0+ sizeof(struct kvm_xen_hvm_config))) + goto out; +printk (KERN_DEBUG "kvm: configured xen hvm\n" ); + r =0 ; + break; + } +#endif default : ; }
Do we need support for reading the msr?
IMO you can drop the debugging printk()s. I don't see how they add much value.
Please submit the patch against a current kernel tree, not kvm-kmod.
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
Hi,
Quoted 1 line
Is the msr number really variable? Isn't it an ABI?
Yes, it is variable. The guests gets the msr number via cpuid ...
Quoted 1 line
Do we need support for reading the msr?
I don't think so.
cheers,
Gerd
Quoted 10 lines · Gerd Hoffmann
Hi,
Quoted 1 line
Is the msr number really variable? Isn't it an ABI?
Yes, it is variable. The guests gets the msr number via cpuid ...
Quoted 1 line
Do we need support for reading the msr?
I don't think so.
Thanks. So Ed, I think you're good to go, but please update Documentation/kvm/api.txt for your next round.
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.
Quoted 14 lines · Ed Swierk
Support for Xen PV-on-HVM guests can be implemented almost entirely in userspace, except for handling one annoying MSR that maps a Xen hypercall blob into guest address space.
A generic mechanism to delegate MSR writes to userspace seems overkill and risks encouraging similar MSR abuse in the future. Thus this patch adds special support for the Xen HVM MSR.
I implemented a new ioctl,
KVM_XEN_HVM_CONFIG, that lets userspace tell KVM which MSR the guest will write to, as well as the starting address and size of the hypercall blobs (one each for 32-bit and 64-bit) that userspace has loaded from files. When the guest writes to the MSR, KVM copies one page of the blob from userspace to the guest.
Quoted 7 lines
I've tested this patch with a hacked-up version of Gerd's userspace code, booting a number of guests (CentOS 5.3 i386 and
x86_64, and FreeBSD 8.0-RC1 amd64) and exercising PV network and block devices.v2: fix ioctl struct padding; renumber CAP and ioctl constants; check
kvm_write_guest()return value; change printks toKERN_DEBUG(I think they're worth keeping for debugging userspace)
I disagree /wrt the print in the IOCTL path (missing configuration can also be reported on access), and the guest triggered path at least requires a pr_debug conversion. Looks fine to me otherwise.
Jan
Quoted 140 lines
Signed-off-by Ed Swierk <[email protected]>---
Index:kvm-kmod/include/asm-x86/kvm.h
===================================================================include/asm-x86/kvm.h +1
-0@@ -59,6 +59,7 @@#define __KVM_HAVE_MSIX#define __KVM_HAVE_MCE#define __KVM_HAVE_PIT_STATE2 +#define __KVM_HAVE_XEN_HVM/* Architectural interrupt line count. */ #define KVM_NR_INTERRUPTS 256Index:
kvm-kmod/include/linux/kvm.h
===================================================================include/linux/kvm.h +13
-0@@ -476,6 +476,9 @@ struct kvm_ioeventfd {#endif #define KVM_CAP_IOEVENTFD 36#define KVM_CAP_SET_IDENTITY_MAP_ADDR 37 +#ifdef __KVM_HAVE_XEN_HVM +#define KVM_CAP_XEN_HVM 38 +#endif #ifdef KVM_CAP_IRQ_ROUTING @@ -528,6 +531,15 @@ struct kvm_x86_mce { };#endif +#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config { +__u32 msr; +__u8 pad[2 ]; +__u8 blob_size[2 ]; +__u64 blob_addr[2 ]; +}; +#endif +#define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)struct kvm_irqfd {@@ -586,6 +598,7 @@ struct kvm_irqfd {#define KVM_CREATE_PIT2 _IOW(KVMIO, 0x77, struct kvm_pit_config)#define KVM_SET_BOOT_CPU_ID _IO(KVMIO, 0x78)#define KVM_IOEVENTFD _IOW(KVMIO, 0x79, struct kvm_ioeventfd) +#define KVM_XEN_HVM_CONFIG _IOW(KVMIO, 0x7a, struct kvm_xen_hvm_config)/* * ioctls for vcpu fds Index:
kvm-kmod/include/linux/kvm_host.h
===================================================================include/linux/kvm_host.h +4
-0@@ -236,6 +236,10 @@ struct kvm {unsigned long mmu_notifier_seq;long mmu_notifier_count;#endif + +#ifdef KVM_CAP_XEN_HVM +struct kvm_xen_hvm_config xen_hvm_config; +#endif };/* The guest did something we don't support. */ Index:
kvm-kmod/x86/x86.c
===================================================================x86/x86.c +44
-0@@ -875,6 +875,35 @@ static int set_msr_mce(struct kvm_vcpu *return 0 ; } +#ifdef KVM_CAP_XEN_HVM +static int xen_hvm_config (struct kvm_vcpu *vcpu,u64 data) +{ +int blob = !!(vcpu->arch.shadow_efer &EFER_LME ); +u32 pnum = data & ~PAGE_MASK ; +u64 paddr = data &PAGE_MASK ; +u8 *page; +int r =1 ; + +if (pnum >= vcpu->kvm->xen_hvm_config.blob_size[blob]) + goto out; + page =kzalloc (PAGE_SIZE ,GFP_KERNEL ); +if (!page) + goto out; +if (copy_from_user (page, (u8 *)vcpu->kvm->xen_hvm_config.blob_addr[blob] + + pnum *PAGE_SIZE ,PAGE_SIZE )) + goto out_free; +if (kvm_write_guest (vcpu->kvm, paddr, page,PAGE_SIZE )) + goto out_free; +printk (KERN_DEBUG "kvm: copied xen hvm blob %d page %d to 0x%llx\n" , + blob, pnum, paddr); + r =0 ; +out_free: +kfree (page); +out: +return r; +} +#endif +int kvm_set_msr_common (struct kvm_vcpu *vcpu,u32 msr,u64 data) {switch (msr) {@@ -990,6 +1019,10 @@ int kvm_set_msr_common(struct kvm_vcpu * "0x%x data 0x%llx\n", msr, data);break ;default : +#ifdef KVM_CAP_XEN_HVM +if (msr && (msr == vcpu->kvm->xen_hvm_config.msr)) +return xen_hvm_config (vcpu, data); +#endif if (!ignore_msrs) {pr_unimpl (vcpu,"unhandled wrmsr: 0x%x data %llx\n" , msr, data);@@ -2453,6 +2486,17 @@ long kvm_arch_vm_ioctl(struct file *filp r =0 ;break ; } +#ifdef KVM_CAP_XEN_HVM +case KVM_XEN_HVM_CONFIG : { + r = -EFAULT ; +if (copy_from_user (&kvm->xen_hvm_config, argp, +sizeof (struct kvm_xen_hvm_config ))) + goto out; +printk (KERN_DEBUG "kvm: configured xen hvm\n" ); + r =0 ; +break ; + } +#endif default : ; }