hvm: Use main memory for video memory. [Was: xend, 8MB video memory and ballooning]
PATCH 1/1 hvm: Use main memory for video memory. [Was: xend, 8MB video memory and ballooning]
hvm: Use main memory for video memory.
When creating an HVM domain, if e.g. another domain is created before qemu allocates video memory, the extra 8MB memory ballooning is not available any more, because it got consumed by the other domain.
This fixes it by taking video memory from the main memory:
- make hvmloader use e820_malloc to reserve some of the main memory and notify ioemu of its address through the Xen platform PCI card.
- add XENMAPSPACE_mfn to the xen_add_to_physmap memory op, to allow ioemu to move the MFNs between the original position and the PCI mapping, when LFB acceleration is disabled/enabled
- add a remove_from_physmap memory op, to allow ioemu to unmap it completely for the case of old guests with acceleration disabled.
- add xc_domain_memory_translate_gpfn_list to libxc to allow ioemu to get the MFNs of the video memory.
- have xend save the PCI memory space instead of ioemu: if a memory page is there, the guest can access it like usual memory, so xend can safely be responsible to save it. The extra benefit is that live migration will apply the logdirty optimization there too.
- handle old saved images, populating the video memory from ioemu if really needed.
tools/firmware/hvmloader/hvmloader.c +15 -3
@@ -430,11 +430,12 @@ static void cmos_write_memory_size(void)cmos_outb (0x35 , (uint8_t )( alt_mem >>8 )); } -static void init_xen_platform_io_base (void ) +static uint16_t init_xen_platform_io_base (void ) {struct bios_info *bios_info = (struct bios_info *)ACPI_PHYSICAL_ADDRESS ;uint32_t devfn, bar_data;uint16_t vendor_id, device_id; +uint16_t xen_pfiob =0 ;for ( devfn =0 ; devfn <128 ; devfn++ ) {
@@ -443,14 +444,17 @@ static void init_xen_platform_io_base(voif ( (vendor_id !=0x5853 ) || (device_id !=0x0001 ) )continue ; bar_data =pci_readl (devfn,PCI_BASE_ADDRESS_0 ); - bios_info->xen_pfiob = bar_data &PCI_BASE_ADDRESS_IO_MASK ; + bios_info->xen_pfiob = xen_pfiob = bar_data &PCI_BASE_ADDRESS_IO_MASK ; } +return xen_pfiob; }int main (void ) {int vgabios_sz =0 , etherboot_sz =0 , rombios_sz, smbios_sz;int extboot_sz =0 ; +uint32_t vga_ram = ~0 ; +uint16_t xen_pfiob;printf ("HVM Loader\n" );
@@ -495,6 +499,11 @@ int main(void)break ; } +if ( virtual_vga != VGA_none ) { + vga_ram =e820_malloc (8 <<20 ); +printf ("VGA RAM at %x\n" , vga_ram); + } + etherboot_sz =scan_etherboot_nic ((void *)ETHERBOOT_PHYSICAL_ADDRESS );if (must_load_extboot () )
@@ -535,7 +544,10 @@ int main(void)ROMBIOS_PHYSICAL_ADDRESS , ROMBIOS_PHYSICAL_ADDRESS + rombios_sz - 1); -init_xen_platform_io_base (); + xen_pfiob =init_xen_platform_io_base (); + +if (xen_pfiob && vga_ram != ~0 ) +outl (xen_pfiob +4 , vga_ram);printf ("Invoking ROMBIOS ...\n" );return 0 ;
tools/firmware/hvmloader/util.c +2 -2
@@ -331,8 +331,8 @@ uint32_t e820_malloc(uint32_t size)int i;struct e820entry *ent = (struct e820entry *)HVM_E820 ; -/* Align allocation request to a reasonable boundary (1kB). */ - size = (size +1023 ) & ~1023 ; +/* Align allocation request to a reasonable boundary (4kB). */ + size = (size +4095 ) & ~4095 ;for ( i = *HVM_E820_NR -1 ; i >=0 ; i-- ) {
tools/ioemu/hw/cirrus_vga.c +66 -124
258 lines not on this page · raw message
tools/ioemu/hw/vga.c +112 -38
198 lines not on this page · raw message
tools/ioemu/hw/vga_int.h +2 -2
@@ -80,9 +80,9 @@#define VGA_MAX_HEIGHT 2048#define VGA_STATE_COMMON \ - uint8_t *vram_alloc; \ uint8_t *vram_ptr; \ - uint8_t *vram_shadow; \ + xen_pfn_t *vram_mfns; \ + uint64_t stolen_vram_addr;/* Address of stolen RAM */ \unsigned long vram_offset; \unsigned int vram_size; \unsigned long bios_offset; \
tools/ioemu/hw/xen_platform.c +38 -0
@@ -34,6 +34,7 @@ typedef struct PCIXenPlatformState {PCIDevice pci_dev;uint8_t platform_flags; +uint64_t vga_stolen_ram; } PCIXenPlatformState;static uint32_t xen_platform_ioport_readb (void *opaque,uint32_t addr)
@@ -69,11 +70,46 @@ static void xen_platform_ioport_writeb(v } +static uint32_t xen_platform_ioport_readl (void *opaque,uint32_t addr) +{ +PCIXenPlatformState *d = opaque; + + addr &=0xff ; + +switch (addr) { +case 4 :/* VGA stolen memory address */ +return d->vga_stolen_ram; +default : +return ~0u ; + } +} + +static void xen_platform_ioport_writel (void *opaque,uint32_t addr,uint32_t val) +{ +PCIXenPlatformState *d = opaque; + + addr &=0xff ; + val &=0xffffffff ; + +switch (addr) { +case 4 :/* VGA stolen memory address */ + d->vga_stolen_ram = val; +xen_vga_stolen_vram_addr (val); +break ; +default : +break ; + } +} + + +static void platform_ioport_map (PCIDevice *pci_dev,int region_num,uint32_t addr,uint32_t size,int type) {PCIXenPlatformState *d = (PCIXenPlatformState *)pci_dev;register_ioport_write (addr, size,1 , xen_platform_ioport_writeb, d); +register_ioport_write (addr, size,4 , xen_platform_ioport_writel, d);register_ioport_read (addr, size,1 , xen_platform_ioport_readb, d); +register_ioport_read (addr, size,4 , xen_platform_ioport_readl, d); }static uint32_t platform_mmio_read (void *opaque,target_phys_addr_t addr)
@@ -155,6 +191,7 @@ void xen_pci_save(QEMUFile *f, void *opapci_device_save (&d->pci_dev, f);qemu_put_8s (f, &d->platform_flags); +qemu_put_be64s (f, &d->vga_stolen_ram); }int xen_pci_load (QEMUFile *f,void *opaque,int version_id)
@@ -173,6 +210,7 @@ int xen_pci_load(QEMUFile *f, void *opaquint8_t flags;qemu_get_8s (f, &flags);xen_platform_ioport_writeb (d,0 , flags); +qemu_get_be64s (f, &d->vga_stolen_ram); }return 0 ;
tools/ioemu/vl.c +0 -32
@@ -7022,38 +7022,6 @@ static BOOL WINAPI qemu_ctrl_handler(DWO#define MAX_NET_CLIENTS 32#include <xg_private.h> - -/* FIXME Flush the shadow page */ -int unset_mm_mapping (int xc_handle,uint32_t domid, -unsigned long nr_pages,unsigned int address_bits, -xen_pfn_t *extent_start) -{ -int err =0 ; - - err =xc_domain_memory_decrease_reservation (xc_handle, domid, - nr_pages,0 , extent_start); -if (err) -fprintf (stderr,"Failed to decrease physmap\n" ); - -return err; -} - -int set_mm_mapping (int xc_handle,uint32_t domid, -unsigned long nr_pages,unsigned int address_bits, -xen_pfn_t *extent_start) -{ -int err =0 ; - - err =xc_domain_memory_populate_physmap ( - xc_handle, domid, nr_pages,0 , -XENMEMF_address_bits (address_bits), extent_start); -if (err) { -fprintf (stderr,"Failed to populate physmap\n" ); -return -1 ; - } - -return 0 ; -}int main (int argc,char **argv)
tools/ioemu/vl.h +3 -0
@@ -1560,6 +1560,9 @@ void timeoffset_get(void);/* xen_platform.c */ #ifndef QEMU_TOOL void pci_xen_platform_init (PCIBus *bus); +void xen_vga_stolen_vram_addr (uint64_t vram_addr); +void xen_vga_populate_vram (uint64_t vram_addr); +void xen_vga_vram_map (uint64_t vram_addr,int copy);#endif /* pci_emulation.c */
tools/libxc/xc_domain.c +27 -0
@@ -531,6 +531,33 @@ int xc_domain_memory_populate_physmap(inDPRINTF ("Failed allocation for dom %d: %ld extents of order %d\n" , domid, nr_extents, extent_order); errno =EBUSY ; + err =-1 ; + } + +return err; +} + +int xc_domain_memory_translate_gpfn_list (int xc_handle, +uint32_t domid, +unsigned long nr_gpfns, +xen_pfn_t *gpfn_list, +xen_pfn_t *mfn_list) +{ +int err; +struct xen_translate_gpfn_list translate_gpfn_list = { + .domid = domid, + .nr_gpfns = nr_gpfns, + }; +set_xen_guest_handle (translate_gpfn_list.gpfn_list, gpfn_list); +set_xen_guest_handle (translate_gpfn_list.mfn_list, mfn_list); + + err =xc_memory_op (xc_handle, XENMEM_translate_gpfn_list, &translate_gpfn_list); + +if ( err !=0 ) + { +DPRINTF ("Failed translation for dom %d (%ld PFNs)\n" , + domid, nr_gpfns); + errno = -err; err =-1 ; }
tools/libxc/xc_domain_save.c +1 -3
@@ -1112,9 +1112,7 @@ int xc_domain_save(int xc_handle, int iocontinue ;/* Skip PFNs that aren't really there */ -if ( hvm && ((n >=0xa0 && n <0xc0 )/* VGA hole */ - || (n >= (HVM_BELOW_4G_MMIO_START >>PAGE_SHIFT ) - && n < (1ULL <<32 ) >>PAGE_SHIFT ))/* MMIO */ ) +if ( hvm && n == shared_info_frame )continue ;/*
tools/libxc/xenctrl.h +6 -0
@@ -628,6 +628,12 @@ int xc_domain_memory_populate_physmap(inunsigned int mem_flags, xen_pfn_t *extent_start); +int xc_domain_memory_translate_gpfn_list (int xc_handle, +uint32_t domid, +unsigned long nr_gpfns, +xen_pfn_t *gpfn_list, +xen_pfn_t *mfn_list); + int xc_domain_ioport_permission(int xc_handle, uint32_t domid, uint32_t first_port,
xen/arch/ia64/xen/mm.c +69 -1
103 lines not on this page · raw message
xen/arch/x86/mm.c +58 -0
82 lines not on this page · raw message
xen/arch/x86/x86_64/compat/mm.c +14 -0
@@ -64,6 +64,20 @@ int compat_arch_memory_op(int op, XEN_GUreturn -EFAULT ;XLAT_add_to_physmap (nat, &cmp); + rc =arch_memory_op (op,guest_handle_from_ptr (nat, void)); + +break ; + } + +case XENMEM_remove_from_physmap: + { +struct compat_remove_from_physmap cmp; +struct xen_remove_from_physmap *nat = (void *)COMPAT_ARG_XLAT_VIRT_BASE ; + +if (copy_from_guest (&cmp, arg,1 ) ) +return -EFAULT ; + +XLAT_remove_from_physmap (nat, &cmp); rc =arch_memory_op (op,guest_handle_from_ptr (nat, void));break ;
xen/include/public/memory.h +17 -0
@@ -204,6 +204,7 @@ struct xen_add_to_physmap {/* Source mapping space. */ #define XENMAPSPACE_shared_info 0/* shared info page */ #define XENMAPSPACE_grant_table 1/* grant table page */ +#define XENMAPSPACE_mfn 2/* usual MFN */ unsigned int space;/* Index into source mapping space. */
@@ -214,6 +215,22 @@ struct xen_add_to_physmap { };typedef struct xen_add_to_physmap xen_add_to_physmap_t ;DEFINE_XEN_GUEST_HANDLE (xen_add_to_physmap_t); + +/* +* Unmaps the page appearing at a particular GPFN from the specified guest's +* pseudophysical address space. +* arg == addr of xen_remove_from_physmap_t. +*/ +#define XENMEM_remove_from_physmap 15 +struct xen_remove_from_physmap { +/* Which domain to change the mapping for. */ +domid_t domid; + +/* GPFN of the current mapping of the page. */ +xen_pfn_t gpfn; +}; +typedef struct xen_remove_from_physmap xen_remove_from_physmap_t ; +DEFINE_XEN_GUEST_HANDLE (xen_remove_from_physmap_t);/* * Translates a list of domain-specific GPFNs into MFNs. Returns a -ve error
xen/include/xlat.lst +1 -0
@@ -33,6 +33,7 @@ ! kexec_image kexec.h ! kexec_range kexec.h ! add_to_physmap memory.h +! remove_from_physmap memory.h ! foreign_memory_map memory.h ! memory_exchange memory.h ! memory_map memory.h
xen/include/xsm/xsm.h +6 -0
@@ -136,6 +136,7 @@ struct xsm_operations {int (*mmu_machphys_update) (struct domain *d,unsigned long mfn);int (*update_va_mapping) (struct domain *d,l1_pgentry_t pte);int (*add_to_physmap) (struct domain *d1,struct domain *d2); +int (*remove_from_physmap) (struct domain *d1,struct domain *d2);#endif };
@@ -532,6 +533,11 @@ static inline int xsm_add_to_physmap(str {return xsm_call (add_to_physmap (d1, d2)); } + +static inline int xsm_remove_from_physmap (struct domain *d1,struct domain *d2) +{ +return xsm_call (remove_from_physmap (d1, d2)); +}#endif /* CONFIG_X86 */ #endif /* __XSM_H */
xen/xsm/dummy.c +6 -0
@@ -385,6 +385,11 @@ static int dummy_add_to_physmap (struct {return 0 ; } + +static int dummy_remove_from_physmap (struct domain *d1,struct domain *d2) +{ +return 0 ; +}#endif struct xsm_operations dummy_xsm_ops;
@@ -484,5 +489,6 @@ void xsm_fixup_ops (struct xsm_operationset_to_dummy_if_null (ops, mmu_machphys_update);set_to_dummy_if_null (ops, update_va_mapping);set_to_dummy_if_null (ops, add_to_physmap); +set_to_dummy_if_null (ops, remove_from_physmap);#endif }
xen/xsm/flask/hooks.c +6 -0
@@ -1028,6 +1028,11 @@ static int flask_add_to_physmap(struct d {return domain_has_perm (d1, d2,SECCLASS_MMU ,MMU__PHYSMAP ); } + +static int flask_remove_from_physmap (struct domain *d1,struct domain *d2) +{ +return domain_has_perm (d1, d2,SECCLASS_MMU ,MMU__PHYSMAP ); +}#endif long do_flask_op (XEN_GUEST_HANDLE (xsm_op_t) u_flask_op);
@@ -1115,6 +1120,7 @@ static struct xsm_operations flask_ops = .mmu_machphys_update = flask_mmu_machphys_update, .update_va_mapping = flask_update_va_mapping, .add_to_physmap = flask_add_to_physmap, + .remove_from_physmap = flask_remove_from_physmap,#endif };
Quoted 5 lines · Samuel Thibault
hvm: Use main memory for video memory.
When creating an HVM domain, if e.g. another domain is created before qemu allocates video memory, the extra 8MB memory ballooning is not available any more, because it got consumed by the other domain.
Really too big for Xen 3.3 at this point, unfortunately. I'll queue it up.
-- Keir
Quoted 9 lines · Keir Fraser, le Thu 31 Jul 2008 14:22:30 +0100, a écrit :
Quoted 5 lines · Samuel Thibault
hvm: Use main memory for video memory.
When creating an HVM domain, if e.g. another domain is created before qemu allocates video memory, the extra 8MB memory ballooning is not available any more, because it got consumed by the other domain.
Really too big for Xen 3.3 at this point, unfortunately.
Ok, then I guess I should document that for stub domains to work one shouldn't use the ballooner. Or a quick&dirty fix would be to add the 8MB extra ballooning for PV guests as well.
Samuel
Quoted 5 lines · Samuel Thibault
Quoted 1 line
Really too big for Xen 3.3 at this point, unfortunately.
Ok, then I guess I should document that for stub domains to work one shouldn't use the ballooner. Or a quick&dirty fix would be to add the 8MB extra ballooning for PV guests as well.
I think documenting the mutual incompatibility is fine for 3.3. But it's up to you.
-- Keir
Quoted 9 lines · Keir Fraser, le Thu 31 Jul 2008 14:41:01 +0100, a écrit :
Quoted 5 lines · Samuel Thibault
Quoted 1 line
Really too big for Xen 3.3 at this point, unfortunately.
Ok, then I guess I should document that for stub domains to work one shouldn't use the ballooner. Or a quick&dirty fix would be to add the 8MB extra ballooning for PV guests as well.
I think documenting the mutual incompatibility is fine for 3.3. But it's up to you.
The sad thing is that although it would fix HVM+ioemu-dom boot, this quick&dirty fix of course fixes the underlying race only for one HVM domain lagging behind. If there are several, 8MB won't be enough, etc. I guess it's simpler to just document.
Samuel
stubdom: update documentation
stubdom/ is now compiled and installed by default HVM+IOEMU-stubdom can not boot if dom0 has to be ballooned.
stubdom/README +5 -10
@@ -1,13 +1,3 @@ To compile -To compile -========== - -Just run make -j 4, that will download / patch / compile -Then make install to install the result. - -Also, run make and make install in $XEN_ROOT/tools/fs-back - - - IOEMU stubdom =============
@@ -15,6 +5,11 @@ Also, run make and make install in $XEN_ General Configuration ===================== + +Due to a race between the creation of the IOEMU stubdomain itself and allocation +of video memory for the HVM domain, you need to avoid the need for ballooning, +by using the hypervisor dom0_mem= option for instance. + In your HVM config "hvmconfig",
tools/libxc/xc_domain_save.c Thu Jul 31 13:20:02 2008 +0100 int xc_domain_save(int xc_handle, int io · line 1117
Quoted 13 lines · Samuel Thibault
diff -r
83b3a8a2a4f3tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c Thu Jul 31 13:20:02 2008 +0100 renamed from tools/libxc/xc_domain_save.c Thu Jul 31 12:58:07 2008 +0100 +1
-3@@ -1112,9 +1112,7 @@ int xc_domain_save(int xc_handle, int io continue; /* Skip PFNs that aren't really there */ - if ( hvm && ((n >= 0xa0 && n < 0xc0) /* VGA hole */ - || (n >= (HVM_BELOW_4G_MMIO_START >> PAGE_SHIFT) - && n < (1ULL<<32) >> PAGE_SHIFT)) /* MMIO */ ) + if ( hvm && n == shared_info_frame ) continue;
What happens if you don't special-case shared_info_frame? I'd've hoped it would just work... does the extra page of allocation during restore fail?
I'm thinking that the grant tables will be mapped in the I/O hole too, and that is from 'Xen heap' just like shared_info is. So special-casing one and not the other sure is odd.
-- Keir
Quoted 8 lines · Keir Fraser, le Thu 31 Jul 2008 14:39:59 +0100, a écrit :
Quoted 6 lines · Samuel Thibault
diff -r
83b3a8a2a4f3tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c Thu Jul 31 13:20:02 2008 +0100 renamed from tools/libxc/xc_domain_save.c Thu Jul 31 12:58:07 2008 +0100 +0
-0@@ -1112,9 +1112,7 @@ int xc_domain_save(int xc_handle, int io continue;
Quoted 9 lines
tools/libxc/xc_domain_save.c Thu Jul 31 13:20:02 2008 +0100 int xc_domain_save(int xc_handle, int io · line 1117
Quoted 7 lines
tools/libxc/xc_domain_save.c Thu Jul 31 13:20:02 2008 +0100 renamed from tools/libxc/xc_domain_save.c Thu Jul 31 12:58:07 2008 +0100 +1
-3/* Skip PFNs that aren't really there */ - if ( hvm && ((n >= 0xa0 && n < 0xc0) /* VGA hole */ - || (n >= (HVM_BELOW_4G_MMIO_START >> PAGE_SHIFT) - && n < (1ULL<<32) >> PAGE_SHIFT)) /* MMIO */ ) + if ( hvm && n == shared_info_frame ) continue;What happens if you don't special-case
shared_info_frame?
I actually hadn't even tried.
Quoted 2 lines
I'd've hoped it would just work... does the extra page of allocation during restore fail?
It does work, but I guess it replaces the mapping of the "magic" page with the mapping of the allocated page, so that shared_info wouldn't work any more.
Quoted 2 lines
I'm thinking that the grant tables will be mapped in the I/O hole too, and that is from 'Xen heap' just like
shared_infois.
Urgl, hadn't thought about those, I guess the PV drivers would re-etablish them on resume anyway?
Quoted 1 line
So special-casing one and not the other sure is odd.
Actually, shouldn't it be up to the hypervisor to not return all these from the SHADOW_OP_PEEK operation?
Samuel
Quoted 6 lines · Samuel Thibault
Quoted 2 lines
I'd've hoped it would just work... does the extra page of allocation during restore fail?
It does work, but I guess it replaces the mapping of the "magic" page with the mapping of the allocated page, so that
shared_infowouldn't work any more.
PV drivers will add_to_physmap() these pages again during resume, which will map the correct pages (shared_info and grant_table, notably) on top of the RAM pages that save/restore will leave behind. things should work fine then and the displaced RAM pages *should* get freed immediately. Even if they don't they will on domain destruction and it's only a few pages...
Quoted 5 lines
Quoted 2 lines
I'm thinking that the grant tables will be mapped in the I/O hole too, and that is from 'Xen heap' just like
shared_infois.Urgl, hadn't thought about those, I guess the PV drivers would re-etablish them on resume anyway?
Yep, as above.
Quoted 4 lines
Quoted 1 line
So special-casing one and not the other sure is odd.
Actually, shouldn't it be up to the hypervisor to not return all these from the
SHADOW_OP_PEEKoperation?
Well, maybe. :-) But shadow code doesn't special-case these Xen-heap pages I think. And doing so could be a pain, and maybe cause other issues?
-- Keir
Quoted 10 lines · Keir Fraser, le Thu 31 Jul 2008 16:03:12 +0100, a écrit :
Quoted 6 lines · Samuel Thibault
Quoted 2 lines
I'd've hoped it would just work... does the extra page of allocation during restore fail?
It does work, but I guess it replaces the mapping of the "magic" page with the mapping of the allocated page, so that
shared_infowouldn't work any more.PV drivers will
add_to_physmap()these pages again during resume,
Ok, so it should be safe indeed.
Samuel