x86/pvh: Support relocating dom0 kernel
Subsystems (3)
v3 cover x86/pvh: Support relocating dom0 kernel
Xen tries to load a PVH dom0 kernel at the fixed guest physical address from the elf headers. For Linux, this defaults to 0x1000000 (16MB), but it can be configured.
Unfortunately there exist firmwares that have reserved regions at this address, so Xen fails to load the dom0 kernel since it's not RAM.
The other issue is that the Linux PVH entry point is not position-independent. It expects to run at the compiled CONFIG_PHYSICAL_ADDRESS.
This patch set expands the PVH dom0 builder to try to relocate the kernel if needed and possible. XEN_ELFNOTE_PHYS32_RELOC is added for kernels to indicate they are relocatable and their acceptable address range and alignment.
The first patch reverts "xen/x86: bzImage parse kernel_alignment" since the alignment will be specified by the ELF note.
The second patch expands ELF note printing beyond just printing integers and strings.
The third patch expands the pvh dom0 kernel placement code.
I'll post an additional patch showing the Linux changes to make PVH relocatable.
Jason Andryuk (3):
Revert "xen/x86: bzImage parse kernel_alignment"
libelf: Expand ELF note printing x86/PVH: Support relocatable dom0 kernels
xen/arch/x86/bzimage.c | 4 +- xen/arch/x86/hvm/dom0_build.c | 98 +++++++++++++++++++++++++++++- xen/arch/x86/include/asm/bzimage.h | 2 +- xen/arch/x86/pv/dom0_build.c | 2 +- xen/common/libelf/libelf-dominfo.c | 83 ++++++++++++++++++------- xen/include/public/elfnote.h | 18 +++++- xen/include/xen/libelf.h | 4 ++ 7 files changed, 181 insertions(+), 30 deletions(-)
Signature · 1 line
2.44.0
PATCH v3 1/3 Revert "xen/x86: bzImage parse kernel_alignment"
A new ELF note will specify the alignment for a relocatable PVH kernel. ELF notes are suitable for vmlinux and other ELF files, so this Linux-specific bzImage parsing in unnecessary.
This reverts commit c44cac229067faeec8f49247d1cf281723ac2d40.
--- xen/arch/x86/bzimage.c | 4 +--- xen/arch/x86/hvm/dom0_build.c | 4 +--- xen/arch/x86/include/asm/bzimage.h | 2 +- xen/arch/x86/pv/dom0_build.c | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-)
xen/arch/x86/bzimage.c +1 -3
@@ -105,7 +105,7 @@ unsigned long __init bzimage_headroom(void *image_start, }int __init bzimage_parse (void *image_base,void **image_start, -unsigned long *image_len,unsigned int *align) +unsigned long *image_len) {struct setup_header *hdr = (struct setup_header *)(*image_start);int err =bzimage_check (hdr, *image_len);
@@ -118,8 +118,6 @@ int __init bzimage_parse(void *image_base, void **image_start, { *image_start += (hdr->setup_sects +1 ) *512 + hdr->payload_offset; *image_len = hdr->payload_length; -if ( align ) - *align = hdr->kernel_alignment; }if (elf_is_elfbinary (*image_start, *image_len) )
xen/arch/x86/hvm/dom0_build.c +1 -3
@@ -548,14 +548,12 @@ static int __init pvh_load_kernel(struct domain *d, const module_t *image,struct elf_binary elf;struct elf_dom_parms parms;paddr_t last_addr; -unsigned int align =0 ;struct hvm_start_info start_info = {0 };struct hvm_modlist_entry mod = {0 };struct vcpu *v = d->vcpu[0 ];int rc; - rc =bzimage_parse (image_base, &image_start, &image_len, &align); -if ( rc !=0 ) +if ( (rc =bzimage_parse (image_base, &image_start, &image_len)) !=0 ) {printk ("Error trying to detect bz compressed kernel\n" );return rc;
xen/arch/x86/include/asm/bzimage.h +1 -1
@@ -6,6 +6,6 @@unsigned long bzimage_headroom (void *image_start,unsigned long image_length);int bzimage_parse (void *image_base,void **image_start, -unsigned long *image_len,unsigned int *align); +unsigned long *image_len);#endif /* __X86_BZIMAGE_H__ */
xen/arch/x86/pv/dom0_build.c +1 -1
@@ -416,7 +416,7 @@ int __init dom0_construct_pv(struct domain *d, d->max_pages = ~0U ; -if ( (rc =bzimage_parse (image_base, &image_start, &image_len,NULL )) !=0 ) +if ( (rc =bzimage_parse (image_base, &image_start, &image_len)) !=0 )return rc;if ( (rc =elf_init (&elf, image_start, image_len)) !=0 )
Signature · 1 line
2.44.0
PATCH v3 2/3 libelf: Expand ELF note printing
In reply to Jason AndryukThe XEN_ELFNOTE_L1_MFN_VALID is an array of pairs of values, but it is printed as:
(XEN) ELF: note: L1_MFN_VALID = 0
This is a limitation of only printing either string or numeric values. Switch from the boolean to an enum which allows more flexibility in printing the values. Introduce ELFNOTE_NAME to only print the name without a value like:
(XEN) ELF: note: L1_MFN_VALID
---
v3:
Remove fatal size check
Don't print values - just the name for presence
--- xen/common/libelf/libelf-dominfo.c | 50 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 21 deletions(-)
xen/common/libelf/libelf-dominfo.c +29 -21
@@ -101,26 +101,30 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,/* *INDENT-OFF* */ static const struct {const char *name; -bool str; +enum { +ELFNOTE_INT , +ELFNOTE_STRING , +ELFNOTE_NAME , + } type; } note_desc[] = { - [XEN_ELFNOTE_ENTRY ] = {"ENTRY" ,0 }, - [XEN_ELFNOTE_HYPERCALL_PAGE ] = {"HYPERCALL_PAGE" ,0 }, - [XEN_ELFNOTE_VIRT_BASE ] = {"VIRT_BASE" ,0 }, - [XEN_ELFNOTE_INIT_P2M ] = {"INIT_P2M" ,0 }, - [XEN_ELFNOTE_PADDR_OFFSET ] = {"PADDR_OFFSET" ,0 }, - [XEN_ELFNOTE_HV_START_LOW ] = {"HV_START_LOW" ,0 }, - [XEN_ELFNOTE_XEN_VERSION ] = {"XEN_VERSION" ,1 }, - [XEN_ELFNOTE_GUEST_OS ] = {"GUEST_OS" ,1 }, - [XEN_ELFNOTE_GUEST_VERSION ] = {"GUEST_VERSION" ,1 }, - [XEN_ELFNOTE_LOADER ] = {"LOADER" ,1 }, - [XEN_ELFNOTE_PAE_MODE ] = {"PAE_MODE" ,1 }, - [XEN_ELFNOTE_FEATURES ] = {"FEATURES" ,1 }, - [XEN_ELFNOTE_SUPPORTED_FEATURES ] = {"SUPPORTED_FEATURES" ,0 }, - [XEN_ELFNOTE_BSD_SYMTAB ] = {"BSD_SYMTAB" ,1 }, - [XEN_ELFNOTE_L1_MFN_VALID ] = {"L1_MFN_VALID" , false }, - [XEN_ELFNOTE_SUSPEND_CANCEL ] = {"SUSPEND_CANCEL" ,0 }, - [XEN_ELFNOTE_MOD_START_PFN ] = {"MOD_START_PFN" ,0 }, - [XEN_ELFNOTE_PHYS32_ENTRY ] = {"PHYS32_ENTRY" ,0 }, + [XEN_ELFNOTE_ENTRY ] = {"ENTRY" ,ELFNOTE_INT }, + [XEN_ELFNOTE_HYPERCALL_PAGE ] = {"HYPERCALL_PAGE" ,ELFNOTE_INT }, + [XEN_ELFNOTE_VIRT_BASE ] = {"VIRT_BASE" ,ELFNOTE_INT }, + [XEN_ELFNOTE_INIT_P2M ] = {"INIT_P2M" ,ELFNOTE_INT }, + [XEN_ELFNOTE_PADDR_OFFSET ] = {"PADDR_OFFSET" ,ELFNOTE_INT }, + [XEN_ELFNOTE_HV_START_LOW ] = {"HV_START_LOW" ,ELFNOTE_INT }, + [XEN_ELFNOTE_XEN_VERSION ] = {"XEN_VERSION" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_GUEST_OS ] = {"GUEST_OS" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_GUEST_VERSION ] = {"GUEST_VERSION" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_LOADER ] = {"LOADER" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_PAE_MODE ] = {"PAE_MODE" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_FEATURES ] = {"FEATURES" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_SUPPORTED_FEATURES ] = {"SUPPORTED_FEATURES" ,ELFNOTE_INT }, + [XEN_ELFNOTE_BSD_SYMTAB ] = {"BSD_SYMTAB" ,ELFNOTE_STRING }, + [XEN_ELFNOTE_L1_MFN_VALID ] = {"L1_MFN_VALID" ,ELFNOTE_NAME }, + [XEN_ELFNOTE_SUSPEND_CANCEL ] = {"SUSPEND_CANCEL" ,ELFNOTE_INT }, + [XEN_ELFNOTE_MOD_START_PFN ] = {"MOD_START_PFN" ,ELFNOTE_INT }, + [XEN_ELFNOTE_PHYS32_ENTRY ] = {"PHYS32_ENTRY" ,ELFNOTE_INT }, };/* *INDENT-ON* */
@@ -136,7 +140,7 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,return 0 ; } -if ( note_desc[type].str ) +if ( note_desc[type].type ==ELFNOTE_STRING ) { str =elf_strval (elf,elf_note_desc (elf, note));if (str ==NULL )
@@ -146,13 +150,17 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, parms->elf_notes[type].type =XEN_ENT_STR ; parms->elf_notes[type].data.str = str; } -else +else if ( note_desc[type].type ==ELFNOTE_INT ) { val =elf_note_numeric (elf, note);elf_msg (elf,"ELF: note: %s = %#" PRIx64"\n" , note_desc[type].name, val); parms->elf_notes[type].type =XEN_ENT_LONG ; parms->elf_notes[type].data.num = val; } +else if ( note_desc[type].type ==ELFNOTE_NAME ) + { +elf_msg (elf,"ELF: note: %s\n" , note_desc[type].name); + } parms->elf_notes[type].name = note_desc[type].name; switch ( type )
Signature · 1 line
2.44.0
Re: [PATCH v3 2/3] libelf: Expand ELF note printing
xen/common/libelf/libelf-dominfo.c elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, · line 107
Quoted 11 lines · Jason Andryuk
xen/common/libelf/libelf-dominfo.c +4
-1@@ -101,26 +101,30 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,/* *INDENT-OFF* */ static const struct { const char *name; - bool str; + enum { + ELFNOTE_INT, + ELFNOTE_STRING, + ELFNOTE_NAME,
This last on could do with a comment, as there are two possible (but distinct) purposes: Notes where the mere presence matters vs ones where printing of the value(s) would simply be cumbersome.
Quoted 56 lines
xen/common/libelf/libelf-dominfo.c +21
-20+ } type; } note_desc[] = { - [XEN_ELFNOTE_ENTRY] = { "ENTRY", 0}, - [XEN_ELFNOTE_HYPERCALL_PAGE] = { "HYPERCALL_PAGE", 0}, - [XEN_ELFNOTE_VIRT_BASE] = { "VIRT_BASE", 0}, - [XEN_ELFNOTE_INIT_P2M] = { "INIT_P2M", 0}, - [XEN_ELFNOTE_PADDR_OFFSET] = { "PADDR_OFFSET", 0}, - [XEN_ELFNOTE_HV_START_LOW] = { "HV_START_LOW", 0}, - [XEN_ELFNOTE_XEN_VERSION] = { "XEN_VERSION", 1}, - [XEN_ELFNOTE_GUEST_OS] = { "GUEST_OS", 1}, - [XEN_ELFNOTE_GUEST_VERSION] = { "GUEST_VERSION", 1}, - [XEN_ELFNOTE_LOADER] = { "LOADER", 1}, - [XEN_ELFNOTE_PAE_MODE] = { "PAE_MODE", 1}, - [XEN_ELFNOTE_FEATURES] = { "FEATURES", 1}, - [XEN_ELFNOTE_SUPPORTED_FEATURES] = { "SUPPORTED_FEATURES", 0}, - [XEN_ELFNOTE_BSD_SYMTAB] = { "BSD_SYMTAB", 1}, - [XEN_ELFNOTE_L1_MFN_VALID] = { "L1_MFN_VALID", false }, - [XEN_ELFNOTE_SUSPEND_CANCEL] = { "SUSPEND_CANCEL", 0 }, - [XEN_ELFNOTE_MOD_START_PFN] = { "MOD_START_PFN", 0 }, - [XEN_ELFNOTE_PHYS32_ENTRY] = { "PHYS32_ENTRY", 0 }, + [XEN_ELFNOTE_ENTRY] = { "ENTRY", ELFNOTE_INT }, + [XEN_ELFNOTE_HYPERCALL_PAGE] = { "HYPERCALL_PAGE", ELFNOTE_INT }, + [XEN_ELFNOTE_VIRT_BASE] = { "VIRT_BASE", ELFNOTE_INT }, + [XEN_ELFNOTE_INIT_P2M] = { "INIT_P2M", ELFNOTE_INT }, + [XEN_ELFNOTE_PADDR_OFFSET] = { "PADDR_OFFSET", ELFNOTE_INT }, + [XEN_ELFNOTE_HV_START_LOW] = { "HV_START_LOW", ELFNOTE_INT }, + [XEN_ELFNOTE_XEN_VERSION] = { "XEN_VERSION", ELFNOTE_STRING }, + [XEN_ELFNOTE_GUEST_OS] = { "GUEST_OS", ELFNOTE_STRING }, + [XEN_ELFNOTE_GUEST_VERSION] = { "GUEST_VERSION", ELFNOTE_STRING }, + [XEN_ELFNOTE_LOADER] = { "LOADER", ELFNOTE_STRING }, + [XEN_ELFNOTE_PAE_MODE] = { "PAE_MODE", ELFNOTE_STRING }, + [XEN_ELFNOTE_FEATURES] = { "FEATURES", ELFNOTE_STRING }, + [XEN_ELFNOTE_SUPPORTED_FEATURES] = { "SUPPORTED_FEATURES", ELFNOTE_INT }, + [XEN_ELFNOTE_BSD_SYMTAB] = { "BSD_SYMTAB", ELFNOTE_STRING }, + [XEN_ELFNOTE_L1_MFN_VALID] = { "L1_MFN_VALID", ELFNOTE_NAME }, + [XEN_ELFNOTE_SUSPEND_CANCEL] = { "SUSPEND_CANCEL", ELFNOTE_INT }, + [XEN_ELFNOTE_MOD_START_PFN] = { "MOD_START_PFN", ELFNOTE_INT }, + [XEN_ELFNOTE_PHYS32_ENTRY] = { "PHYS32_ENTRY", ELFNOTE_INT }, };/* *INDENT-ON* */ @@ -136,7 +140,7 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,return 0 ; } -if ( note_desc[type].str ) +if ( note_desc[type].type ==ELFNOTE_STRING ) { str =elf_strval (elf,elf_note_desc (elf, note));if (str ==NULL )@@ -146,13 +150,17 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, parms->elf_notes[type].type =XEN_ENT_STR ; parms->elf_notes[type].data.str = str; } -else +else if ( note_desc[type].type ==ELFNOTE_INT )
xen/common/libelf/libelf-dominfo.c elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, · line 163
Quoted 10 lines
xen/common/libelf/libelf-dominfo.c +4
-0{ val =elf_note_numeric (elf, note);elf_msg (elf,"ELF: note: %s = %#" PRIx64"\n" , note_desc[type].name, val); parms->elf_notes[type].type =XEN_ENT_LONG ; parms->elf_notes[type].data.num = val; } + else if ( note_desc[type].type == ELFNOTE_NAME ) + { +elf_msg (elf,"ELF: note: %s\n" , note_desc[type].name); + }
Imo this would better be expressed as (default-less) switch() now. That would also save me from commenting on the excess braces in the last "else if". I guess I wouldn't mind making both adjustments while committing, so long as you agree. With them in place
Jan
PATCH v3 3/3 x86/PVH: Support relocatable dom0 kernels
In reply to Jason AndryukXen tries to load a PVH dom0 kernel at the fixed guest physical address from the elf headers. For Linux, this defaults to 0x1000000 (16MB), but it can be configured.
Unfortunately there exist firmwares that have reserved regions at this address, so Xen fails to load the dom0 kernel since it's not RAM.
The PVH entry code is not relocatable - it loads from absolute addresses, which fail when the kernel is loaded at a different address. With a suitably modified kernel, a reloctable entry point is possible.
Add XEN_ELFNOTE_PHYS32_RELOC which specifies optional minimum, maximum and alignment needed for the kernel. The presence of the NOTE indicates the kernel supports a relocatable entry path.
Change the loading to check for an acceptable load address. If the kernel is relocatable, support finding an alternate load address.
---
ELF Note printing looks like:
(XEN) ELF: note: PHYS32_RELOC = max: 0xffffffff min: 0x1000000 align: 0x200000
v2:
Use elfnote for min, max & align - use 64bit values. Print original and relocated memory addresses
Use check_and_adjust_load_address() name
Return relocated base instead of offset
Use PAGE_ALIGN
Don't load above max_phys (expected to be 4GB in kernel elf note) Use single line comments
Exit check_load_address loop earlier
Add __init to find_kernel_memory()
v3:
Remove kernel_start/end page rounding Change loop comment to rely on a sorted memory map. Reorder E820_RAM check first
Use %p for dest_base
Use PRIpaddr
Use 32bit phys_min/max/align
Consolidate to if ( x || y ) continue
Use max_t
Add parms->phys_reloc
Use common "%pd kernel: " prefix for messages Re-order phys_entry assignment
Print range ends inclusively
Remove extra "Unable to load kernel" message s/PVH_RELOCATION/PHYS32_RELOC/
Make PHYS32_RELOC contents optional
Use 2MB default alignment
Update ELF Note comment
Update XEN_ELFNOTE_MAX
--- xen/arch/x86/hvm/dom0_build.c | 94 ++++++++++++++++++++++++++++++ xen/common/libelf/libelf-dominfo.c | 33 +++++++++++ xen/include/public/elfnote.h | 18 +++++- xen/include/xen/libelf.h | 4 ++ 4 files changed, 148 insertions(+), 1 deletion(-)
xen/arch/x86/hvm/dom0_build.c +94 -0
@@ -537,6 +537,97 @@ static paddr_t __init find_memory(return INVALID_PADDR ; } +static bool __init check_load_address ( +const struct domain *d,const struct elf_binary *elf) +{ +paddr_t kernel_start = (paddr_t )elf->dest_base; +paddr_t kernel_end = kernel_start + elf->dest_size; +unsigned int i; + +/* Relies on a sorted memory map. */ +for ( i =0 ; i < d->arch.nr_e820; i++ ) + { +paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size; + +if ( start >= kernel_end ) +return false; + +if ( d->arch.e820[i].type ==E820_RAM && + start <= kernel_start && + end >= kernel_end ) +return true; + } + +return false; +} + +/* Find an e820 RAM region that fits the kernel at a suitable alignment. */ +static paddr_t __init find_kernel_memory ( +const struct domain *d,struct elf_binary *elf, +const struct elf_dom_parms *parms) +{ +paddr_t kernel_size = elf->dest_size; +unsigned int i; + +for ( i =0 ; i < d->arch.nr_e820; i++ ) + { +paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size; +paddr_t kstart, kend; + +if ( d->arch.e820[i].type !=E820_RAM || + d->arch.e820[i].size < kernel_size ) +continue ; + + kstart =ROUNDUP (start, parms->phys_align); + kstart =max_t (paddr_t, kstart, parms->phys_min); + kend = kstart + kernel_size; + +if ( kend > parms->phys_max ) +return 0 ; + +if ( kend <= end ) +return kstart; + } + +return 0 ; +} + +/* Check the kernel load address, and adjust if necessary and possible. */ +static bool __init check_and_adjust_load_address ( +const struct domain *d,struct elf_binary *elf,struct elf_dom_parms *parms) +{ +paddr_t reloc_base; + +if (check_load_address (d, elf) ) +return true; + +if ( !parms->phys_reloc ) + { +printk ("%pd kernel: Address conflict and not relocatable\n" , d); +return false; + } + + reloc_base =find_kernel_memory (d, elf, parms); +if ( reloc_base ==0 ) + { +printk ("%pd kernel: Failed find a load address\n" , d); +return false; + } + +if ( opt_dom0_verbose ) +printk ("%pd kernel: Moving [%p, %p] -> [%" PRIpaddr", %" PRIpaddr"]\n" , d, + elf->dest_base, elf->dest_base + elf->dest_size -1 , + reloc_base, reloc_base + elf->dest_size -1 ); + + parms->phys_entry = reloc_base + + (parms->phys_entry - (paddr_t )elf->dest_base); + elf->dest_base = (char *)reloc_base; + +return true; +} + static int __init pvh_load_kernel(struct domain *d, const module_t *image, unsigned long image_headroom, module_t *initrd, void *image_base,
@@ -585,6 +676,9 @@ static int __init pvh_load_kernel(struct domain *d, const module_t *image, elf.dest_base = (void *)(parms.virt_kstart - parms.virt_base); elf.dest_size = parms.virt_kend - parms.virt_kstart; +if ( !check_and_adjust_load_address (d, &elf, &parms) ) +return -ENOMEM ; +elf_set_vcpu (&elf, v); rc =elf_load_binary (&elf);if ( rc <0 )
xen/common/libelf/libelf-dominfo.c +33 -0
@@ -105,6 +105,7 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,ELFNOTE_INT ,ELFNOTE_STRING ,ELFNOTE_NAME , +ELFNOTE_CUSTOM , } type; } note_desc[] = { [XEN_ELFNOTE_ENTRY] = { "ENTRY", ELFNOTE_INT },
@@ -125,6 +126,7 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, [XEN_ELFNOTE_SUSPEND_CANCEL] = { "SUSPEND_CANCEL", ELFNOTE_INT }, [XEN_ELFNOTE_MOD_START_PFN] = { "MOD_START_PFN", ELFNOTE_INT }, [XEN_ELFNOTE_PHYS32_ENTRY] = { "PHYS32_ENTRY", ELFNOTE_INT }, + [XEN_ELFNOTE_PHYS32_RELOC] = { "PHYS32_RELOC", ELFNOTE_CUSTOM }, };/* *INDENT-ON* */
@@ -132,6 +134,7 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,uint64_t val =0 ;unsigned int i;unsigned type =elf_uval (elf, note, type); +unsigned descsz =elf_uval (elf, note, descsz);if ( (type >=sizeof (note_desc) /sizeof (note_desc[0 ])) || (note_desc[type].name ==NULL ) )
@@ -161,6 +164,10 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, {elf_msg (elf,"ELF: note: %s\n" , note_desc[type].name); } + else if ( note_desc[type].type == ELFNOTE_CUSTOM ) + { +elf_msg (elf,"ELF: note: %s" , note_desc[type].name); + } parms->elf_notes[type].name = note_desc[type].name; switch ( type )
@@ -225,6 +232,28 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,case XEN_ELFNOTE_PHYS32_ENTRY : parms->phys_entry = val;break ; + +case XEN_ELFNOTE_PHYS32_RELOC : + parms->phys_reloc = true; + +if ( descsz >=4 ) + { + parms->phys_max =elf_note_numeric_array (elf, note,4 ,0 ); +elf_msg (elf," = max: %#" PRIx32, parms->phys_max); + } +if ( descsz >=8 ) + { + parms->phys_min =elf_note_numeric_array (elf, note,4 ,1 ); +elf_msg (elf," min: %#" PRIx32, parms->phys_min); + } +if ( descsz >=12 ) + { + parms->phys_align =elf_note_numeric_array (elf, note,4 ,2 ); +elf_msg (elf," align: %#" PRIx32, parms->phys_align); + } + +elf_msg (elf,"\n" ); +break ; }return 0 ; }
@@ -536,6 +565,10 @@ elf_errorstatus elf_xen_parse(struct elf_binary *elf, parms->p2m_base =UNSET_ADDR ; parms->elf_paddr_offset =UNSET_ADDR ; parms->phys_entry =UNSET_ADDR32 ; + parms->phys_min =0 ; + parms->phys_max =0xffffffff ; + parms->phys_align =0x200000 ; + parms->phys_reloc = false;/* Find and parse elf notes. */ count =elf_phdr_count (elf);
xen/include/public/elfnote.h +17 -1
@@ -194,10 +194,26 @@*/ #define XEN_ELFNOTE_PHYS32_ENTRY 18 +/* +* Physical loading constraints for PVH kernels +* +* Used to place constraints on the guest physical loading addresses and +* alignment for a PVH kernel. +* +* The presence of this note indicates the kernel supports relocating itself. +* +* The note may include up to three 32bit values in the following order: +* - a maximum address for the entire image to be loaded below (default +* 0xffffffff) +* - a minimum address for the start of the image (default 0) +* - a required start alignment (default 0x200000) +*/ +#define XEN_ELFNOTE_PHYS32_RELOC 19 +/* * The number of the highest elfnote defined. */ -#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_ENTRY +#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_RELOC/* * System information exported through crash notes.
xen/include/xen/libelf.h +4 -0
@@ -434,10 +434,14 @@ struct elf_dom_parms {uint32_t f_supported[XENFEAT_NR_SUBMAPS ];uint32_t f_required[XENFEAT_NR_SUBMAPS ];uint32_t phys_entry; +uint32_t phys_min; +uint32_t phys_max; +uint32_t phys_align;/* calculated */ uint64_t virt_kstart;uint64_t virt_kend; +bool phys_reloc; };static inline void elf_xen_feature_set (int nr,uint32_t * addr)
Signature · 1 line
2.44.0
Re: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 543
Quoted 10 lines · Jason Andryuk
xen/arch/x86/hvm/dom0_build.c +4
-0@@ -537,6 +537,97 @@ static paddr_t __init find_memory(return INVALID_PADDR ; } +static bool __init check_load_address ( +const struct domain *d,const struct elf_binary *elf) +{ +paddr_t kernel_start = (paddr_t )elf->dest_base;
As before I think it is illegitimate to cast a pointer to paddr_t. The variable type wants to remain such, but the cast wants to be to unsigned long or uintptr_t (or else at least a comment wants adding).
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 551
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c +8
-0+paddr_t kernel_end = kernel_start + elf->dest_size; +unsigned int i; + +/* Relies on a sorted memory map. */ +for ( i =0 ; i < d->arch.nr_e820; i++ ) + { +paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size;
Nit: Shorter if you use "start" here (also again in the other function).
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 556
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+if ( start >= kernel_end ) +return false; + + if ( d->arch.e820[i].type == E820_RAM &&
Nit: Excess blank before &&.
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 559
Quoted 3 lines
xen/arch/x86/hvm/dom0_build.c +3
-0+ start <= kernel_start && + end >= kernel_end ) +return true;
As to the comment ahead of the function: This further relies on adjacent entries of the same type having got merged.
Quoted 15 lines
xen/arch/x86/hvm/dom0_build.c +15
-0+ } + +return false; +} + +/* Find an e820 RAM region that fits the kernel at a suitable alignment. */ +static paddr_t __init find_kernel_memory ( +const struct domain *d,struct elf_binary *elf, +const struct elf_dom_parms *parms) +{ +paddr_t kernel_size = elf->dest_size; +unsigned int i; + +for ( i =0 ; i < d->arch.nr_e820; i++ ) + {
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 584
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size; +paddr_t kstart, kend; + +if ( d->arch.e820[i].type !=E820_RAM || + d->arch.e820[i].size < kernel_size ) +continue ; + + kstart =ROUNDUP (start, parms->phys_align); + kstart =max_t (paddr_t, kstart, parms->phys_min);
I'd generally try to avoid max_t(), but I cannot think of any good way of expressing this without using it.
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 587
Quoted 3 lines
xen/arch/x86/hvm/dom0_build.c +3
-0+ kend = kstart + kernel_size; + +if ( kend > parms->phys_max )
So despite its default phys_max is exclusive aiui. Otherwise with kend being exclusive too, this would look to be off by one.
Quoted 12 lines
xen/arch/x86/hvm/dom0_build.c +12
-0+return 0 ; + +if ( kend <= end ) +return kstart; + } + +return 0 ; +} + +/* Check the kernel load address, and adjust if necessary and possible. */ +static bool __init check_and_adjust_load_address ( +const struct domain *d,struct elf_binary *elf,struct elf_dom_parms *parms)
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 609
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+{ +paddr_t reloc_base; + +if (check_load_address (d, elf) ) +return true; + +if ( !parms->phys_reloc ) + { +printk ("%pd kernel: Address conflict and not relocatable\n" , d); +return false;
This better wouldn't result in -ENOMEM in the caller. Then again reasons are logged, so the specific error code perhaos doesn't matter that much.
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 613
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+ } + + reloc_base =find_kernel_memory (d, elf, parms); +if ( reloc_base ==0 )
With ! used above please be consistent and do so here, too.
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 626
Quoted 13 lines
xen/arch/x86/hvm/dom0_build.c +13
-0+ { +printk ("%pd kernel: Failed find a load address\n" , d); +return false; + } + +if ( opt_dom0_verbose ) +printk ("%pd kernel: Moving [%p, %p] -> [%" PRIpaddr", %" PRIpaddr"]\n" , d, + elf->dest_base, elf->dest_base + elf->dest_size -1 , + reloc_base, reloc_base + elf->dest_size -1 ); + + parms->phys_entry = reloc_base + + (parms->phys_entry - (paddr_t )elf->dest_base); + elf->dest_base = (char *)reloc_base;
Just as a remark, no request to change anything: We're not dealing with strings here. Hence char * isn't quite right, just that "dest_base" is of that type (for whatever reason).
xen/arch/x86/hvm/dom0_build.c elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, · line 170
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c +4
-0@@ -161,6 +164,10 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, {elf_msg (elf,"ELF: note: %s\n" , note_desc[type].name); } + else if ( note_desc[type].type == ELFNOTE_CUSTOM ) + { +elf_msg (elf,"ELF: note: %s" , note_desc[type].name); + }
This could do with a brief comment, to make clear the \n isn't accidentally missing here.
Quoted 15 lines
xen/arch/x86/hvm/dom0_build.c +11
-0@@ -225,6 +232,28 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,case XEN_ELFNOTE_PHYS32_ENTRY : parms->phys_entry = val;break ; + +case XEN_ELFNOTE_PHYS32_RELOC : + parms->phys_reloc = true; + +if ( descsz >=4 ) + { + parms->phys_max =elf_note_numeric_array (elf, note,4 ,0 ); +elf_msg (elf," = max: %#" PRIx32, parms->phys_max); + } +if ( descsz >=8 ) + {
xen/arch/x86/hvm/dom0_build.c elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, · line 255
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+ parms->phys_min =elf_note_numeric_array (elf, note,4 ,1 ); +elf_msg (elf," min: %#" PRIx32, parms->phys_min); + } +if ( descsz >=12 ) + { + parms->phys_align =elf_note_numeric_array (elf, note,4 ,2 ); +elf_msg (elf," align: %#" PRIx32, parms->phys_align); + } + +elf_msg (elf,"\n" );
Imo the newline wants adding outside of the switch(), for everything set to ELFNOTE_CUSTOM. In fact with that I think ELFNOTE_CUSTOM and ELFNOTE_NAME could be folded (with what's NAME right now simply printing nothing in the switch here). Which suggests that I may better not commit the (slightly adjusted) earlier patch, yet.
xen/arch/x86/hvm/dom0_build.c elf_errorstatus elf_xen_parse(struct elf_binary *elf, · line 570
Quoted 7 lines
xen/arch/x86/hvm/dom0_build.c +3
-0@@ -536,6 +565,10 @@ elf_errorstatus elf_xen_parse(struct elf_binary *elf, parms->p2m_base =UNSET_ADDR ; parms->elf_paddr_offset =UNSET_ADDR ; parms->phys_entry =UNSET_ADDR32 ; + parms->phys_min =0 ; + parms->phys_max =0xffffffff ; + parms->phys_align =0x200000 ;
I think this wants to be MB(2) (requiring a pre-patch to centralize MB() in the tool stack to tools/include/xen-tools/common-macros.h). And I further think this needs to be an arch-specific constant, even if right now the note is expected to be present only for x86. Which then also needs saying ...
Quoted 11 lines
xen/include/public/elfnote.h +5
-0@@ -194,10 +194,26 @@*/ #define XEN_ELFNOTE_PHYS32_ENTRY 18 +/* +* Physical loading constraints for PVH kernels +* +* Used to place constraints on the guest physical loading addresses and +* alignment for a PVH kernel.
xen/include/public/elfnote.h line 211
Quoted 10 lines
xen/include/public/elfnote.h +10
-0+* +* The presence of this note indicates the kernel supports relocating itself. +* +* The note may include up to three 32bit values in the following order: +* - a maximum address for the entire image to be loaded below (default +* 0xffffffff) +* - a minimum address for the start of the image (default 0) +* - a required start alignment (default 0x200000) +*/ +#define XEN_ELFNOTE_PHYS32_RELOC 19
... in the comment here. The reasoning behind not (intermediately) falling back to the alignment in the ELF headers also wants adding to the patch description (or a code comment, albeit there's no really good place to put it, imo).
Additionally I think the pieces of the comment want re-ordering. The primary purpose is indicating relocatability; when not relocating, the constraints aren't applied in any way.
Jan
Stefano Stabellini
maintainerRe: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
Quoted 7 lines · Jan Beulich
Quoted 6 lines · Jason Andryuk
xen/arch/x86/hvm/dom0_build.c +0
-0@@ -537,6 +537,97 @@ static paddr_t __init find_memory(return INVALID_PADDR ; }
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 543
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+static bool __init check_load_address ( +const struct domain *d,const struct elf_binary *elf) +{ +paddr_t kernel_start = (paddr_t )elf->dest_base;As before I think it is illegitimate to cast a pointer to
paddr_t. The variable type wants to remain such, but the cast wants to be to unsigned long oruintptr_t(or else at least a comment wants adding).
uintptr_t is a good suggestion. uintptr_t is the recommended way by the C standard and MISRA by extension to cast integers to points and vice versa. In Xen we have used unsigned long for the same purpose although it is not the best way any longer (something else to document).
Re: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
In reply to Jan BeulichQuoted 7 lines · Jan Beulich
Quoted 6 lines · Jason Andryuk
xen/arch/x86/hvm/dom0_build.c +0
-0@@ -537,6 +537,97 @@ static paddr_t __init find_memory(return INVALID_PADDR ; }
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 543
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+static bool __init check_load_address ( +const struct domain *d,const struct elf_binary *elf) +{ +paddr_t kernel_start = (paddr_t )elf->dest_base;As before I think it is illegitimate to cast a pointer to
paddr_t. The variable type wants to remain such, but the cast wants to be to unsigned long oruintptr_t(or else at least a comment wants adding).
Ok, thanks. This explains it clearly.
Quoted 24 lines
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+/* Find an e820 RAM region that fits the kernel at a suitable alignment. */ +static paddr_t __init find_kernel_memory ( +const struct domain *d,struct elf_binary *elf, +const struct elf_dom_parms *parms) +{ +paddr_t kernel_size = elf->dest_size; +unsigned int i; + +for ( i =0 ; i < d->arch.nr_e820; i++ ) + {xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 584
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size; +paddr_t kstart, kend; + +if ( d->arch.e820[i].type !=E820_RAM || + d->arch.e820[i].size < kernel_size ) +continue ; + + kstart =ROUNDUP (start, parms->phys_align); + kstart =max_t (paddr_t, kstart, parms->phys_min);I'd generally try to avoid
max_t(), but I cannot think of any good way of expressing this without using it.
Quoted 6 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 587
Quoted 3 lines
xen/arch/x86/hvm/dom0_build.c +3
-0+ kend = kstart + kernel_size; + +if ( kend > parms->phys_max )So despite its default
phys_maxis exclusive aiui. Otherwise with kend being exclusive too, this would look to be off by one.
Yes, I'll fix the off-by-one. Hmmm, phys_max being 32bit, we want it to be inclusive to represent the maximum range.
Quoted 17 lines
Quoted 17 lines
xen/arch/x86/hvm/dom0_build.c +17
-0+return 0 ; + +if ( kend <= end ) +return kstart; + } + +return 0 ; +} + +/* Check the kernel load address, and adjust if necessary and possible. */ +static bool __init check_and_adjust_load_address ( +const struct domain *d,struct elf_binary *elf,struct elf_dom_parms *parms) +{ +paddr_t reloc_base; + +if (check_load_address (d, elf) ) +return true;
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 609
Quoted 5 lines
xen/arch/x86/hvm/dom0_build.c +5
-0+ +if ( !parms->phys_reloc ) + { +printk ("%pd kernel: Address conflict and not relocatable\n" , d); +return false;This better wouldn't result in
-ENOMEMin the caller. Then again reasons are logged, so the specific error code perhaos doesn't matter that much.
Failure here is turned into -ENOMEM by pvh_load_kernel(). -ENOMEM is already returned for later failure with find_memory(), so I thought it was acceptable. Without this code, elf_load_binary() would failed with -1 and that would be returned. I'll change it to whatever you prefer.
Quoted 6 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 613
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+ } + + reloc_base =find_kernel_memory (d, elf, parms); +if ( reloc_base ==0 )With ! used above please be consistent and do so here, too.
phys_reloc is a bool, and reloc_base is a paddr_t.
Quoted 9 lines
Quoted 9 lines
xen/arch/x86/hvm/dom0_build.c +9
-0+ { +printk ("%pd kernel: Failed find a load address\n" , d); +return false; + } + +if ( opt_dom0_verbose ) +printk ("%pd kernel: Moving [%p, %p] -> [%" PRIpaddr", %" PRIpaddr"]\n" , d, + elf->dest_base, elf->dest_base + elf->dest_size -1 , + reloc_base, reloc_base + elf->dest_size -1 );
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 626
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+ + parms->phys_entry = reloc_base + + (parms->phys_entry - (paddr_t )elf->dest_base); + elf->dest_base = (char *)reloc_base;Just as a remark, no request to change anything: We're not dealing with strings here. Hence char * isn't quite right, just that "
dest_base" is of that type (for whatever reason).
I think the reason is just to be byte addressable.
Quoted 23 lines
Quoted 23 lines
xen/arch/x86/hvm/dom0_build.c +19
-0@@ -225,6 +232,28 @@ elf_errorstatus elf_xen_parse_note(struct elf_binary *elf,case XEN_ELFNOTE_PHYS32_ENTRY : parms->phys_entry = val;break ; + +case XEN_ELFNOTE_PHYS32_RELOC : + parms->phys_reloc = true; + +if ( descsz >=4 ) + { + parms->phys_max =elf_note_numeric_array (elf, note,4 ,0 ); +elf_msg (elf," = max: %#" PRIx32, parms->phys_max); + } +if ( descsz >=8 ) + { + parms->phys_min =elf_note_numeric_array (elf, note,4 ,1 ); +elf_msg (elf," min: %#" PRIx32, parms->phys_min); + } +if ( descsz >=12 ) + { + parms->phys_align =elf_note_numeric_array (elf, note,4 ,2 ); +elf_msg (elf," align: %#" PRIx32, parms->phys_align); + }
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c elf_errorstatus elf_xen_parse_note(struct elf_binary *elf, · line 255
Quoted 2 lines
xen/arch/x86/hvm/dom0_build.c +2
-0+ +elf_msg (elf,"\n" );Imo the newline wants adding outside of the
switch(), for everything set toELFNOTE_CUSTOM. In fact with that I thinkELFNOTE_CUSTOMandELFNOTE_NAMEcould be folded (with what's NAME right now simply printing nothing in the switch here). Which suggests that I may better not commit the (slightly adjusted) earlier patch, yet.
Yes, please hold off and I'll re-work both.
Quoted 5 lines
Quoted 5 lines
xen/arch/x86/hvm/dom0_build.c +1
-0@@ -536,6 +565,10 @@ elf_errorstatus elf_xen_parse(struct elf_binary *elf, parms->p2m_base =UNSET_ADDR ; parms->elf_paddr_offset =UNSET_ADDR ; parms->phys_entry =UNSET_ADDR32 ; + parms->phys_min =0 ;
Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c elf_errorstatus elf_xen_parse(struct elf_binary *elf, · line 570
Quoted 2 lines
xen/arch/x86/hvm/dom0_build.c +2
-0+ parms->phys_max =0xffffffff ; + parms->phys_align =0x200000 ;I think this wants to be MB(2) (requiring a pre-patch to centralize
MB()in the tool stack totools/include/xen-tools/common-macros.h). And I further think this needs to be an arch-specific constant, even if right now the note is expected to be present only for x86. Which then also needs saying ...
Ok, I'll look into this.
Quoted 27 lines
Quoted 11 lines
xen/include/public/elfnote.h +5
-0@@ -194,10 +194,26 @@*/ #define XEN_ELFNOTE_PHYS32_ENTRY 18 +/* +* Physical loading constraints for PVH kernels +* +* Used to place constraints on the guest physical loading addresses and +* alignment for a PVH kernel. xen/include/public/elfnote.h line 211
Quoted 10 lines
xen/include/public/elfnote.h +10
-0+* +* The presence of this note indicates the kernel supports relocating itself. +* +* The note may include up to three 32bit values in the following order: +* - a maximum address for the entire image to be loaded below (default +* 0xffffffff) +* - a minimum address for the start of the image (default 0) +* - a required start alignment (default 0x200000) +*/ +#define XEN_ELFNOTE_PHYS32_RELOC 19... in the comment here. The reasoning behind not (intermediately) falling back to the alignment in the ELF headers also wants adding to the patch description (or a code comment, albeit there's no really good place to put it, imo).
Quoted 3 lines
Additionally I think the pieces of the comment want re-ordering. The primary purpose is indicating relocatability; when not relocating, the constraints aren't applied in any way.
Yes, this is a good point.
Thanks for the review.
-Jason
Re: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
Quoted 30 lines · Jason Andryuk
Quoted 29 lines · Jan Beulich
Quoted 10 lines · Jason Andryuk
xen/arch/x86/hvm/dom0_build.c +10
-0+/* Find an e820 RAM region that fits the kernel at a suitable alignment. */ +static paddr_t __init find_kernel_memory ( +const struct domain *d,struct elf_binary *elf, +const struct elf_dom_parms *parms) +{ +paddr_t kernel_size = elf->dest_size; +unsigned int i; + +for ( i =0 ; i < d->arch.nr_e820; i++ ) + {xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 584
Quoted 10 lines
xen/arch/x86/hvm/dom0_build.c +10
-0+paddr_t start = d->arch.e820[i].addr; +paddr_t end = d->arch.e820[i].addr + d->arch.e820[i].size; +paddr_t kstart, kend; + +if ( d->arch.e820[i].type !=E820_RAM || + d->arch.e820[i].size < kernel_size ) +continue ; + + kstart =ROUNDUP (start, parms->phys_align); + kstart =max_t (paddr_t, kstart, parms->phys_min);I'd generally try to avoid
max_t(), but I cannot think of any good way of expressing this without using it.Quoted 3 lines
xen/arch/x86/hvm/dom0_build.c +3
-0+ kend = kstart + kernel_size; + +if ( kend > parms->phys_max )
Quoted 5 lines
Quoted 2 lines
So despite its default
phys_maxis exclusive aiui. Otherwise with kend being exclusive too, this would look to be off by one.Yes, I'll fix the off-by-one. Hmmm,
phys_maxbeing 32bit, we want it to be inclusive to represent the maximum range.
I specifically didn't ask for that, as I think it would end up a little awkward. But I also don't mind you adjusting things to that effect.
Quoted 26 lines
Quoted 17 lines
Quoted 17 lines
xen/arch/x86/hvm/dom0_build.c +17
-0+return 0 ; + +if ( kend <= end ) +return kstart; + } + +return 0 ; +} + +/* Check the kernel load address, and adjust if necessary and possible. */ +static bool __init check_and_adjust_load_address ( +const struct domain *d,struct elf_binary *elf,struct elf_dom_parms *parms) +{ +paddr_t reloc_base; + +if (check_load_address (d, elf) ) +return true;Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 609
Quoted 5 lines
xen/arch/x86/hvm/dom0_build.c +5
-0+ +if ( !parms->phys_reloc ) + { +printk ("%pd kernel: Address conflict and not relocatable\n" , d); +return false;This better wouldn't result in
-ENOMEMin the caller. Then again reasons are logged, so the specific error code perhaos doesn't matter that much.
Quoted 4 lines
Failure here is turned into
-ENOMEMbypvh_load_kernel().-ENOMEMis
already returned for later failure withfind_memory(), so I thought it was acceptable. Without this code,elf_load_binary()would failed with -1 and that would be returned. I'll change it to whatever you prefer.
ENOSPC would likely make it easily distinguishable from anything else.
Quoted 8 lines
Quoted 6 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 613
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+ } + + reloc_base =find_kernel_memory (d, elf, parms); +if ( reloc_base ==0 )With ! used above please be consistent and do so here, too.
phys_relocis a bool, andreloc_baseis apaddr_t.
Hmm, I see. But still - we generally prefer ! over "== 0" (or even "== NULL").
Quoted 18 lines
Quoted 9 lines
Quoted 9 lines
xen/arch/x86/hvm/dom0_build.c +9
-0+ { +printk ("%pd kernel: Failed find a load address\n" , d); +return false; + } + +if ( opt_dom0_verbose ) +printk ("%pd kernel: Moving [%p, %p] -> [%" PRIpaddr", %" PRIpaddr"]\n" , d, + elf->dest_base, elf->dest_base + elf->dest_size -1 , + reloc_base, reloc_base + elf->dest_size -1 );Quoted 8 lines
xen/arch/x86/hvm/dom0_build.c static paddr_t __init find_memory( · line 626
Quoted 4 lines
xen/arch/x86/hvm/dom0_build.c +4
-0+ + parms->phys_entry = reloc_base + + (parms->phys_entry - (paddr_t )elf->dest_base); + elf->dest_base = (char *)reloc_base;Just as a remark, no request to change anything: We're not dealing with strings here. Hence char * isn't quite right, just that "
dest_base" is of that type (for whatever reason).
Quoted 1 line
I think the reason is just to be byte addressable.
Sure, but that would call for void *, unsigned char *, or uint8_t *.
Jan
Re: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
In reply to Jason AndryukQuoted 17 lines · Jason Andryuk
Quoted 16 lines · Jan Beulich
xen/common/libelf/libelf-dominfo.c elf_errorstatus elf_xen_parse(struct elf_binary · line 571
Quoted 8 lines · Jason Andryuk
xen/common/libelf/libelf-dominfo.c +3
-0@@ -536,6 +565,10 @@ elf_errorstatus elf_xen_parse(struct elf_binary *elf, parms->p2m_base = UNSET_ADDR; parms->elf_paddr_offset = UNSET_ADDR; parms->phys_entry = UNSET_ADDR32; + parms->phys_min = 0; + parms->phys_max = 0xffffffff; + parms->phys_align = 0x200000;I think this wants to be MB(2) (requiring a pre-patch to centralize
MB()in the tool stack totools/include/xen-tools/common-macros.h). And I further think this needs to be an arch-specific constant, even if right now the note is expected to be present only for x86. Which then also needs saying ...
Are you thinking something like the following in libelf-dominfo.c:
#define X86_PHYS_ALIGN_DEFAULT MB(2)
#define X86_PHYS_MAX_DEFAULT (GB(4) - 1)
and setting as:parms->phys_max = X86_PHYS_MAX_DEFAULT;parms->phys_align = X86_PHYS_ALIGN_DEFAULT;
libelf is arch neutral, so there isn't a natural place to introduce arch-specific defines. Or were you looking for each arch to set it? We only care about x86 right now, so we can do something like:
#if x86
#define ARCH_PHYS_MAX_DEFAULT (GB(4) - 1)
#define ARCH_PHYS_ALIGN_DEFAULT MB(2)
#else
#define ARCH_PHYS_MAX_DEFAULT 0
#define ARCH_PHYS_ALIGN_DEFAULT 0
#endif
Thanks,
Jason
Re: [PATCH v3 3/3] x86/PVH: Support relocatable dom0 kernels
Quoted 32 lines · Jason Andryuk
Quoted 17 lines · Jason Andryuk
Quoted 16 lines · Jan Beulich
xen/common/libelf/libelf-dominfo.c elf_errorstatus elf_xen_parse(struct elf_binary · line 571
Quoted 8 lines · Jason Andryuk
xen/common/libelf/libelf-dominfo.c +3
-0@@ -536,6 +565,10 @@ elf_errorstatus elf_xen_parse(struct elf_binary *elf, parms->p2m_base = UNSET_ADDR; parms->elf_paddr_offset = UNSET_ADDR; parms->phys_entry = UNSET_ADDR32; + parms->phys_min = 0; + parms->phys_max = 0xffffffff; + parms->phys_align = 0x200000;I think this wants to be MB(2) (requiring a pre-patch to centralize
MB()in the tool stack totools/include/xen-tools/common-macros.h). And I further think this needs to be an arch-specific constant, even if right now the note is expected to be present only for x86. Which then also needs saying ...Are you thinking something like the following in
libelf-dominfo.c:#define
X86_PHYS_ALIGN_DEFAULTMB(2)
#defineX86_PHYS_MAX_DEFAULT(GB(4) - 1)and setting as:
parms->phys_max=X86_PHYS_MAX_DEFAULT;parms->phys_align=X86_PHYS_ALIGN_DEFAULT;libelf is arch neutral, so there isn't a natural place to introduce arch-specific defines. Or were you looking for each arch to set it? We only care about x86 right now, so we can do something like:
Quoted 7 lines
#if x86
#defineARCH_PHYS_MAX_DEFAULT(GB(4) - 1)
#defineARCH_PHYS_ALIGN_DEFAULTMB(2)
#else
#defineARCH_PHYS_MAX_DEFAULT0
#defineARCH_PHYS_ALIGN_DEFAULT0
#endif
More like the latter. The former only if the phys_* fields themselves were to also become x86-only.
As you say, libelf in its present shape doesn't easily lend itself to such arch-specifics.
Jan
PATCH v3 1/1 RFC: x86/pvh: Make Xen PVH entrypoint PIC for x86-64
In reply to Jason AndryukThe Xen PVH entrypoint is 32bit non-PIC code running at a default load address of 0x1000000 (16MB) (CONFIG_PHYSICAL_START). Xen loads the kernel at that physical address inside the PVH container.
When running a PVH Dom0, the system reserved addresses are mapped 1-1 into the PVH container. There exist system firmwares (Coreboot/EDK2) with reserved memory at 16MB. This creates a conflict where the PVH kernel cannot be loaded at that address.
Modify the PVH entrypoint to be position-indepedent to allow flexibility in load address. Only the 64bit entry path is converted. A 32bit kernel is not PIC, so calling into other parts of the kernel, like xen_prepare_pvh() and mk_pgtable_32(), don't work properly when relocated.
Initial PVH entry runs at the physical addresses and then transitions to the identity mapped address. While executing xen_prepare_pvh() calls through pv_ops function pointers transition to the high mapped addresses. Additionally, __va() is called on some hvm_start_info physical addresses, we need the directmap address range is used. So we need to run page tables with all of those ranges mapped.
Modifying init_top_pgt tables ran into issue since startup_64/__startup_64() will modify those page tables again. Use a dedicated set of page tables - pvh_init_top_pgt - for the PVH entry to avoid unwanted interactions.
In xen_pvh_init(), __pa() is called to find the physical address of the hypercall page. Set phys_base temporarily before calling into xen_prepare_pvh(), which calls xen_pvh_init(), and clear it afterwards. __startup_64() assumes phys_base is zero and adds load_delta to it. If phys_base is already set, the calculation results in an incorrect cr3.
TODO: Sync elfnote.h from xen.git commit xxxxxxxxxx when it is commited.
---
Put this out as an example for the Xen modifications
Instead of setting and clearing phys_base, add a dedicated variable? Clearing phys_base is a little weird, but it leaves the kernel more consistent when running non-entry code.
Make __startup_64() exit if phys_base is already set to allow calling multiple times, and use that and init_top_pgt instead of adding additional page tables? That won't work. __startup_64 is 64bit code, and pvh needs to create page tables in 32bit code before it can transition to 64bit long mode. Hence it can't be re-used to relocate page tables.
--- arch/x86/platform/pvh/head.S | 184 +++++++++++++++++++++++++++++--- include/xen/interface/elfnote.h | 18 +++- 2 files changed, 189 insertions(+), 13 deletions(-)
arch/x86/platform/pvh/head.S +172 -12
@@ -50,11 +50,32 @@ #define PVH_CS_SEL (PVH_GDT_ENTRY_CS * 8) #define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8) +#define rva(x) ((x) - pvh_start_xen) + SYM_CODE_START_LOCAL(pvh_start_xen) UNWIND_HINT_END_OF_STACK cld - lgdt (_pa(gdt)) + /* + * See the comment for startup_32 for more details. We need to + * execute a call to get the execution address to be position + * independent, but we don't have a stack. Save and restore the + * magic field of start_info in ebx, and use that as the stack. + */ + mov (%ebx), %eax + leal 4(%ebx), %esp + ANNOTATE_INTRA_FUNCTION_CALL + call 1f +1: popl %ebp + mov %eax, (%ebx) + subl $ rva(1b), %ebp + movl $0, %esp + + leal rva(gdt)(%ebp), %eax + movl %eax, %ecx + leal rva(gdt_start)(%ebp), %ecx + movl %ecx, 2(%eax) + lgdt (%eax) mov $PVH_DS_SEL,%eax mov %eax,%ds
@@ -62,14 +83,14 @@ SYM_CODE_START_LOCAL(pvh_start_xen) mov %eax,%ss /* Stash hvm_start_info. */ - mov $_pa(pvh_start_info), %edi + leal rva(pvh_start_info)(%ebp), %edi mov %ebx, %esi - mov _pa(pvh_start_info_sz), %ecx + movl rva(pvh_start_info_sz)(%ebp), %ecx shr $2,%ecx rep movsl - mov $_pa(early_stack_end), %esp + leal rva(early_stack_end)(%ebp), %esp /* Enable PAE mode. */ mov %cr4, %eax
@@ -83,29 +104,83 @@ SYM_CODE_START_LOCAL(pvh_start_xen) btsl $_EFER_LME, %eax wrmsr + mov %ebp, %ebx + subl $LOAD_PHYSICAL_ADDR, %ebx /* offset */ + jz .Lpagetable_done + + /* Fixup page-tables for relocation. */ + leal rva(pvh_init_top_pgt)(%ebp), %edi + movl $512, %ecx +2: + movl 0x00(%edi), %eax + addl 0x04(%edi), %eax + jz 1f + addl %ebx, 0x00(%edi) +1: + addl $8, %edi + decl %ecx + jnz 2b + + /* L3 ident has a single entry. */ + leal rva(pvh_level3_ident_pgt)(%ebp), %edi + addl %ebx, 0x00(%edi) + + leal rva(pvh_level3_kernel_pgt)(%ebp), %edi + addl %ebx, (4096 - 16)(%edi) + addl %ebx, (4096 - 8)(%edi) + + /* pvh_level2_ident_pgt is fine - large pages */ + + /* pvh_level2_kernel_pgt needs adjustment - large pages */ + leal rva(pvh_level2_kernel_pgt)(%ebp), %edi + movl $512, %ecx +2: + movl 0x00(%edi), %eax + addl 0x04(%edi), %eax + jz 1f + addl %ebx, 0x00(%edi) +1: + addl $8, %edi + decl %ecx + jnz 2b + +.Lpagetable_done: /* Enable pre-constructed page tables. */ - mov $_pa(init_top_pgt), %eax + leal rva(pvh_init_top_pgt)(%ebp), %eax mov %eax, %cr3 mov $(X86_CR0_PG | X86_CR0_PE), %eax mov %eax, %cr0 /* Jump to 64-bit mode. */ - ljmp $PVH_CS_SEL, $_pa(1f) + pushl $PVH_CS_SEL + leal rva(1f)(%ebp), %eax + pushl %eax + lretl /* 64-bit entry point. */ .code64 1: /* Set base address in stack canary descriptor. */ mov $MSR_GS_BASE,%ecx - mov $_pa(canary), %eax + leal rva(canary)(%ebp), %eax xor %edx, %edx wrmsr + /* Calculate load offset from LOAD_PHYSICAL_ADDR and store in + * phys_base. __pa() needs phys_base set to calculate the the + * hypercall page in xen_pvh_init(). */ + movq %rbp, %rbx + subq $LOAD_PHYSICAL_ADDR, %rbx + movq %rbx, phys_base(%rip) call xen_prepare_pvh + /* Clear phys_base. startup_64/__startup_64 will *add* to its value, + so start from 0. */ + xor %rbx, %rbx + movq %rbx, phys_base(%rip) /* startup_64 expects boot_params in %rsi. */ - mov $_pa(pvh_bootparams), %rsi - mov $_pa(startup_64), %rax + lea rva(pvh_bootparams)(%ebp), %rsi + lea rva(startup_64)(%ebp), %rax ANNOTATE_RETPOLINE_SAFE jmp *%rax
@@ -137,13 +212,14 @@ SYM_CODE_START_LOCAL(pvh_start_xen) ljmp $PVH_CS_SEL, $_pa(startup_32) #endif + SYM_CODE_END(pvh_start_xen) .section ".init.data","aw" .balign 8 SYM_DATA_START_LOCAL(gdt) - .word gdt_end - gdt_start - .long _pa(gdt_start) + .word gdt_end - gdt_start - 1 + .long _pa(gdt_start) /* x86-64 will overwrite if relocated. */ .word 0 SYM_DATA_END(gdt) SYM_DATA_START_LOCAL(gdt_start)
@@ -163,5 +239,89 @@ SYM_DATA_START_LOCAL(early_stack) .fill BOOT_STACK_SIZE, 1, 0 SYM_DATA_END_LABEL(early_stack, SYM_L_LOCAL, early_stack_end) +#ifdef CONFIG_X86_64 +/* + * We are not able to switch in one step to the final KERNEL ADDRESS SPACE + * because we need identity-mapped pages. + */ +#define l4_index(x) (((x) >> 39) & 511) +#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) + +L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4) +L4_START_KERNEL = l4_index(__START_KERNEL_map) +L3_START_KERNEL = pud_index(__START_KERNEL_map) + +#define SYM_DATA_START_PAGE_ALIGNED(name) \ + SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) + +/* Automate the creation of 1 to 1 mapping pmd entries */ +#define PMDS(START, PERM, COUNT) \ + i = 0 ; \ + .rept (COUNT) ; \ + .quad (START) + (i << PMD_SHIFT) + (PERM) ; \ + i = i + 1 ; \ + .endr + +/* + * Xen PVH needs a set of identity mapped and kernel high mapping + * page tables. pvh_start_xen starts running on the identity mapped + * page tables, but xen_prepare_pvh calls into the high mapping. + * These page tables need to be relocatable and are only used until + * startup_64 transitions to init_top_pgt. + */ +SYM_DATA_START_PAGE_ALIGNED(pvh_init_top_pgt) + .quad pvh_level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .org pvh_init_top_pgt + L4_PAGE_OFFSET*8, 0 + .quad pvh_level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .org pvh_init_top_pgt + L4_START_KERNEL*8, 0 + /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ + .quad pvh_level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC +SYM_DATA_END(pvh_init_top_pgt) + +SYM_DATA_START_PAGE_ALIGNED(pvh_level3_ident_pgt) + .quad pvh_level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .fill 511, 8, 0 +SYM_DATA_END(pvh_level3_ident_pgt) +SYM_DATA_START_PAGE_ALIGNED(pvh_level2_ident_pgt) + /* + * Since I easily can, map the first 1G. + * Don't set NX because code runs from these pages. + * + * Note: This sets _PAGE_GLOBAL despite whether + * the CPU supports it or it is enabled. But, + * the CPU should ignore the bit. + */ + PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD) +SYM_DATA_END(pvh_level2_ident_pgt) +SYM_DATA_START_PAGE_ALIGNED(pvh_level3_kernel_pgt) + .fill L3_START_KERNEL,8,0 + /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */ + .quad pvh_level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .quad 0 /* no fixmap */ +SYM_DATA_END(pvh_level3_kernel_pgt) + +SYM_DATA_START_PAGE_ALIGNED(pvh_level2_kernel_pgt) + /* + * Kernel high mapping. + * + * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in + * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled, + * 512 MiB otherwise. + * + * (NOTE: after that starts the module area, see MODULES_VADDR.) + * + * This table is eventually used by the kernel during normal runtime. + * Care must be taken to clear out undesired bits later, like _PAGE_RW + * or _PAGE_GLOBAL in some cases. + */ + PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE) +SYM_DATA_END(pvh_level2_kernel_pgt) + + ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_RELOC, + .long KERNEL_IMAGE_SIZE; + .long LOAD_PHYSICAL_ADDR; + .long CONFIG_PHYSICAL_ALIGN) +#endif + ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, - _ASM_PTR (pvh_start_xen - __START_KERNEL_map)) + .long (pvh_start_xen - __START_KERNEL_map))
include/xen/interface/elfnote.h +17 -1
@@ -185,9 +185,25 @@*/ #define XEN_ELFNOTE_PHYS32_ENTRY 18 +/* +* Physical loading constraints for PVH kernels +* +* Used to place constraints on the guest physical loading addresses and +* alignment for a PVH kernel. +* +* The presence of this note indicates the kernel supports relocating itself. +* +* The note may include up to three 32bit values in the following order: +* - a maximum address for the entire image to be loaded below (default +* 0xffffffff) +* - a minimum address for the start of the image (default 0) +* - a required start alignment (default 0x200000) +*/ +#define XEN_ELFNOTE_PHYS32_RELOC 19 +/* * The number of the highest elfnote defined. */ -#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_ENTRY +#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_RELOC#endif /* __XEN_PUBLIC_ELFNOTE_H__ */
Signature · 1 line
2.44.0