xen: reserve RMRR to avoid conflicting MMIO/RAM
v2 cover xen: reserve RMRR to avoid conflicting MMIO/RAM
v2:
* Don't use e820map to define RMRR maps directly to avoid any confusion.
* In patch #3 we introduce construct_rmrr_e820_maps() to check if we can insert RMRR maps and then we will sort all e820 entries.
* Clean patch #4
* In patch #5 we reuse check_mmio_hole() to check if current mmio range is fine to RMRR maps. If not, we just issue error to notify the user since mostly mmio should be configured again.
While we work for supporting RMRR mapping for Windows GFX driver in case shared table,
http://osdir.com/ml/general/2014-07/msg55347.html
http://osdir.com/ml/general/2014-07/msg55348.html
we realize we should reserve RMRR range to avoid any potential MMIO/RAM overlap with our discussion so here these preliminary patches are intended to cover this.
----------------------------------------------------------------
Tiejun Chen (6):
xen:x86: record RMRR mappings
xen:x86: introduce a new hypercall to get RMRR mappings tools:firmware:hvmloader: reserve RMRR mappings in e820 xen:x86: add XENMEM_RMRR_memory_map to expose RMRR tools:libxc: check if mmio BAR is out of RMRR mappings xen:vtd: make USB RMRR mapping safe
tools/firmware/hvmloader/e820.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/firmware/hvmloader/e820.h | 6 ++++++ tools/firmware/hvmloader/util.c | 15 +++++++++++++++ tools/firmware/hvmloader/util.h | 2 ++ tools/libxc/xc_domain.c | 25 +++++++++++++++++++++++++ tools/libxc/xc_hvm_build_x86.c | 23 +++++++++++++++++++++++ tools/libxc/xenctrl.h | 4 ++++ xen/arch/x86/e820.c | 2 ++ xen/arch/x86/mm.c | 31 +++++++++++++++++++++++++++++++ xen/arch/x86/x86_64/compat/mm.c | 8 ++++++++ xen/drivers/passthrough/vtd/dmar.c | 14 ++++++++++++++ xen/drivers/passthrough/vtd/iommu.c | 8 -------- xen/include/asm-x86/e820.h | 3 +++ xen/include/public/memory.h | 10 +++++++++- 14 files changed, 236 insertions(+), 9 deletions(-)
Thanks
Tiejun
RFC PATCH v2 1/6 xen:x86: record RMRR mappings
We will expose RMRR mappings to VM so need to record all RMRR mappings firstly.
--- xen/arch/x86/e820.c | 2 ++ xen/drivers/passthrough/vtd/dmar.c | 14 ++++++++++++++ xen/include/asm-x86/e820.h | 3 +++ 3 files changed, 19 insertions(+)
xen/arch/x86/e820.c +2 -0
@@ -34,6 +34,8 @@ static bool_t __initdata e820_verbose;boolean_param ("e820-verbose" , e820_verbose);struct e820map e820; +/* Used to record RMRR mapping. */ +rmrr_maps_t rmrr_maps;/* * This function checks if the entire range <start,end> is mapped with type.
xen/drivers/passthrough/vtd/dmar.c +14 -0
@@ -29,6 +29,7 @@#include <xen/pci.h> #include <xen/pci_regs.h> #include <asm/string.h> +#include <asm/e820.h> #include "dmar.h" #include "iommu.h" #include "extern.h"
@@ -80,6 +81,18 @@ static int __init acpi_register_rmrr_unit(struct acpi_rmrr_unit *rmrr)return 0 ; } +/* Record RMRR mapping to ready expose VM. */ +static int __init rmrr_maps_register (struct acpi_rmrr_unit *rmrr) +{ + rmrr_maps.map[rmrr_maps.nr_map].addr = rmrr->base_address; + rmrr_maps.map[rmrr_maps.nr_map].size = rmrr->end_address - + rmrr->base_address; + rmrr_maps.map[rmrr_maps.nr_map].type =E820_RESERVED ; + rmrr_maps.nr_map++; + +return 0 ; +} +static void __init disable_all_dmar_units (void ) {struct acpi_drhd_unit *drhd, *_drhd;
@@ -675,6 +688,7 @@ acpi_parse_one_rmrr(struct acpi_dmar_header *header)" end_address %" PRIx64"\n" , rmrru->base_address, rmrru->end_address);acpi_register_rmrr_unit (rmrru); +rmrr_maps_register (rmrru); } }
xen/include/asm-x86/e820.h +3 -0
@@ -23,6 +23,8 @@ struct e820map {struct e820entry map[E820MAX ]; }; +typedef struct e820map rmrr_maps_t ; +extern int e820_all_mapped (u64 start,u64 end,unsigned type);extern int reserve_e820_ram (struct e820map *e820,uint64_t s,uint64_t e); extern int e820_change_range_type(
@@ -32,6 +34,7 @@ extern int e820_add_range( struct e820map *, uint64_t s, uint64_t e, uint32_t type);extern unsigned long init_e820 (const char *,struct e820entry *,int *);extern struct e820map e820; +extern rmrr_maps_t rmrr_maps;/* These symbols live in the boot trampoline. */ extern struct e820entry e820map[];
Signature · 1 line
1.9.1
RFC PATCH v2 2/6 xen:x86: introduce a new hypercall to get RMRR mappings
In reply to Tiejun ChenWe need this new hypercall to get RMRR mapping for VM.
--- xen/arch/x86/x86_64/compat/mm.c | 8 ++++++++ xen/include/public/memory.h | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-)
xen/arch/x86/x86_64/compat/mm.c +8 -0
@@ -132,6 +132,14 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)break ; } +case XENMEM_RMRR_memory_map: + { +if (copy_to_guest (arg, &rmrr_maps,1 ) ) +return -EFAULT ; + +return 0 ; + } +case XENMEM_machphys_mapping: {struct domain *d = current->domain;
xen/include/public/memory.h +9 -1
@@ -523,7 +523,15 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ -/* Next available subop number is 26 */ +/* +* Returns the RMRR memory map as it was when the domain +* was started. +*/ +#define XENMEM_RMRR_memory_map 26 +typedef struct xen_memory_map xen_rmrr_memory_map_t ; +DEFINE_XEN_GUEST_HANDLE (xen_rmrr_memory_map_t); + +/* Next available subop number is 27 */ #endif /* __XEN_PUBLIC_MEMORY_H__ */
Signature · 1 line
1.9.1
RFC PATCH v2 3/6 tools:firmware:hvmloader: reserve RMRR mappings in e820
In reply to Tiejun ChenWe need to reserve all RMRR mappings in e820 to avoid any potential guest memory conflict.
--- tools/firmware/hvmloader/e820.c | 94 +++++++++++++++++++++++++++++++++++++++++ tools/firmware/hvmloader/e820.h | 6 +++ tools/firmware/hvmloader/util.c | 15 +++++++ tools/firmware/hvmloader/util.h | 2 + 4 files changed, 117 insertions(+)
tools/firmware/hvmloader/e820.c +94 -0
@@ -68,12 +68,89 @@ void dump_e820_table(struct e820entry *e820, unsigned int nr) } } +static unsigned int construct_rmrr_e820_maps (unsigned int nr, +struct rmrr_map *e820_rmrr_map, +struct e820entry *e820) +{ +unsigned int i =0 , j =0 , m =0 , sum_nr =0 ; +uint64_t start, end, rmrr_start, rmrr_end; +unsigned int insert =0 , do_insert =0 ; + +do_real_construct: + sum_nr = nr + e820_rmrr_map->nr_map; +for ( i =0 ; i < e820_rmrr_map->nr_map; i++ ) + { + rmrr_start = e820_rmrr_map->map[i].addr; + rmrr_end = e820_rmrr_map->map[i].addr + + e820_rmrr_map->map[i].size +1 ; + +for ( j =0 ; j < nr; j++ ) + { + end = e820[j].addr + e820[j].size; + start = e820[j+1 ].addr; + +/* Between those existing e820 entries. */ +if ( (rmrr_start > end) && (rmrr_end < start) ) + { +if (do_insert) + { +/* Move to free this entry. */ +for ( m = sum_nr -1 ; m > j; m-- ) + { + e820[m].addr = e820[m-1 ].addr; + e820[m].size = e820[m-1 ].size; + e820[m].type = e820[m-1 ].type; + } + +/* Then fill RMRR into that entry. */ + e820[j+1 ].addr = rmrr_start; + e820[j+1 ].size = rmrr_end - rmrr_start; + e820[j+1 ].type =E820_RESERVED ; + nr++; + } + insert++; + } +/* Already at the end. */ +else if ( (rmrr_start > end) && !start ) + { +if (do_insert) + { + e820[nr].addr = rmrr_start; + e820[nr].size = rmrr_end - rmrr_start; + e820[nr].type =E820_RESERVED ; + nr++; + } + insert++; + } + } + } + +/* Just return if done. */ +if (do_insert) +return nr; + +/* Fine to construct RMRR mappings into e820. */ +if ( insert == e820_rmrr_map->nr_map) + { + do_insert =1 ; + goto do_real_construct; + } +/* Overlap. */ +else + { +printf ("RMRR overlap with thoese existing e820 entries!\n" ); +printf ("So we don't construct RMRR mapping in e820!\n" ); + } + +return nr; +}/* Create an E820 table based on memory parameters provided in hvm_info. */ int build_e820_table (struct e820entry *e820,unsigned int lowmem_reserved_base,unsigned int bios_image_base) {unsigned int nr =0 ; +struct rmrr_map *rmrr_maps;if ( !lowmem_reserved_base ) lowmem_reserved_base =0xA0000 ;
@@ -169,6 +246,23 @@ int build_e820_table(struct e820entry *e820, nr++; } +/* We'd better reserve RMRR mapping for each VM to avoid potential +* memory conflict. +*/ + rmrr_maps =get_rmrr_map_info (); +if ( rmrr_maps->nr_map ) + { +if ( (nr + rmrr_maps->nr_map) >E820MAX ) + { +printf (" No free space to insert all RMRR mapping entry!!\n" ); +return nr; + } +else + { + nr =construct_rmrr_e820_maps (nr, rmrr_maps, e820); + } + } +return nr; }
tools/firmware/hvmloader/e820.h +6 -0
@@ -15,6 +15,12 @@ struct e820entry {uint32_t type; } __attribute__((packed)); +#define E820MAX 128 + +struct rmrr_map { +unsigned int nr_map; +struct e820entry map[E820MAX ]; +};#endif /* __HVMLOADER_E820_H__ */ /*
tools/firmware/hvmloader/util.c +15 -0
@@ -766,6 +766,21 @@ struct shared_info *get_shared_info(void)return shared_info; } +struct rmrr_map *get_rmrr_map_info (void ) +{ +static int no_rmrr =1 ; + +if ( no_rmrr ==0 ) +return &rmrr_e820map; + +if (hypercall_memory_op (XENMEM_RMRR_memory_map, &rmrr_e820map) !=0 ) +BUG (); + + no_rmrr =0 ; + +return &rmrr_e820map; +} +uint16_t get_cpu_mhz (void ) {struct shared_info *shared_info =get_shared_info ();
tools/firmware/hvmloader/util.h +2 -0
@@ -236,6 +236,8 @@ unsigned long create_pir_tables(void);void smp_initialise (void );#include "e820.h" +struct rmrr_map rmrr_e820map; +struct rmrr_map *get_rmrr_map_info (void );int build_e820_table (struct e820entry *e820,unsigned int lowmem_reserved_base,unsigned int bios_image_base);
Signature · 1 line
1.9.1
RFC PATCH v2 4/6 xen:x86: add XENMEM_RMRR_memory_map to expose RMRR
In reply to Tiejun ChenWe should expose RMRR mapping to libxc, then setup_guest() can check if current MMIO is not covered by any RMRR mapping.
--- xen/arch/x86/mm.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
xen/arch/x86/mm.c +31 -0
@@ -4769,6 +4769,37 @@ long arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)return 0 ; } +case XENMEM_RMRR_memory_map: + { +struct xen_memory_map map; +XEN_GUEST_HANDLE (e820entry_t) buffer; +XEN_GUEST_HANDLE_PARAM (e820entry_t) buffer_param; +unsigned int i; + +if (copy_from_guest (&map, arg,1 ) ) +return -EFAULT ; +if ( map.nr_entries < rmrr_maps.nr_map +1 ) +return -EINVAL ; + + buffer_param =guest_handle_cast (map.buffer, e820entry_t); + buffer =guest_handle_from_param (buffer_param, e820entry_t); +if ( !guest_handle_okay (buffer, map.nr_entries) ) +return -EFAULT ; + +for ( i =0 ; i < rmrr_maps.nr_map; ++i ) + { +if (__copy_to_guest_offset (buffer, i, rmrr_maps.map + i,1 ) ) +return -EFAULT ; + } + + map.nr_entries = rmrr_maps.nr_map; + +if (__copy_to_guest (arg, &map,1 ) ) +return -EFAULT ; + +return 0 ; + } +case XENMEM_machphys_mapping: {struct xen_machphys_mapping mapping = {
Signature · 1 line
1.9.1
RFC PATCH v2 5/6 tools:libxc: check if mmio BAR is out of RMRR mappings
In reply to Tiejun ChenWe need to avoid allocating MMIO BAR conficting to RMRR range.
--- tools/libxc/xc_domain.c | 25 +++++++++++++++++++++++++ tools/libxc/xc_hvm_build_x86.c | 23 +++++++++++++++++++++++ tools/libxc/xenctrl.h | 4 ++++ 3 files changed, 52 insertions(+)
tools/libxc/xc_domain.c +25 -0
@@ -648,6 +648,31 @@ int xc_domain_set_memory_map(xc_interface *xch,return rc; } + +int xc_get_rmrr_map (xc_interface *xch, +struct e820entry entries[], +uint32_t max_entries) +{ +int rc; +struct xen_memory_map memmap = { + .nr_entries = max_entries + }; +DECLARE_HYPERCALL_BOUNCE (entries,sizeof (struct e820entry ) * max_entries, +XC_HYPERCALL_BUFFER_BOUNCE_OUT ); + +if ( !entries ||xc_hypercall_bounce_pre (xch, entries) || max_entries <=1 ) +return -1 ; + + +set_xen_guest_handle (memmap.buffer, entries); + + rc =do_memory_op (xch, XENMEM_RMRR_memory_map, &memmap,sizeof (memmap)); + +xc_hypercall_bounce_post (xch, entries); + +return rc ? rc : memmap.nr_entries; +} +int xc_get_machine_memory_map (xc_interface *xch,struct e820entry entries[],uint32_t max_entries)
tools/libxc/xc_hvm_build_x86.c +23 -0
@@ -262,6 +262,8 @@ static int setup_guest(xc_interface *xch,int claim_enabled = args->claim_enabled;xen_pfn_t special_array[NR_SPECIAL_PAGES ];xen_pfn_t ioreq_server_array[NR_IOREQ_SERVER_PAGES ]; +struct e820entry map[E820MAX ]; +uint64_t rmrr_start =0 , rmrr_end =0 ;if ( nr_pages > target_pages ) pod_mode = XENMEMF_populate_on_demand;
@@ -300,6 +302,27 @@ static int setup_guest(xc_interface *xch, goto error_out; } +/* We should check if mmio range is out of RMRR mapping. */ + rc =xc_get_rmrr_map (xch, map,E820MAX ); +if (rc <0 ) + { +PERROR ("Could not get RMRR info on domain" ); + } +else if ( rc ) + { +for ( i =0 ; i < rc; i++ ) + { + rmrr_start = map[i].addr; + rmrr_end = map[i].addr + map[i].size +1 ; +if (check_mmio_hole (rmrr_start, map[i].size +1 , mmio_start, mmio_size) ) + { +PERROR ("MMIO: [%lx]<->[%lx] overlap RMRR [%lx]<->[%lx]\n" , + mmio_start, (mmio_start + mmio_size), rmrr_start, rmrr_end); + goto error_out; + } + } + } +for ( i =0 ; i < nr_pages; i++ ) page_array[i] = i;for ( i = mmio_start >>PAGE_SHIFT ; i < nr_pages; i++ )
tools/libxc/xenctrl.h +4 -0
@@ -1270,6 +1270,10 @@ int xc_domain_set_memory_map(xc_interface *xch,int xc_get_machine_memory_map (xc_interface *xch,struct e820entry entries[],uint32_t max_entries); + +int xc_get_rmrr_map (xc_interface *xch, +struct e820entry entries[], +uint32_t max_entries);#endif int xc_domain_set_time_offset(xc_interface *xch, uint32_t domid,
Signature · 1 line
1.9.1
RFC PATCH v2 6/6 xen:vtd: make USB RMRR mapping safe
In reply to Tiejun ChenWe already reserve all RMRR mappings so USB should be safe with RMRR.
--- xen/drivers/passthrough/vtd/iommu.c | 8 -------- 1 file changed, 8 deletions(-)
xen/drivers/passthrough/vtd/iommu.c +0 -8
@@ -2273,16 +2273,8 @@ static int intel_iommu_assign_device(if ( ret ) goto done; -/* FIXME: Because USB RMRR conflicts with guest bios region, -* ignore USB RMRR temporarily. -*/ seg = pdev->seg; bus = pdev->bus; -if (is_usb_device (seg, bus, pdev->devfn) ) - { - ret =0 ; - goto done; - }/* Setup rmrr identity mapping */ for_each_rmrr_device ( rmrr, bdf, i )
Signature · 1 line
1.9.1