Skip to contentSkip to conversation

xen: reserve RMRR to avoid conflicting MMIO/RAM

RFC PATCH Superseded Tiejun Chen 1 participant 7 messages Aug 12, 2014

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v2 cover xen: reserve RMRR to avoid conflicting MMIO/RAM

v2:

* Don't use e820map to define RMRR maps directly to avoid any confusion.
* In patch #3 we introduce construct_rmrr_e820_maps() to check if we can insert RMRR maps and then we will sort all e820 entries.
* Clean patch #4
* In patch #5 we reuse check_mmio_hole() to check if current mmio range is fine to RMRR maps. If not, we just issue error to notify the user since mostly mmio should be configured again.

While we work for supporting RMRR mapping for Windows GFX driver in case shared table,

http://osdir.com/ml/general/2014-07/msg55347.html
http://osdir.com/ml/general/2014-07/msg55348.html

we realize we should reserve RMRR range to avoid any potential MMIO/RAM overlap with our discussion so here these preliminary patches are intended to cover this.

----------------------------------------------------------------
Tiejun Chen (6):
xen:x86: record RMRR mappings
xen:x86: introduce a new hypercall to get RMRR mappings tools:firmware:hvmloader: reserve RMRR mappings in e820 xen:x86: add XENMEM_RMRR_memory_map to expose RMRR tools:libxc: check if mmio BAR is out of RMRR mappings xen:vtd: make USB RMRR mapping safe

 tools/firmware/hvmloader/e820.c     | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/firmware/hvmloader/e820.h     |  6 ++++++
 tools/firmware/hvmloader/util.c     | 15 +++++++++++++++
 tools/firmware/hvmloader/util.h     |  2 ++
 tools/libxc/xc_domain.c             | 25 +++++++++++++++++++++++++
 tools/libxc/xc_hvm_build_x86.c      | 23 +++++++++++++++++++++++
 tools/libxc/xenctrl.h               |  4 ++++
 xen/arch/x86/e820.c                 |  2 ++
 xen/arch/x86/mm.c                   | 31 +++++++++++++++++++++++++++++++
 xen/arch/x86/x86_64/compat/mm.c     |  8 ++++++++
 xen/drivers/passthrough/vtd/dmar.c  | 14 ++++++++++++++
 xen/drivers/passthrough/vtd/iommu.c |  8 --------
 xen/include/asm-x86/e820.h          |  3 +++
 xen/include/public/memory.h         | 10 +++++++++-
 14 files changed, 236 insertions(+), 9 deletions(-)

Thanks
Tiejun

#1 lore raw

RFC PATCH v2 1/6 xen:x86: record RMRR mappings

We will expose RMRR mappings to VM so need to record all RMRR mappings firstly.

---
 xen/arch/x86/e820.c                |  2 ++
 xen/drivers/passthrough/vtd/dmar.c | 14 ++++++++++++++
 xen/include/asm-x86/e820.h         |  3 +++
 3 files changed, 19 insertions(+)
xen/arch/x86/e820.c +2 -0
index 55fe0d6..db44afd 100644
@@ -34,6 +34,8 @@ static bool_t __initdata e820_verbose;
 boolean_param("e820-verbose", e820_verbose);
 
 struct e820map e820;
+/* Used to record RMRR mapping. */
+rmrr_maps_t rmrr_maps;
 
 /*
  * This function checks if the entire range <start,end> is mapped with type.
xen/drivers/passthrough/vtd/dmar.c +14 -0
index 1152c3a..24321e3 100644
@@ -29,6 +29,7 @@
 #include <xen/pci.h>
 #include <xen/pci_regs.h>
 #include <asm/string.h>
+#include <asm/e820.h>
 #include "dmar.h"
 #include "iommu.h"
 #include "extern.h"
@@ -80,6 +81,18 @@ static int __init acpi_register_rmrr_unit(struct acpi_rmrr_unit *rmrr)
     return 0;
 }
 
+/* Record RMRR mapping to ready expose VM. */
+static int __init rmrr_maps_register(struct acpi_rmrr_unit *rmrr)
+{
+    rmrr_maps.map[rmrr_maps.nr_map].addr = rmrr->base_address;
+    rmrr_maps.map[rmrr_maps.nr_map].size = rmrr->end_address -
+                                           rmrr->base_address;
+    rmrr_maps.map[rmrr_maps.nr_map].type = E820_RESERVED;
+    rmrr_maps.nr_map++;
+
+    return 0;
+}
+
 static void __init disable_all_dmar_units(void)
 {
     struct acpi_drhd_unit *drhd, *_drhd;
@@ -675,6 +688,7 @@ acpi_parse_one_rmrr(struct acpi_dmar_header *header)
                         " end_address %"PRIx64"\n",
                         rmrru->base_address, rmrru->end_address);
             acpi_register_rmrr_unit(rmrru);
+            rmrr_maps_register(rmrru);
         }
     }
 
xen/include/asm-x86/e820.h +3 -0
index 71a804c..74262c9 100644
@@ -23,6 +23,8 @@ struct e820map {
     struct e820entry map[E820MAX];
 };
 
+typedef struct e820map rmrr_maps_t;
+
 extern int e820_all_mapped(u64 start, u64 end, unsigned type);
 extern int reserve_e820_ram(struct e820map *e820, uint64_t s, uint64_t e);
 extern int e820_change_range_type(
@@ -32,6 +34,7 @@ extern int e820_add_range(
     struct e820map *, uint64_t s, uint64_t e, uint32_t type);
 extern unsigned long init_e820(const char *, struct e820entry *, int *);
 extern struct e820map e820;
+extern rmrr_maps_t rmrr_maps;
 
 /* These symbols live in the boot trampoline. */
 extern struct e820entry e820map[];
Signature · 1 line
1.9.1
#2 lore raw

RFC PATCH v2 2/6 xen:x86: introduce a new hypercall to get RMRR mappings

In reply to Tiejun Chen

We need this new hypercall to get RMRR mapping for VM.

---
 xen/arch/x86/x86_64/compat/mm.c |  8 ++++++++
 xen/include/public/memory.h     | 10 +++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)
xen/arch/x86/x86_64/compat/mm.c +8 -0
index 69c6195..878c17c 100644
@@ -132,6 +132,14 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         break;
     }
 
+    case XENMEM_RMRR_memory_map:
+    {
+        if ( copy_to_guest(arg, &rmrr_maps, 1) )
+            return -EFAULT;
+
+        return 0;
+    }
+
     case XENMEM_machphys_mapping:
     {
         struct domain *d = current->domain;
xen/include/public/memory.h +9 -1
index 2c57aa0..75bbb36 100644
@@ -523,7 +523,15 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
 
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
-/* Next available subop number is 26 */
+/*
+ * Returns the RMRR memory map as it was when the domain
+ * was started.
+ */
+#define XENMEM_RMRR_memory_map           26
+typedef struct xen_memory_map xen_rmrr_memory_map_t;
+DEFINE_XEN_GUEST_HANDLE(xen_rmrr_memory_map_t);
+
+/* Next available subop number is 27 */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
 
Signature · 1 line
1.9.1
#3 lore raw

RFC PATCH v2 3/6 tools:firmware:hvmloader: reserve RMRR mappings in e820

In reply to Tiejun Chen

We need to reserve all RMRR mappings in e820 to avoid any potential guest memory conflict.

---
 tools/firmware/hvmloader/e820.c | 94 +++++++++++++++++++++++++++++++++++++++++
 tools/firmware/hvmloader/e820.h |  6 +++
 tools/firmware/hvmloader/util.c | 15 +++++++
 tools/firmware/hvmloader/util.h |  2 +
 4 files changed, 117 insertions(+)
tools/firmware/hvmloader/e820.c +94 -0
index 2e05e93..7f54eab 100644
@@ -68,12 +68,89 @@ void dump_e820_table(struct e820entry *e820, unsigned int nr)
     }
 }
 
+static unsigned int construct_rmrr_e820_maps(unsigned int nr,
+                                     struct rmrr_map *e820_rmrr_map,
+                                     struct e820entry *e820)
+{
+    unsigned int i = 0, j = 0, m = 0, sum_nr = 0;
+    uint64_t start, end, rmrr_start, rmrr_end;
+    unsigned int insert = 0, do_insert = 0;
+
+do_real_construct:
+    sum_nr = nr + e820_rmrr_map->nr_map;
+    for ( i = 0; i < e820_rmrr_map->nr_map; i++ )
+    {
+        rmrr_start = e820_rmrr_map->map[i].addr;
+        rmrr_end = e820_rmrr_map->map[i].addr +
+                   e820_rmrr_map->map[i].size + 1;
+
+        for ( j = 0; j < nr; j++ )
+        {
+            end = e820[j].addr + e820[j].size;
+            start = e820[j+1].addr;
+
+            /* Between those existing e820 entries. */
+            if ( (rmrr_start > end) && (rmrr_end < start) )
+            {
+                if (do_insert)
+                {
+                    /* Move to free this entry. */
+                    for ( m = sum_nr - 1; m > j; m-- )
+                    {
+                        e820[m].addr = e820[m-1].addr;
+                        e820[m].size = e820[m-1].size;
+                        e820[m].type = e820[m-1].type;
+                    }
+
+                    /* Then fill RMRR into that entry. */
+                    e820[j+1].addr = rmrr_start;
+                    e820[j+1].size = rmrr_end - rmrr_start;
+                    e820[j+1].type = E820_RESERVED;
+                    nr++;
+                }
+                insert++;
+            }
+            /* Already at the end. */
+            else if ( (rmrr_start > end) && !start )
+            {
+                if (do_insert)
+                {
+                    e820[nr].addr = rmrr_start;
+                    e820[nr].size = rmrr_end - rmrr_start;
+                    e820[nr].type = E820_RESERVED;
+                    nr++;
+                }
+                insert++;
+            }
+        }
+    }
+
+    /* Just return if done. */
+    if (do_insert)
+        return nr;
+
+    /* Fine to construct RMRR mappings into e820. */
+    if ( insert == e820_rmrr_map->nr_map)
+    {
+        do_insert = 1;
+        goto do_real_construct;
+    }
+    /* Overlap. */
+    else
+    {
+        printf("RMRR overlap with thoese existing e820 entries!\n");
+        printf("So we don't construct RMRR mapping in e820!\n");
+    }
+
+    return nr;
+}
 /* Create an E820 table based on memory parameters provided in hvm_info. */
 int build_e820_table(struct e820entry *e820,
                      unsigned int lowmem_reserved_base,
                      unsigned int bios_image_base)
 {
     unsigned int nr = 0;
+    struct rmrr_map *rmrr_maps;
 
     if ( !lowmem_reserved_base )
             lowmem_reserved_base = 0xA0000;
@@ -169,6 +246,23 @@ int build_e820_table(struct e820entry *e820,
         nr++;
     }
 
+    /* We'd better reserve RMRR mapping for each VM to avoid potential
+     * memory conflict.
+     */
+    rmrr_maps = get_rmrr_map_info();
+    if ( rmrr_maps->nr_map )
+    {
+        if ( (nr + rmrr_maps->nr_map) > E820MAX )
+        {
+            printf(" No free space to insert all RMRR mapping entry!!\n");
+            return nr;
+        }
+        else
+        {
+            nr = construct_rmrr_e820_maps(nr, rmrr_maps, e820);
+        }
+    }
+
     return nr;
 }
 
tools/firmware/hvmloader/e820.h +6 -0
index b2ead7f..a61b80b 100644
@@ -15,6 +15,12 @@ struct e820entry {
     uint32_t type;
 } __attribute__((packed));
 
+#define E820MAX 128
+
+struct rmrr_map {
+    unsigned int nr_map;
+    struct e820entry map[E820MAX];
+};
 #endif /* __HVMLOADER_E820_H__ */
 
 /*
tools/firmware/hvmloader/util.c +15 -0
index 80d822f..86c1c99 100644
@@ -766,6 +766,21 @@ struct shared_info *get_shared_info(void)
     return shared_info;
 }
 
+struct rmrr_map *get_rmrr_map_info(void)
+{
+    static int no_rmrr = 1;
+
+    if ( no_rmrr == 0 )
+        return &rmrr_e820map;
+
+    if ( hypercall_memory_op(XENMEM_RMRR_memory_map, &rmrr_e820map) != 0 )
+        BUG();
+
+    no_rmrr = 0;
+
+    return &rmrr_e820map;
+}
+
 uint16_t get_cpu_mhz(void)
 {
     struct shared_info *shared_info = get_shared_info();
tools/firmware/hvmloader/util.h +2 -0
index a70e4aa..5f48a86 100644
@@ -236,6 +236,8 @@ unsigned long create_pir_tables(void);
 void smp_initialise(void);
 
 #include "e820.h"
+struct rmrr_map rmrr_e820map;
+struct rmrr_map *get_rmrr_map_info(void);
 int build_e820_table(struct e820entry *e820,
                      unsigned int lowmem_reserved_base,
                      unsigned int bios_image_base);
Signature · 1 line
1.9.1
#4 lore raw

RFC PATCH v2 4/6 xen:x86: add XENMEM_RMRR_memory_map to expose RMRR

In reply to Tiejun Chen

We should expose RMRR mapping to libxc, then setup_guest() can check if current MMIO is not covered by any RMRR mapping.

---
 xen/arch/x86/mm.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
xen/arch/x86/mm.c +31 -0
index d23cb3f..9bb31a7 100644
@@ -4769,6 +4769,37 @@ long arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         return 0;
     }
 
+    case XENMEM_RMRR_memory_map:
+    {
+        struct xen_memory_map map;
+        XEN_GUEST_HANDLE(e820entry_t) buffer;
+        XEN_GUEST_HANDLE_PARAM(e820entry_t) buffer_param;
+        unsigned int i;
+
+        if ( copy_from_guest(&map, arg, 1) )
+            return -EFAULT;
+        if ( map.nr_entries < rmrr_maps.nr_map + 1 )
+            return -EINVAL;
+
+        buffer_param = guest_handle_cast(map.buffer, e820entry_t);
+        buffer = guest_handle_from_param(buffer_param, e820entry_t);
+        if ( !guest_handle_okay(buffer, map.nr_entries) )
+            return -EFAULT;
+
+        for ( i = 0; i < rmrr_maps.nr_map; ++i )
+        {
+            if ( __copy_to_guest_offset(buffer, i, rmrr_maps.map + i, 1) )
+                return -EFAULT;
+        }
+
+        map.nr_entries = rmrr_maps.nr_map;
+
+        if ( __copy_to_guest(arg, &map, 1) )
+            return -EFAULT;
+
+        return 0;
+    }
+
     case XENMEM_machphys_mapping:
     {
         struct xen_machphys_mapping mapping = {
Signature · 1 line
1.9.1
#5 lore raw

RFC PATCH v2 5/6 tools:libxc: check if mmio BAR is out of RMRR mappings

In reply to Tiejun Chen

We need to avoid allocating MMIO BAR conficting to RMRR range.

---
 tools/libxc/xc_domain.c        | 25 +++++++++++++++++++++++++
 tools/libxc/xc_hvm_build_x86.c | 23 +++++++++++++++++++++++
 tools/libxc/xenctrl.h          |  4 ++++
 3 files changed, 52 insertions(+)
tools/libxc/xc_domain.c +25 -0
index 0230c6c..c1dddb1 100644
@@ -648,6 +648,31 @@ int xc_domain_set_memory_map(xc_interface *xch,
 
     return rc;
 }
+
+int xc_get_rmrr_map(xc_interface *xch,
+                    struct e820entry entries[],
+                    uint32_t max_entries)
+{
+    int rc;
+    struct xen_memory_map memmap = {
+        .nr_entries = max_entries
+    };
+    DECLARE_HYPERCALL_BOUNCE(entries, sizeof(struct e820entry) * max_entries,
+                             XC_HYPERCALL_BUFFER_BOUNCE_OUT);
+
+    if ( !entries || xc_hypercall_bounce_pre(xch, entries) || max_entries <= 1)
+        return -1;
+
+
+    set_xen_guest_handle(memmap.buffer, entries);
+
+    rc = do_memory_op(xch, XENMEM_RMRR_memory_map, &memmap, sizeof(memmap));
+
+    xc_hypercall_bounce_post(xch, entries);
+
+    return rc ? rc : memmap.nr_entries;
+}
+
 int xc_get_machine_memory_map(xc_interface *xch,
                               struct e820entry entries[],
                               uint32_t max_entries)
tools/libxc/xc_hvm_build_x86.c +23 -0
index c81a25b..2196cdb 100644
@@ -262,6 +262,8 @@ static int setup_guest(xc_interface *xch,
     int claim_enabled = args->claim_enabled;
     xen_pfn_t special_array[NR_SPECIAL_PAGES];
     xen_pfn_t ioreq_server_array[NR_IOREQ_SERVER_PAGES];
+    struct e820entry map[E820MAX];
+    uint64_t rmrr_start = 0, rmrr_end = 0;
 
     if ( nr_pages > target_pages )
         pod_mode = XENMEMF_populate_on_demand;
@@ -300,6 +302,27 @@ static int setup_guest(xc_interface *xch,
         goto error_out;
     }
 
+    /* We should check if mmio range is out of RMRR mapping. */
+    rc = xc_get_rmrr_map(xch, map, E820MAX);
+    if (rc < 0)
+    {
+        PERROR("Could not get RMRR info on domain");
+    }
+    else if ( rc )
+    {
+        for ( i = 0; i < rc; i++ )
+        {
+            rmrr_start = map[i].addr;
+            rmrr_end = map[i].addr + map[i].size + 1;
+            if ( check_mmio_hole(rmrr_start, map[i].size + 1, mmio_start, mmio_size) )
+            {
+                PERROR("MMIO: [%lx]<->[%lx] overlap RMRR [%lx]<->[%lx]\n",
+                       mmio_start, (mmio_start + mmio_size), rmrr_start, rmrr_end);
+                goto error_out;
+            }
+        }
+    }
+
     for ( i = 0; i < nr_pages; i++ )
         page_array[i] = i;
     for ( i = mmio_start >> PAGE_SHIFT; i < nr_pages; i++ )
tools/libxc/xenctrl.h +4 -0
index 5beb846..14ef42f 100644
@@ -1270,6 +1270,10 @@ int xc_domain_set_memory_map(xc_interface *xch,
 int xc_get_machine_memory_map(xc_interface *xch,
                               struct e820entry entries[],
                               uint32_t max_entries);
+
+int xc_get_rmrr_map(xc_interface *xch,
+                    struct e820entry entries[],
+                    uint32_t max_entries);
 #endif
 int xc_domain_set_time_offset(xc_interface *xch,
                               uint32_t domid,
Signature · 1 line
1.9.1
#6 lore raw

RFC PATCH v2 6/6 xen:vtd: make USB RMRR mapping safe

In reply to Tiejun Chen

We already reserve all RMRR mappings so USB should be safe with RMRR.

---
 xen/drivers/passthrough/vtd/iommu.c | 8 --------
 1 file changed, 8 deletions(-)
xen/drivers/passthrough/vtd/iommu.c +0 -8
index 042b882..319ae05 100644
@@ -2273,16 +2273,8 @@ static int intel_iommu_assign_device(
     if ( ret )
         goto done;
 
-    /* FIXME: Because USB RMRR conflicts with guest bios region,
-     * ignore USB RMRR temporarily.
-     */
     seg = pdev->seg;
     bus = pdev->bus;
-    if ( is_usb_device(seg, bus, pdev->devfn) )
-    {
-        ret = 0;
-        goto done;
-    }
 
     /* Setup rmrr identity mapping */
     for_each_rmrr_device( rmrr, bdf, i )
Signature · 1 line
1.9.1