Patches for PCI passthrough with modified E820 (v3).

diff -r 697ac895c11c -r 6c05be60fbc7 tools/libxc/xc_domain.c
@@ -478,37 +478,64 @@ int xc_domain_pin_memory_cacheattr(xc_in
 }
 
 #if defined(__i386__) || defined(__x86_64__)
-#include "xc_e820.h"
+int xc_domain_set_memory_map(xc_interface *xch,
+                               uint32_t domid,
+                               struct e820entry entries[],
+                               uint32_t nr_entries)
+{
+    int rc;
+    struct xen_foreign_memory_map fmap = {
+        .domid = domid,
+        .map = { .nr_entries = nr_entries }
+    };
+    DECLARE_HYPERCALL_BOUNCE(entries, nr_entries * sizeof(struct e820entry),
+                             XC_HYPERCALL_BUFFER_BOUNCE_IN);
+
+    if ( !entries || xc_hypercall_bounce_pre(xch, entries) )
+        return -1;
+
+    set_xen_guest_handle(fmap.map.buffer, entries);
+
+    rc = do_memory_op(xch, XENMEM_set_memory_map, &fmap, sizeof(fmap));
+
+    xc_hypercall_bounce_post(xch, entries);
+
+    return rc;
+}
+int xc_get_machine_memory_map(xc_interface *xch,
+                              struct e820entry entries[],
+                              uint32_t max_entries)
+{
+    int rc;
+    struct xen_memory_map memmap = {
+        .nr_entries = max_entries
+    };
+    DECLARE_HYPERCALL_BOUNCE(entries, sizeof(struct e820entry) * max_entries,
+                             XC_HYPERCALL_BUFFER_BOUNCE_OUT);
+
+    if ( !entries || xc_hypercall_bounce_pre(xch, entries) || max_entries <= 1)
+        return -1;
+
+
+    set_xen_guest_handle(memmap.buffer, entries);
+
+    rc = do_memory_op(xch, XENMEM_machine_memory_map, &memmap, sizeof(memmap));
+
+    xc_hypercall_bounce_post(xch, entries);
+
+    return rc ? rc : memmap.nr_entries;
+}
 int xc_domain_set_memmap_limit(xc_interface *xch,
                                uint32_t domid,
                                unsigned long map_limitkb)
 {
-    int rc;
-    struct xen_foreign_memory_map fmap = {
-        .domid = domid,
-        .map = { .nr_entries = 1 }
-    };
-    DECLARE_HYPERCALL_BUFFER(struct e820entry, e820);
+    struct e820entry e820;
 
-    e820 = xc_hypercall_buffer_alloc(xch, e820, sizeof(*e820));
+    e820.addr = 0;
+    e820.size = (uint64_t)map_limitkb << 10;
+    e820.type = E820_RAM;
 
-    if ( e820 == NULL )
-    {
-        PERROR("Could not allocate memory for xc_domain_set_memmap_limit hypercall");
-        return -1;
-    }
-
-    e820->addr = 0;
-    e820->size = (uint64_t)map_limitkb << 10;
-    e820->type = E820_RAM;
-
-    set_xen_guest_handle(fmap.map.buffer, e820);
-
-    rc = do_memory_op(xch, XENMEM_set_memory_map, &fmap, sizeof(fmap));
-
-    xc_hypercall_buffer_free(xch, e820);
-
-    return rc;
+    return xc_domain_set_memory_map(xch, domid, &e820, 1);
 }
 #else
 int xc_domain_set_memmap_limit(xc_interface *xch,

diff -r 697ac895c11c -r 6c05be60fbc7 tools/libxc/xc_e820.h
@@ -26,6 +26,9 @@
 #define E820_RESERVED     2
 #define E820_ACPI         3
 #define E820_NVS          4
+#define E820_UNUSABLE     5
+
+#define E820MAX           (128)
 
 struct e820entry {
     uint64_t addr;

diff -r 697ac895c11c -r 6c05be60fbc7 tools/libxc/xenctrl.h
@@ -966,6 +966,17 @@ int xc_domain_set_memmap_limit(xc_interf
                                uint32_t domid,
                                unsigned long map_limitkb);
 
+#if defined(__i386__) || defined(__x86_64__)
+#include "xc_e820.h"
+int xc_domain_set_memory_map(xc_interface *xch,
+                               uint32_t domid,
+                               struct e820entry entries[],
+                               uint32_t nr_entries);
+
+int xc_get_machine_memory_map(xc_interface *xch,
+                              struct e820entry entries[],
+                              uint32_t max_entries);
+#endif
 int xc_domain_set_time_offset(xc_interface *xch,
                               uint32_t domid,
                               int32_t time_offset_seconds);

diff -r 6c05be60fbc7 xen/arch/x86/domain.c
@@ -657,6 +657,8 @@ int arch_domain_create(struct domain *d,
         /* 32-bit PV guest by default only if Xen is not 64-bit. */
         d->arch.is_32bit_pv = d->arch.has_32bit_shinfo =
             (CONFIG_PAGING_LEVELS != 4);
+
+        spin_lock_init(&d->arch.pv_domain.e820_lock);
     }
 
     /* initialize default tsc behavior in case tools don't */

@@ -696,6 +698,8 @@ void arch_domain_destroy(struct domain *
 
     if ( is_hvm_domain(d) )
         hvm_domain_destroy(d);
+    else
+        xfree(d->arch.pv_domain.e820);
 
     vmce_destroy_msr(d);
     pci_release_devices(d);

81 lines not on this page · raw message

diff -r 6c05be60fbc7 xen/include/asm-x86/domain.h
@@ -241,7 +241,8 @@ struct pv_domain
     unsigned long pirq_eoi_map_mfn;
 
     /* Pseudophysical e820 map (XENMEM_memory_map).  */
-    struct e820entry e820[3];
+    spinlock_t e820_lock;
+    struct e820entry *e820;
     unsigned int nr_e820;
 };

diff -r a81f4c7b4e39 tools/libxl/libxl_create.c
@@ -516,6 +516,14 @@ static int do_domain_create(libxl__gc *g
     for (i = 0; i < d_config->num_pcidevs; i++)
         libxl__device_pci_add(gc, domid, &d_config->pcidevs[i], 1);
 
+    if (!d_config->c_info.hvm && d_config->b_info.u.pv.machine_e820) {
+        int rc = 0;
+        rc = libxl__e820_alloc(ctx, domid, d_config);
+        if (rc)
+            LIBXL__LOG(ctx, LIBXL__LOG_WARNING,
+                      "Failed while collecting E820 with: %d (errno:%d)\n",
+                      rc, errno);
+    }
     if ( cb && (d_config->c_info.hvm || d_config->b_info.u.pv.bootloader )) {
         if ( (*cb)(ctx, domid, priv) )
             goto error_out;

diff -r a81f4c7b4e39 tools/libxl/libxl_internal.h
@@ -344,4 +344,5 @@ _hidden int libxl__error_set(libxl__gc *
 _hidden int libxl__file_reference_map(libxl_file_reference *f);
 _hidden int libxl__file_reference_unmap(libxl_file_reference *f);
 
+_hidden int libxl__e820_alloc(libxl_ctx *ctx, uint32_t domid, libxl_domain_config *d_config);
 #endif

157 lines not on this page · raw message

diff -r a81f4c7b4e39 tools/libxl/xl_cmdimpl.c
@@ -373,6 +373,7 @@ static void printf_info(int domid,
         printf("\t\t\t(kernel %s)\n", b_info->kernel.path);
         printf("\t\t\t(cmdline %s)\n", b_info->u.pv.cmdline);
         printf("\t\t\t(ramdisk %s)\n", b_info->u.pv.ramdisk.path);
+        printf("\t\t\t(machine_e820 %d)\n", b_info->u.pv.machine_e820);
         printf("\t\t)\n");
     }
     printf("\t)\n");

@@ -984,6 +985,8 @@ skip_vfb:
             if (!libxl_device_pci_parse_bdf(ctx, pcidev, buf))
                 d_config->num_pcidevs++;
         }
+        if (d_config->num_pcidevs && !c_info->hvm)
+          b_info->u.pv.machine_e820 = true;
     }
 
     switch (xlu_cfg_get_list(config, "cpuid", &cpuids, 0, 1)) {

diff -r 6377bb56ffd1 -r cc15803af031 tools/libxl/libxl_pci.c
@@ -1114,22 +1114,98 @@ static int e820_sanitize(libxl_ctx *ctx,
 		map_limitkb, ram_end >> 12, delta_kb, start_kb ,start >> 12,
                 balloon_kb);
 
+    /* This whole code below is to guard against if the Intel IGD is passed into
+     * the guest. If we don't pass in IGD, this whole code can be ignored.
+     *
+     * The reason for this code is that Intel boxes fill their E820 with
+     * E820_RAM amongst E820_RESERVED and we can't just ditch the E820_RAM.
+     * That is b/c any "gaps" in the E820 is considered PCI I/O space by
+     * Linux and it would be utilized by the Intel IGD as I/O space while
+     * in reality it was an RAM region.
+     *
+     * What this means is that we have to walk the E820 and for any region
+     * that is RAM and below 4GB and above ram_end, needs to change its type
+     * to E820_UNUSED. We also need to move some of the E820_RAM regions if
+     * the overlap with ram_end. */
+    for (i = 0; i < nr; i++) {
+	uint64_t end = src[i].addr + src[i].size;
+
+	/* We don't care about E820_UNUSABLE, but we need to
+         * change the type to zero b/c the loop after this
+         * sticks E820_UNUSABLE on the guest's E820 but ignores
+         * the ones with type zero. */
+        if ((src[i].type == E820_UNUSABLE) ||
+	    /* Any region that is within the "RAM region" can
+             * be safely ditched. */
+            (end < ram_end)) {
+		src[i].type = 0;
+                continue;
+        }
+
+        /* Look only at RAM regions. */
+	if (src[i].type != E820_RAM)
+            continue;
+
+        /* We only care about RAM regions below 4GB. */
+        if (src[i].addr >= (1ULL<<32))
+            continue;
+
+	/* E820_RAM overlaps with our RAM region. Move it */
+	if (src[i].addr < ram_end) {
+		uint64_t delta;
+
+		src[i].type = E820_UNUSABLE;
+		delta = ram_end - src[i].addr;
+		/* The end < ram_end should weed this out */
+		if (src[i].size - delta < 0)
+			src[i].type = 0;
+		else {
+			src[i].size -= delta;
+			src[i].addr = ram_end;
+		}
+		if (src[i].addr + src[i].size != end) {
+			/* We messed up somewhere */
+			src[i].type = 0;
+                        LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "Computed E820 wrongly. Continuing on.");
+		}
+	}
+	/* Lastly, convert the RAM to UNSUABLE. Look in the Linux kernel
+           at git commit 2f14ddc3a7146ea4cd5a3d1ecd993f85f2e4f948
+            "xen/setup: Inhibit resource API from using System RAM E820
+           gaps as PCI mem gaps" for full explanation. */
+	if (end > ram_end)
+          src[i].type = E820_UNUSABLE;
+    }
+
     /* Check if there is a region between ram_end and start. */
     if (start > ram_end) {
+        int add_unusable = 1;
+        for (i = 0; i < nr && add_unusable; i++) {
+          if (src[i].type != E820_UNUSABLE)
+            continue;
+          if (ram_end != src[i].addr)
+            continue;
+          if (start != src[i].addr + src[i].size)
+            /* there is one, adjust it */
+            src[i].size = start - src[i].addr;
+
+          add_unusable = 0;
+      }
       /* .. and if not present, add it in. This is to guard against
        the Linux guest assuming that the gap between the end of
        RAM region and the start of the E820_[ACPI,NVS,RESERVED]
        is PCI I/O space. Which it certainly is _not_. */
-      e820[idx].type = E820_UNUSABLE;
-      e820[idx].addr = ram_end;
-      e820[idx].size = start - ram_end;
-      idx++;
+      if (add_unusable) {
+        e820[idx].type = E820_UNUSABLE;
+        e820[idx].addr = ram_end;
+        e820[idx].size = start - ram_end;
+        idx++;
+      }
     } 
-     /* Almost done: copy them over, ignoring the undesireable ones */
-     for (i = 0; i < nr; i++) {
+    /* Almost done: copy them over, ignoring the undesireable ones */
+    for (i = 0; i < nr; i++) {
             if ((src[i].type == E820_RAM) ||
-	        (src[i].type == E820_UNUSABLE) ||
-	         (src[i].type == 0))
+	        (src[i].type == 0))
 		continue;
             e820[idx].type = src[i].type;
             e820[idx].addr = src[i].addr;