Skip to contentSkip to conversation

pc: inform SeaBIOS where 64-bit PCI hole begins

PATCH In review Igor Mammedov 4 participants 12 messages Nov 12, 2013 – Nov 15, 2013

lore
Share & tools
lore
#0 lore raw

v3 cover pc: inform SeaBIOS where 64-bit PCI hole begins

* simplify PCI address space mapping into system address space, replacing code duplication in piix/q53 PCs with a helper function

* add fw_cfg 'etc/reserved-memory-end' to allow QEMU reserve additional address space before 64-bit PCI hole. Which will be need for reserving memory hotplug region in highmem.

v3:
* rebased on top of current master (1.7-rc0), wich includes dependency "memory: Change MemoryRegion priorities from unsigned to signed"
* s/pcimem64-minimum-address/reserved-memory-end/ per Michael suggestion

v2:
* use negative priority to map PCI address space under RAM memory regions which allows simplify code by removing pci_hole & pci_hole64 memory region aliases

Igor Mammedov (1):
pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Michael S. Tsirkin (1):
pc: map PCI address space as catchall region for not mapped addresses

 hw/i386/pc.c              | 28 ++++++++++++++++------------
 hw/i386/pc_piix.c         |  2 --
 hw/pci-host/piix.c        | 27 +++++----------------------
 hw/pci-host/q35.c         | 28 ++++++----------------------
 include/hw/i386/pc.h      | 15 +++------------
 include/hw/pci-host/q35.h |  2 --
 6 files changed, 30 insertions(+), 72 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH v3 1/2 pc: map PCI address space as catchall region for not mapped addresses

From: "Michael S. Tsirkin" <[email protected]>

With a help of negative memory region priority PCI address space is mapped underneath RAM regions effectively catching every access to addresses not mapped by any other region.
It simplifies PCI address space mapping into system address space.

---
 hw/i386/pc.c              | 20 ++++++--------------
 hw/i386/pc_piix.c         |  2 --
 hw/pci-host/piix.c        | 26 ++++----------------------
 hw/pci-host/q35.c         | 27 +++++----------------------
 include/hw/i386/pc.h      | 14 ++------------
 include/hw/pci-host/q35.h |  2 --
 6 files changed, 17 insertions(+), 74 deletions(-)
hw/i386/pc.c +6 -14
index 12c436e..6c82ada 100644
@@ -1093,21 +1093,13 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
     return guest_info;
 }
 
-void pc_init_pci64_hole(PcPciInfo *pci_info, uint64_t pci_hole64_start,
-                        uint64_t pci_hole64_size)
+/* setup pci memory address space mapping into system address space */
+void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
+                            MemoryRegion *pci_address_space)
 {
-    if ((sizeof(hwaddr) == 4) || (!pci_hole64_size)) {
-        return;
-    }
-    /*
-     * BIOS does not set MTRR entries for the 64 bit window, so no need to
-     * align address to power of two.  Align address at 1G, this makes sure
-     * it can be exactly covered with a PAT entry even when using huge
-     * pages.
-     */
-    pci_info->w64.begin = ROUND_UP(pci_hole64_start, 0x1ULL << 30);
-    pci_info->w64.end = pci_info->w64.begin + pci_hole64_size;
-    assert(pci_info->w64.begin <= pci_info->w64.end);
+    /* Set to lower priority than RAM */
+    memory_region_add_subregion_overlap(system_memory, 0x0,
+                                        pci_address_space, -1);
 }
 
 void pc_acpi_init(const char *default_dsdt)
hw/i386/pc_piix.c +0 -2
index 4fdb7b6..23e4857 100644
@@ -150,8 +150,6 @@ static void pc_init1(QEMUMachineInitArgs *args,
     if (pci_enabled) {
         pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
                               system_memory, system_io, args->ram_size,
-                              below_4g_mem_size,
-                              0x100000000ULL - below_4g_mem_size,
                               above_4g_mem_size,
                               pci_memory, ram_memory);
     } else {
hw/pci-host/piix.c +4 -22
index bad3953..5d4e290 100644
@@ -102,8 +102,6 @@ struct PCII440FXState {
     MemoryRegion *system_memory;
     MemoryRegion *pci_address_space;
     MemoryRegion *ram_memory;
-    MemoryRegion pci_hole;
-    MemoryRegion pci_hole_64bit;
     PAMMemoryRegion pam_regions[13];
     MemoryRegion smram_region;
     uint8_t smm_enabled;
@@ -312,8 +310,6 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    hwaddr pci_hole_start,
-                    hwaddr pci_hole_size,
                     ram_addr_t above_4g_mem_size,
                     MemoryRegion *pci_address_space,
                     MemoryRegion *ram_memory)
@@ -326,7 +322,6 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     PCII440FXState *f;
     unsigned i;
     I440FXState *i440fx;
-    uint64_t pci_hole64_size;
 
     dev = qdev_create(NULL, TYPE_I440FX_PCI_HOST_BRIDGE);
     s = PCI_HOST_BRIDGE(dev);
@@ -354,23 +349,10 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
         i440fx->pci_info.w32.begin = 0xe0000000;
     }
 
-    memory_region_init_alias(&f->pci_hole, OBJECT(d), "pci-hole", f->pci_address_space,
-                             pci_hole_start, pci_hole_size);
-    memory_region_add_subregion(f->system_memory, pci_hole_start, &f->pci_hole);
-
-    pci_hole64_size = pci_host_get_hole64_size(i440fx->pci_hole64_size);
-
-    pc_init_pci64_hole(&i440fx->pci_info, 0x100000000ULL + above_4g_mem_size,
-                       pci_hole64_size);
-    memory_region_init_alias(&f->pci_hole_64bit, OBJECT(d), "pci-hole64",
-                             f->pci_address_space,
-                             i440fx->pci_info.w64.begin,
-                             pci_hole64_size);
-    if (pci_hole64_size) {
-        memory_region_add_subregion(f->system_memory,
-                                    i440fx->pci_info.w64.begin,
-                                    &f->pci_hole_64bit);
-    }
+    /* setup pci memory mapping */
+    pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
+                           f->pci_address_space);
+
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
     memory_region_add_subregion_overlap(f->system_memory, 0xa0000,
hw/pci-host/q35.c +5 -22
index b8feed1..d1792de 100644
@@ -350,28 +350,11 @@ static int mch_init(PCIDevice *d)
 {
     int i;
     MCHPCIState *mch = MCH_PCI_DEVICE(d);
-    uint64_t pci_hole64_size;
-
-    /* setup pci memory regions */
-    memory_region_init_alias(&mch->pci_hole, OBJECT(mch), "pci-hole",
-                             mch->pci_address_space,
-                             mch->below_4g_mem_size,
-                             0x100000000ULL - mch->below_4g_mem_size);
-    memory_region_add_subregion(mch->system_memory, mch->below_4g_mem_size,
-                                &mch->pci_hole);
-
-    pci_hole64_size = pci_host_get_hole64_size(mch->pci_hole64_size);
-    pc_init_pci64_hole(&mch->pci_info, 0x100000000ULL + mch->above_4g_mem_size,
-                       pci_hole64_size);
-    memory_region_init_alias(&mch->pci_hole_64bit, OBJECT(mch), "pci-hole64",
-                             mch->pci_address_space,
-                             mch->pci_info.w64.begin,
-                             pci_hole64_size);
-    if (pci_hole64_size) {
-        memory_region_add_subregion(mch->system_memory,
-                                    mch->pci_info.w64.begin,
-                                    &mch->pci_hole_64bit);
-    }
+
+    /* setup pci memory mapping */
+    pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
+                           mch->pci_address_space);
+
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
     memory_region_init_alias(&mch->smram_region, OBJECT(mch), "smram-region",
include/hw/i386/pc.h +2 -12
index 03cc0ba..8b3be3c 100644
@@ -128,17 +128,9 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 #define PCI_HOST_PROP_PCI_HOLE64_SIZE  "pci-hole64-size"
 #define DEFAULT_PCI_HOLE64_SIZE (~0x0ULL)
 
-static inline uint64_t pci_host_get_hole64_size(uint64_t pci_hole64_size)
-{
-    if (pci_hole64_size == DEFAULT_PCI_HOLE64_SIZE) {
-        return 1ULL << 62;
-    } else {
-        return pci_hole64_size;
-    }
-}
 
-void pc_init_pci64_hole(PcPciInfo *pci_info, uint64_t pci_hole64_start,
-                        uint64_t pci_hole64_size);
+void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
+                            MemoryRegion *pci_address_space);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
@@ -187,8 +179,6 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    hwaddr pci_hole_start,
-                    hwaddr pci_hole_size,
                     ram_addr_t above_4g_mem_size,
                     MemoryRegion *pci_memory,
                     MemoryRegion *ram_memory);
include/hw/pci-host/q35.h +0 -2
index aee91aa..28a623c 100644
@@ -53,8 +53,6 @@ typedef struct MCHPCIState {
     MemoryRegion *address_space_io;
     PAMMemoryRegion pam_regions[13];
     MemoryRegion smram_region;
-    MemoryRegion pci_hole;
-    MemoryRegion pci_hole_64bit;
     PcPciInfo pci_info;
     uint8_t smm_enabled;
     ram_addr_t below_4g_mem_size;
Signature · 1 line
1.8.3.1
Tested-by on v3 1/2
#2 lore raw

Re: [Qemu-devel] [PATCH 1/2] pc: map PCI address space as catchall region for not mapped addresses

Quoted 9 lines · Igor Mammedov

From: "Michael S. Tsirkin" <[email protected]>

With a help of negative memory region priority PCI address space is mapped underneath RAM regions effectively catching every access to addresses not mapped by any other region.
It simplifies PCI address space mapping into system address space.

Quoted 8 lines
---
 hw/i386/pc.c              | 20 ++++++--------------
 hw/i386/pc_piix.c         |  2 --
 hw/pci-host/piix.c        | 26 ++++----------------------
 hw/pci-host/q35.c         | 27 +++++----------------------
 include/hw/i386/pc.h      | 14 ++------------
 include/hw/pci-host/q35.h |  2 --
 6 files changed, 17 insertions(+), 74 deletions(-)

Tested with OVMF running from flash (and storing nvvars there).

Also tested with my pending series "i440fx-test: check firmware visibility" (reviews welcome :)).

GTest: result: OK
GTest: run: /i440fx/firmware/bios
(MSG: qemu cmdline: -S -display none -bios /tmp/fw_blob_UAGE6W)
GTest: result: OK
GTest: run: /i440fx/firmware/pflash
(MSG: qemu cmdline: -S -display none -pflash /tmp/fw_blob_DHXN6W)
GTest: result: OK

Thanks!
Laszlo

#3 lore raw

PATCH v3 2/2 pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

In reply to Igor Mammedov

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

---
 hw/i386/pc.c         | 14 +++++++++++++-
 hw/pci-host/piix.c   |  3 ++-
 hw/pci-host/q35.c    |  3 ++-
 include/hw/i386/pc.h |  3 ++-
 4 files changed, 19 insertions(+), 4 deletions(-)
hw/i386/pc.c +13 -1
index 6c82ada..b504047 100644
@@ -1095,11 +1095,23 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 /* setup pci memory address space mapping into system address space */
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space)
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end)
 {
+    uint64_t *val;
+    FWCfgState *fw_cfg = fw_cfg_find();
+
     /* Set to lower priority than RAM */
     memory_region_add_subregion_overlap(system_memory, 0x0,
                                         pci_address_space, -1);
+    g_assert(fw_cfg);
+    /*
+     *  Align address at 1G, this makes sure it can be exactly covered
+     *  with a PAT entry even when using huge pages.
+     */
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(ROUND_UP(reserved_memory_end, 0x1ULL << 30));
+    fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
 }
 
 void pc_acpi_init(const char *default_dsdt)
hw/pci-host/piix.c +2 -1
index 5d4e290..16205e7 100644
@@ -351,7 +351,8 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
-                           f->pci_address_space);
+                           f->pci_address_space,
+                           0x100000000ULL + above_4g_mem_size);
 
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
hw/pci-host/q35.c +2 -1
index d1792de..1293353 100644
@@ -353,7 +353,8 @@ static int mch_init(PCIDevice *d)
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
-                           mch->pci_address_space);
+                           mch->pci_address_space,
+                           0x100000000ULL + mch->above_4g_mem_size);
 
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
include/hw/i386/pc.h +2 -1
index 8b3be3c..2663046 100644
@@ -130,7 +130,8 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space);
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
Signature · 1 line
1.8.3.1
#4 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Does this make sense?

Also, would it make sense to use the e820 interface that Gerd has added, instead of a new fw_cfg file?

Thanks,

Paolo

Quoted 80 lines
---
 hw/i386/pc.c         | 14 +++++++++++++-
 hw/pci-host/piix.c   |  3 ++-
 hw/pci-host/q35.c    |  3 ++-
 include/hw/i386/pc.h |  3 ++-
 4 files changed, 19 insertions(+), 4 deletions(-)
hw/i386/pc.c +13 -1
index 6c82ada..b504047 100644
@@ -1095,11 +1095,23 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 /* setup pci memory address space mapping into system address space */
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space)
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end)
 {
+    uint64_t *val;
+    FWCfgState *fw_cfg = fw_cfg_find();
+
     /* Set to lower priority than RAM */
     memory_region_add_subregion_overlap(system_memory, 0x0,
                                         pci_address_space, -1);
+    g_assert(fw_cfg);
+    /*
+     *  Align address at 1G, this makes sure it can be exactly covered
+     *  with a PAT entry even when using huge pages.
+     */
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(ROUND_UP(reserved_memory_end, 0x1ULL << 30));
+    fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
 }
 
 void pc_acpi_init(const char *default_dsdt)
hw/pci-host/piix.c +2 -1
index 5d4e290..16205e7 100644
@@ -351,7 +351,8 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
-                           f->pci_address_space);
+                           f->pci_address_space,
+                           0x100000000ULL + above_4g_mem_size);
 
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
hw/pci-host/q35.c +2 -1
index d1792de..1293353 100644
@@ -353,7 +353,8 @@ static int mch_init(PCIDevice *d)
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
-                           mch->pci_address_space);
+                           mch->pci_address_space,
+                           0x100000000ULL + mch->above_4g_mem_size);
 
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
include/hw/i386/pc.h +2 -1
index 8b3be3c..2663046 100644
@@ -130,7 +130,8 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space);
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
#5 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Quoted 10 lines · On Tue, 12 Nov 2013 19:26:02 +0100 Paolo Bonzini <[email protected]> wrote:

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

Quoted 6 lines

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Does this make sense?

it will shift BARs only when 'etc/reserved-memory-end' != above4gram_end, presently 'etc/reserved-memory-end' == above4gram_end, so it doesn't affect new->old or old->new migration.
Just to be sure I've done new->old and old->new migration testing, it looks like nothing is broken by patch.

When 'etc/reserved-memory-end' > above4gram_end, new->old migration is unlikely (unrealistically) to be broken due to default 64bit PCI hole size in old QEMU is 1ULL << 62, so to become broken guest should have a BAR that will go beyond 1ULL << 62 border.

But your question have reminded me to make sure that memory hotplug should be disabled for old machine type to avoid clash of memory hotplug region and old 64-bit PCI window.

Quoted 3 lines

Also, would it make sense to use the e820 interface that Gerd has added, instead of a new fw_cfg file?

Gerd suggested it before, but we've come to agreement that dedicated file is cleaner approach than extending standard e820 table with QEMU specific extensions. http://www.mail-archive.com/[email protected]/msg200359.html

Quoted 87 lines

Thanks,

Paolo

Quoted 80 lines
---
 hw/i386/pc.c         | 14 +++++++++++++-
 hw/pci-host/piix.c   |  3 ++-
 hw/pci-host/q35.c    |  3 ++-
 include/hw/i386/pc.h |  3 ++-
 4 files changed, 19 insertions(+), 4 deletions(-)
hw/i386/pc.c +13 -1
index 6c82ada..b504047 100644
@@ -1095,11 +1095,23 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 /* setup pci memory address space mapping into system address space */
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space)
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end)
 {
+    uint64_t *val;
+    FWCfgState *fw_cfg = fw_cfg_find();
+
     /* Set to lower priority than RAM */
     memory_region_add_subregion_overlap(system_memory, 0x0,
                                         pci_address_space, -1);
+    g_assert(fw_cfg);
+    /*
+     *  Align address at 1G, this makes sure it can be exactly covered
+     *  with a PAT entry even when using huge pages.
+     */
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(ROUND_UP(reserved_memory_end, 0x1ULL << 30));
+    fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
 }
 
 void pc_acpi_init(const char *default_dsdt)
hw/pci-host/piix.c +2 -1
index 5d4e290..16205e7 100644
@@ -351,7 +351,8 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
-                           f->pci_address_space);
+                           f->pci_address_space,
+                           0x100000000ULL + above_4g_mem_size);
 
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
hw/pci-host/q35.c +2 -1
index d1792de..1293353 100644
@@ -353,7 +353,8 @@ static int mch_init(PCIDevice *d)
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
-                           mch->pci_address_space);
+                           mch->pci_address_space,
+                           0x100000000ULL + mch->above_4g_mem_size);
 
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
include/hw/i386/pc.h +2 -1
index 8b3be3c..2663046 100644
@@ -130,7 +130,8 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space);
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
Signature · 2 lines
Regards,
  Igor
#6 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

In reply to Paolo Bonzini
Quoted 10 lines · Paolo Bonzini

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

Quoted 6 lines

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Does this make sense?

Yes.
Generally FW CFG must not be added/removed for a given machine types, otherwise guest that is migrated while reading it will
get a corrupted result: half old and half new.

Quoted 6 lines

Also, would it make sense to use the e820 interface that Gerd has added, instead of a new fw_cfg file?

Thanks,

Paolo

This was already discussed on previous revisions of this patch. Have you seen that discussion?

Quoted 81 lines
Quoted 81 lines
---
 hw/i386/pc.c         | 14 +++++++++++++-
 hw/pci-host/piix.c   |  3 ++-
 hw/pci-host/q35.c    |  3 ++-
 include/hw/i386/pc.h |  3 ++-
 4 files changed, 19 insertions(+), 4 deletions(-)
hw/i386/pc.c +13 -1
index 6c82ada..b504047 100644
@@ -1095,11 +1095,23 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 /* setup pci memory address space mapping into system address space */
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space)
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end)
 {
+    uint64_t *val;
+    FWCfgState *fw_cfg = fw_cfg_find();
+
     /* Set to lower priority than RAM */
     memory_region_add_subregion_overlap(system_memory, 0x0,
                                         pci_address_space, -1);
+    g_assert(fw_cfg);
+    /*
+     *  Align address at 1G, this makes sure it can be exactly covered
+     *  with a PAT entry even when using huge pages.
+     */
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(ROUND_UP(reserved_memory_end, 0x1ULL << 30));
+    fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
 }
 
 void pc_acpi_init(const char *default_dsdt)
hw/pci-host/piix.c +2 -1
index 5d4e290..16205e7 100644
@@ -351,7 +351,8 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
-                           f->pci_address_space);
+                           f->pci_address_space,
+                           0x100000000ULL + above_4g_mem_size);
 
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
hw/pci-host/q35.c +2 -1
index d1792de..1293353 100644
@@ -353,7 +353,8 @@ static int mch_init(PCIDevice *d)
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
-                           mch->pci_address_space);
+                           mch->pci_address_space,
+                           0x100000000ULL + mch->above_4g_mem_size);
 
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
include/hw/i386/pc.h +2 -1
index 8b3be3c..2663046 100644
@@ -130,7 +130,8 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 
 
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space);
+                            MemoryRegion *pci_address_space,
+                            uint64_t reserved_memory_end);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
#7 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Il 12/11/2013 23:10, Michael S. Tsirkin ha scritto:

Quoted 2 lines

This was already discussed on previous revisions of this patch. Have you seen that discussion?

Obviously not. :)

Paolo

#8 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

In reply to Michael S. Tsirkin
Quoted 16 lines · On Wed, 13 Nov 2013 00:10:39 +0200 "Michael S. Tsirkin" <[email protected]> wrote:
Quoted 15 lines · Paolo Bonzini

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Quoted 6 lines
Quoted 1 line

Does this make sense?

Yes.
Generally FW CFG must not be added/removed for a given machine types, otherwise guest that is migrated while reading it will
get a corrupted result: half old and half new.

Is it true for a file 'etc/reserved-memory-end' though?

I've debugged SeaBIOS to learn more about it, and new->old migration with following reboot, showed that file is not found by SeaBIOS (well since old QEMU doesn't have it), as result SeaBIOS fallbacks to the old behavior placing 64-PCI bars right above ram_over_4G as it was intended.

And with 'etc/reserved-memory-end' == ram_over_4G_end as it is in this patch, there isn't issue whatsoever.

Looks like there is no migrations issues with files, provided SeaBIOS knows how to deal with a missing file.

Signature · 2 lines
Regards,
  Igor
#9 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Quoted 19 lines · Igor Mammedov
Quoted 16 lines · On Wed, 13 Nov 2013 00:10:39 +0200 "Michael S. Tsirkin" <[email protected]> wrote:
Quoted 15 lines · Paolo Bonzini

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

Does this make sense?

Yes.
Generally FW CFG must not be added/removed for a given machine types, otherwise guest that is migrated while reading it will
get a corrupted result: half old and half new.

Is it true for a file 'etc/reserved-memory-end' though?

It's true for any FW CFG entry.

Quoted 10 lines

I've debugged SeaBIOS to learn more about it, and new->old migration with following reboot, showed that file is not found by SeaBIOS (well since old QEMU doesn't have it), as result SeaBIOS fallbacks to the old behavior placing 64-PCI bars right above ram_over_4G as it was intended.

And with 'etc/reserved-memory-end' == ram_over_4G_end as it is in this patch, there isn't issue whatsoever.

Looks like there is no migrations issues with files, provided SeaBIOS knows how to deal with a missing file.

Here's an example of the issue:

    qemu_cfg_read_entry(&count, QEMU_CFG_FILE_DIR, sizeof(count));

1. <<<<<<

    count = be32_to_cpu(count);

    u32 e;
    for (e = 0; e < count; e++) {
        struct QemuCfgFile qfile;
        qemu_cfg_read(&qfile, sizeof(qfile));
        qemu_romfile_add(qfile.name, be16_to_cpu(qfile.select)
                         , 0, be32_to_cpu(qfile.size));
    }

2. <<<<<<<

If migration happens during qemu_cfg_read_entry before point 1, you can get a byte from old count and a byte from new, resulting in a corrupt count.

If migration happens at point 2, you will get incorrect mapping between file selector and name. System will likely fail to boot.

There are probably other issues.

Bottom line: FW CFG is guest visible state. Migration must not change it.

Quoted 3 lines
Signature · 2 lines
Regards,
  Igor
#10 lore raw

Re: [Qemu-devel] [PATCH 2/2] pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

Quoted 65 lines · On Thu, 14 Nov 2013 09:40:12 +0200 "Michael S. Tsirkin" <[email protected]> wrote:
Quoted 19 lines · Igor Mammedov
Quoted 16 lines · On Wed, 13 Nov 2013 00:10:39 +0200 "Michael S. Tsirkin" <[email protected]> wrote:
Quoted 15 lines · Paolo Bonzini

Il 12/11/2013 14:58, Igor Mammedov ha scritto:

Quoted 8 lines

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

I may be royally wrong, but I think the new file should only be added to new machine types. Otherwise, after migrating old machine types from new QEMU to old QEMU, you may end up with PCI BARs mapped outside the "PCI windows" that exist until before patch 1/2 of this series.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

Does this make sense?

Yes.
Generally FW CFG must not be added/removed for a given machine types, otherwise guest that is migrated while reading it will
get a corrupted result: half old and half new.

Is it true for a file 'etc/reserved-memory-end' though?

It's true for any FW CFG entry.

Quoted 10 lines

I've debugged SeaBIOS to learn more about it, and new->old migration with following reboot, showed that file is not found by SeaBIOS (well since old QEMU doesn't have it), as result SeaBIOS fallbacks to the old behavior placing 64-PCI bars right above ram_over_4G as it was intended.

And with 'etc/reserved-memory-end' == ram_over_4G_end as it is in this patch, there isn't issue whatsoever.

Looks like there is no migrations issues with files, provided SeaBIOS knows how to deal with a missing file.

Here's an example of the issue:

    qemu_cfg_read_entry(&count, QEMU_CFG_FILE_DIR, sizeof(count));

1. <<<<<<

    count = be32_to_cpu(count);

    u32 e;
    for (e = 0; e < count; e++) {
        struct QemuCfgFile qfile;
        qemu_cfg_read(&qfile, sizeof(qfile));
        qemu_romfile_add(qfile.name, be16_to_cpu(qfile.select)
                         , 0, be32_to_cpu(qfile.size));
    }

2. <<<<<<<

If migration happens during qemu_cfg_read_entry before point 1, you can get a byte from old count and a byte from new, resulting in a corrupt count.

Quoted 8 lines

If migration happens at point 2, you will get incorrect mapping between file selector and name. System will likely fail to boot.

There are probably other issues.

Bottom line: FW CFG is guest visible state. Migration must not change it.

Thanks for explanation,
I'll resend this patch with machine compatibility changes.

Quoted 4 lines
Quoted 4 lines
Signature · 3 lines
-- 
Regards,
  Igor
Signature · 2 lines
Regards,
  Igor
#11 lore raw

PATCH v4 2/2 pc: add 'etc/reserved-memory-end' fw_cfg interface for SeaBIOS

In reply to Michael S. Tsirkin

'etc/reserved-memory-end' will allow QEMU to tell BIOS where PCI BARs mapping could safely start in high memory.

Allowing BIOS to start mapping 64-bit PCI BARs at address where it wouldn't conflict with other mappings QEMU might place before it.

That permits QEMU to reserve extra address space before 64-bit PCI hole for memory hotplug.

---
v2:
* disable 'etc/reserved-memory-end' for 1.7 and older machine types

---
 hw/i386/pc.c         | 27 +++++++++++++++++++++++++--
 hw/i386/pc_piix.c    |  1 +
 hw/i386/pc_q35.c     |  1 +
 hw/pci-host/piix.c   |  3 ++-
 hw/pci-host/q35.c    |  3 ++-
 include/hw/i386/pc.h | 11 +++++++++--
 6 files changed, 40 insertions(+), 6 deletions(-)
hw/i386/pc.c +25 -2
index 6c82ada..ba82c67 100644
@@ -1094,14 +1094,37 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 }
 
 /* setup pci memory address space mapping into system address space */
-void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space)
+void pc_pci_as_mapping_init_1_7(Object *owner, MemoryRegion *system_memory,
+                                MemoryRegion *pci_address_space,
+                                uint64_t reserved_memory_end)
 {
     /* Set to lower priority than RAM */
     memory_region_add_subregion_overlap(system_memory, 0x0,
                                         pci_address_space, -1);
 }
 
+static
+void pc_pci_as_mapping_init_1_8(Object *owner, MemoryRegion *system_memory,
+                                MemoryRegion *pci_address_space,
+                                uint64_t reserved_memory_end)
+{
+    uint64_t *val;
+    FWCfgState *fw_cfg = fw_cfg_find();
+    g_assert(fw_cfg);
+    /*
+     *  Align address at 1G, this makes sure it can be exactly covered
+     *  with a PAT entry even when using huge pages.
+     */
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(ROUND_UP(reserved_memory_end, 0x1ULL << 30));
+    fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
+
+    pc_pci_as_mapping_init_1_7(owner, system_memory, pci_address_space,
+                               reserved_memory_end);
+}
+
+pc_pci_as_mapping_init_fn pc_pci_as_mapping_init = pc_pci_as_mapping_init_1_8;
+
 void pc_acpi_init(const char *default_dsdt)
 {
     char *filename;
hw/i386/pc_piix.c +1 -0
index 0456ab1..c0904f9 100644
@@ -240,6 +240,7 @@ static void pc_init_pci(QEMUMachineInitArgs *args)
 
 static void pc_compat_1_7(QEMUMachineInitArgs *args)
 {
+    pc_pci_as_mapping_init = pc_pci_as_mapping_init_1_7;
 }
 
 static void pc_compat_1_6(QEMUMachineInitArgs *args)
hw/i386/pc_q35.c +1 -0
index b18e966..c165843 100644
@@ -224,6 +224,7 @@ static void pc_q35_init(QEMUMachineInitArgs *args)
 
 static void pc_compat_1_7(QEMUMachineInitArgs *args)
 {
+    pc_pci_as_mapping_init = pc_pci_as_mapping_init_1_7;
 }
 
 static void pc_compat_1_6(QEMUMachineInitArgs *args)
hw/pci-host/piix.c +2 -1
index 5d4e290..16205e7 100644
@@ -351,7 +351,8 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
-                           f->pci_address_space);
+                           f->pci_address_space,
+                           0x100000000ULL + above_4g_mem_size);
 
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
hw/pci-host/q35.c +2 -1
index d1792de..1293353 100644
@@ -353,7 +353,8 @@ static int mch_init(PCIDevice *d)
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
-                           mch->pci_address_space);
+                           mch->pci_address_space,
+                           0x100000000ULL + mch->above_4g_mem_size);
 
     /* smram */
     cpu_smm_register(&mch_set_smm, mch);
include/hw/i386/pc.h +9 -2
index 8b3be3c..bf1b168 100644
@@ -129,8 +129,15 @@ PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
 #define DEFAULT_PCI_HOLE64_SIZE (~0x0ULL)
 
 
-void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
-                            MemoryRegion *pci_address_space);
+typedef
+void (*pc_pci_as_mapping_init_fn)(Object *owner, MemoryRegion *system_memory,
+                                  MemoryRegion *pci_address_space,
+                                  uint64_t reserved_memory_end);
+extern pc_pci_as_mapping_init_fn pc_pci_as_mapping_init;
+
+void pc_pci_as_mapping_init_1_7(Object *owner, MemoryRegion *system_memory,
+                                MemoryRegion *pci_address_space,
+                                uint64_t reserved_memory_end);
 
 FWCfgState *pc_memory_init(MemoryRegion *system_memory,
                            const char *kernel_filename,
Signature · 1 line
1.8.3.1