target/arm: Reduced-IPA space and highmem=off fixes
Marc Zyngier
maintainerv2 cover target/arm: Reduced-IPA space and highmem=off fixes
Here's another stab at enabling QEMU on systems with pathologically reduced IPA ranges such as the Apple M1 (original version at [1]). Eventually, we're able to run a KVM guest with more than just 3GB of RAM on a system with a 36bit IPA space, and at most 123 vCPUs.
This series does a few things:
- decouple the enabling of the highmem PCIe region from the highmem attribute
- introduce a new attribute to control the enabling of the highmem GICv3 redistributors
- correctly cap the PA range with highmem is off
- generalise the highmem behaviour to any PA range
- disable both highmem PCIe and GICv3 RDs when they are outside of the PA range
This has been tested on an M1-based Mac-mini running Linux v5.15-rc3.
[1] https://lore.kernel.org/r/[email protected]
Marc Zyngier (5):
hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam hw/arm/virt: Add a control for the the highmem redistributors hw/arm/virt: Honor highmem setting when computing the memory map hw/arm/virt: Use the PA range to compute the memory map hw/arm/virt: Disable highmem devices that don't fit in the PA range
hw/arm/virt-acpi-build.c | 12 ++++----- hw/arm/virt.c | 53 ++++++++++++++++++++++++++++++++++------ include/hw/arm/virt.h | 4 ++- 3 files changed, 54 insertions(+), 15 deletions(-)
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v2 1/5 hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of highmem_ecam. Move the enablement of this PCIe region over to highmem_ecam.
--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)
hw/arm/virt-acpi-build.c +4 -6
@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],
@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) { cfg.mmio64 = memmap[VIRT_HIGH_PCIE_MMIO ]; }
@@ -712,8 +711,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)acpi_dsdt_add_fw_cfg (scope, &memmap[VIRT_FW_CFG ]);acpi_dsdt_add_virtio (scope, &memmap[VIRT_MMIO ], (irqmap[VIRT_MMIO ] +ARM_SPI_BASE ),NUM_VIRTIO_TRANSPORTS ); -acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), - vms->highmem, vms->highmem_ecam, vms); +acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), vms);if (vms->acpi_dev) { build_ged_aml(scope, "\\_SB."GED_DEVICE, HOTPLUG_HANDLER(vms->acpi_dev),
hw/arm/virt.c +2 -2
@@ -1362,7 +1362,7 @@ static void create_pcie(VirtMachineState *vms) mmio_reg, base_mmio, size_mmio);memory_region_add_subregion (get_system_memory (), base_mmio, mmio_alias); -if (vms->highmem) { +if (vms->highmem_ecam) {/* Map high MMIO space */ MemoryRegion *high_mmio_alias =g_new0 (MemoryRegion,1 );
@@ -1416,7 +1416,7 @@ static void create_pcie(VirtMachineState *vms)qemu_fdt_setprop_sized_cells (ms->fdt, nodename,"reg" ,2 , base_ecam,2 , size_ecam); -if (vms->highmem) { +if (vms->highmem_ecam) { qemu_fdt_setprop_sized_cells(ms->fdt, nodename, "ranges", 1, FDT_PCI_RANGE_IOPORT, 2, 0, 2, base_pio, 2, size_pio,
Signature · 1 line
2.30.2
Re: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Quoted 37 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +3
-4@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) { cfg.mmio64 = memmap[VIRT_HIGH_PCIE_MMIO ]; }
hw/arm/virt-acpi-build.c build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) · line 714
Quoted 6 lines
hw/arm/virt-acpi-build.c +1
-2@@ -712,8 +711,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)acpi_dsdt_add_fw_cfg (scope, &memmap[VIRT_FW_CFG ]);acpi_dsdt_add_virtio (scope, &memmap[VIRT_MMIO ], (irqmap[VIRT_MMIO ] +ARM_SPI_BASE ),NUM_VIRTIO_TRANSPORTS ); -acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), - vms->highmem, vms->highmem_ecam, vms); +acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), vms);
While tidying this interface, could also remove the superfluous ().
Quoted 28 lines
hw/arm/virt-acpi-build.c +0
-0if (vms->acpi_dev) { build_ged_aml(scope, "\\_SB."GED_DEVICE, HOTPLUG_HANDLER(vms->acpi_dev),hw/arm/virt.c +2
-2@@ -1362,7 +1362,7 @@ static void create_pcie(VirtMachineState *vms) mmio_reg, base_mmio, size_mmio);memory_region_add_subregion (get_system_memory (), base_mmio, mmio_alias); -if (vms->highmem) { +if (vms->highmem_ecam) {/* Map high MMIO space */ MemoryRegion *high_mmio_alias =g_new0 (MemoryRegion,1 );@@ -1416,7 +1416,7 @@ static void create_pcie(VirtMachineState *vms)qemu_fdt_setprop_sized_cells (ms->fdt, nodename,"reg" ,2 , base_ecam,2 , size_ecam); -if (vms->highmem) { +if (vms->highmem_ecam) { qemu_fdt_setprop_sized_cells(ms->fdt, nodename, "ranges", 1, FDT_PCI_RANGE_IOPORT, 2, 0, 2, base_pio, 2, size_pio,Signature · 1 line
2.30.2
Thanks,
drew
Re: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
In reply to Marc ZyngierHi Marc,
Quoted 28 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +2
-3@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],
hw/arm/virt-acpi-build.c static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, · line 171
Quoted 5 lines
hw/arm/virt-acpi-build.c +1
-1@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) {
highmem_ecam is more restrictive than use_highmem:vms->highmem_ecam &= vms->highmem && (!firmware_loaded || aarch64);
If I remember correctly there was a problem using highmem ECAM with 32b AAVMF FW.
However 5125f9cd2532 ("hw/arm/virt: Add high MMIO PCI region, 512G in size") introduced high MMIO PCI region without this constraint.
So to me we should keep vms->highmem here
Eric
Quoted 35 lines
hw/arm/virt-acpi-build.c +1
-2cfg.mmio64 = memmap[VIRT_HIGH_PCIE_MMIO ]; }@@ -712,8 +711,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)acpi_dsdt_add_fw_cfg (scope, &memmap[VIRT_FW_CFG ]);acpi_dsdt_add_virtio (scope, &memmap[VIRT_MMIO ], (irqmap[VIRT_MMIO ] +ARM_SPI_BASE ),NUM_VIRTIO_TRANSPORTS ); -acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), - vms->highmem, vms->highmem_ecam, vms); +acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), vms);if (vms->acpi_dev) { build_ged_aml(scope, "\\_SB."GED_DEVICE, HOTPLUG_HANDLER(vms->acpi_dev),hw/arm/virt.c +2
-2@@ -1362,7 +1362,7 @@ static void create_pcie(VirtMachineState *vms) mmio_reg, base_mmio, size_mmio);memory_region_add_subregion (get_system_memory (), base_mmio, mmio_alias); -if (vms->highmem) { +if (vms->highmem_ecam) {/* Map high MMIO space */ MemoryRegion *high_mmio_alias =g_new0 (MemoryRegion,1 );@@ -1416,7 +1416,7 @@ static void create_pcie(VirtMachineState *vms)qemu_fdt_setprop_sized_cells (ms->fdt, nodename,"reg" ,2 , base_ecam,2 , size_ecam); -if (vms->highmem) { +if (vms->highmem_ecam) { qemu_fdt_setprop_sized_cells(ms->fdt, nodename, "ranges", 1, FDT_PCI_RANGE_IOPORT, 2, 0, 2, base_pio, 2, size_pio,
Marc Zyngier
maintainerRe: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Eric,
Picking this up again after a stupidly long time...
Quoted 40 lines · Eric Auger
Hi Marc,
Quoted 28 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +2
-3@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],hw/arm/virt-acpi-build.c static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, · line 171
Quoted 5 lines
hw/arm/virt-acpi-build.c +1
-1@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) {
highmem_ecamis more restrictive thanuse_highmem:vms->highmem_ecam&=vms->highmem&& (!firmware_loaded || aarch64);
Quoted 5 lines
If I remember correctly there was a problem using highmem ECAM with 32b AAVMF FW.
However
5125f9cd2532("hw/arm/virt: Add high MMIO PCI region, 512G in size") introduced high MMIO PCI region without this constraint.
Then I really don't understand the point of this highmem_ecam. We only register the highmem version if highmem_ecam is set (see the use of VIRT_ECAM_ID() to pick the right ECAM window).
So keying this on highmem makes it expose a device that may not be there the first place since, as you pointed out that highmem_ecam can be false in cases where highmem is true.
Quoted 1 line
So to me we should keep
vms->highmemhere
I really must be missing how this is supposed to work.
M.
Signature · 5 lines
Without deviation from the norm, progress is not possible. _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Marc,
Quoted 45 lines · Marc Zyngier
Hi Eric,
Picking this up again after a stupidly long time...
Quoted 39 lines · Eric Auger
Hi Marc,
Quoted 28 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +2
-3@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],hw/arm/virt-acpi-build.c static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, · line 171
Quoted 5 lines
hw/arm/virt-acpi-build.c +1
-1@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) {
highmem_ecamis more restrictive thanuse_highmem:vms->highmem_ecam&=vms->highmem&& (!firmware_loaded || aarch64);
Quoted 8 lines
Quoted 5 lines
If I remember correctly there was a problem using highmem ECAM with 32b AAVMF FW.
However
5125f9cd2532("hw/arm/virt: Add high MMIO PCI region, 512G in size") introduced high MMIO PCI region without this constraint.Then I really don't understand the point of this
highmem_ecam. We only register the highmem version ifhighmem_ecamis set (see the use ofVIRT_ECAM_ID()to pick the right ECAM window).
but aren't we talking about different regions? On one hand the [high] MMIO region (512GB wide) and the [high] ECAM region (256MB large). To me you can enable either independently. High MMIO region is used by some devices likes ivshmem/video cards while high ECAM was introduced to extend the number of supported buses: 601d626d148a (hw/arm/virt: Add a new 256MB ECAM region).
with the above change the high MMIO region won't be set with 32b FW+kernel and LPAE whereas it is currently.
high ECAM was not supported by 32b FW, hence the highmem_ecam.
but maybe I miss your point?
Eric
Quoted 10 lines
So keying this on highmem makes it expose a device that may not be there the first place since, as you pointed out that
highmem_ecamcan be false in cases where highmem is true.Quoted 1 line
So to me we should keep
vms->highmemhereI really must be missing how this is supposed to work.
M.
_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerRe: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Eric,
Quoted 65 lines · Eric Auger
Hi Marc,
Quoted 45 lines · Marc Zyngier
Hi Eric,
Picking this up again after a stupidly long time...
Quoted 39 lines · Eric Auger
Hi Marc,
Quoted 28 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +2
-3@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],hw/arm/virt-acpi-build.c static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, · line 171
Quoted 5 lines
hw/arm/virt-acpi-build.c +1
-1@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) {
highmem_ecamis more restrictive thanuse_highmem:vms->highmem_ecam&=vms->highmem&& (!firmware_loaded || aarch64);Quoted 8 lines
Quoted 5 lines
If I remember correctly there was a problem using highmem ECAM with 32b AAVMF FW.
However
5125f9cd2532("hw/arm/virt: Add high MMIO PCI region, 512G in size") introduced high MMIO PCI region without this constraint.Then I really don't understand the point of this
highmem_ecam. We only register the highmem version ifhighmem_ecamis set (see the use ofVIRT_ECAM_ID()to pick the right ECAM window).but aren't we talking about different regions? On one hand the [high] MMIO region (512GB wide) and the [high] ECAM region (256MB large). To me you can enable either independently. High MMIO region is used by some devices likes ivshmem/video cards while high ECAM was introduced to extend the number of supported buses:
601d626d148a(hw/arm/virt: Add a new 256MB ECAM region).
Quoted 6 lines
with the above change the high MMIO region won't be set with 32b FW+kernel and LPAE whereas it is currently.
high ECAM was not supported by 32b FW, hence the
highmem_ecam.but maybe I miss your point?
There are two issues.
First, I have been conflating the ECAM and MMIO ranges, and you only made me realise that they were supposed to be independent. I still think the keying on highmem is wrong, but the main issue is that the highmem* flags don't quite describe the shape of the platform.
All these booleans indicate is whether the feature they describe (the high MMIO range, the high ECAM range, and in one of my patches the high RD range) are *allowed* to live above 4GB, but do not express whether then are actually usable (i.e. fit in the PA range).
Maybe we need to be more thorough in the way we describe the extended region in the VirtMachineState structure:
- highmem: overall control for anything that *can* live above 4GB
- highmem_ecam: Has a PCIe ECAM region above 256GB
- highmem_mmio: Has a PCIe MMIO region above 256GB
- highmem_redist: Has 512 RDs above 256GB
Crucially, the last 3 items must fit in the PA range or be disabled.
We have highmem_ecam which is keyed on highmem, but not on the PA range. highmem_mmio doesn't exist at all (we use highmem instead), and I'm only introducing highmem_redist.
For these 3 ranges, we should have something like
vms->highmem_xxx &= (vms->highmem &&
(vms->memmap[XXX].base + vms->vms->memmap[XXX].size) < vms->highest_gpa);
and treat them as independent entities. Unless someone shouts, I'm going to go ahead and implement this logic.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Re: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Marc,
Quoted 93 lines · Marc Zyngier
Hi Eric,
Quoted 63 lines · Eric Auger
Hi Marc,
Quoted 45 lines · Marc Zyngier
Hi Eric,
Picking this up again after a stupidly long time...
Quoted 39 lines · Eric Auger
Hi Marc,
Quoted 28 lines · Marc Zyngier
Currently, the highmem PCIe region is oddly keyed on the highmem attribute instead of
highmem_ecam. Move the enablement of this PCIe region over tohighmem_ecam.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-)hw/arm/virt-acpi-build.c +2
-3@@ -157,10 +157,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],hw/arm/virt-acpi-build.c static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, · line 171
Quoted 5 lines
hw/arm/virt-acpi-build.c +1
-1@@ -169,7 +168,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_ecam) {
highmem_ecamis more restrictive thanuse_highmem:vms->highmem_ecam&=vms->highmem&& (!firmware_loaded || aarch64);Quoted 8 lines
Quoted 5 lines
If I remember correctly there was a problem using highmem ECAM with 32b AAVMF FW.
However
5125f9cd2532("hw/arm/virt: Add high MMIO PCI region, 512G in size") introduced high MMIO PCI region without this constraint.Then I really don't understand the point of this
highmem_ecam. We only register the highmem version ifhighmem_ecamis set (see the use ofVIRT_ECAM_ID()to pick the right ECAM window).but aren't we talking about different regions? On one hand the [high] MMIO region (512GB wide) and the [high] ECAM region (256MB large). To me you can enable either independently. High MMIO region is used by some devices likes ivshmem/video cards while high ECAM was introduced to extend the number of supported buses:
601d626d148a(hw/arm/virt: Add a new 256MB ECAM region).Quoted 6 lines
with the above change the high MMIO region won't be set with 32b FW+kernel and LPAE whereas it is currently.
high ECAM was not supported by 32b FW, hence the
highmem_ecam.but maybe I miss your point?
There are two issues.
First, I have been conflating the ECAM and MMIO ranges, and you only made me realise that they were supposed to be independent. I still think the keying on highmem is wrong, but the main issue is that the highmem* flags don't quite describe the shape of the platform.
All these booleans indicate is whether the feature they describe (the high MMIO range, the high ECAM range, and in one of my patches the high RD range) are *allowed* to live above 4GB, but do not express whether then are actually usable (i.e. fit in the PA range).
Maybe we need to be more thorough in the way we describe the extended region in the VirtMachineState structure:
- highmem: overall control for anything that *can* live above 4GB
-highmem_ecam: Has a PCIe ECAM region above 256GB
-highmem_mmio: Has a PCIe MMIO region above 256GB
-highmem_redist: Has 512 RDs above 256GB
Quoted 4 lines
Crucially, the last 3 items must fit in the PA range or be disabled.
We have
highmem_ecamwhich is keyed on highmem, but not on the PA range.highmem_mmiodoesn't exist at all (we use highmem instead),
"highmem_ecam is keyed on highmem but not on the PA range". True but it is properly taken into account in highest_gpa computation so eventually we make sure it does not overflow the IPA limit. Same for the high mmio region which is keyed on highmem.
Quoted 6 lines
and I'm only introducing
highmem_redist.For these 3 ranges, we should have something like
vms->highmem_xxx&= (vms->highmem&&(vms->memmap[XXX].base + vms->vms->memmap[XXX].size) < vms->highest_gpa);
couldn't you simply introduce highmem_redist which is truly missing. You could set it in virt_set_memmap() in case you skip extended_map overlay and use it in virt_gicv3_redist_region_count() as you did? In addition to the device memory top address check against the 4GB limit if !highmem, we should be fine then?
Eric
Quoted 8 lines
and treat them as independent entities. Unless someone shouts, I'm going to go ahead and implement this logic.
Thanks,
M.
_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerRe: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Eric,
Quoted 6 lines · Eric Auger
couldn't you simply introduce
highmem_redistwhich is truly missing. You could set it invirt_set_memmap()in case you skipextended_mapoverlay and use it invirt_gicv3_redist_region_count()as you did? In addition to the device memory top address check against the 4GB limit if !highmem, we should be fine then?
No, highmem really isn't nearly enough.
Imagine you have (like I do) a system with 36 bits of IPA space. Create a VM with 8GB of RAM (which means the low-end of IPA space is already 9GB). Obviously, highmem=true here. With the current code, we will try to expose this PCI MMIO range, which falls way out of the IPA space (you need at least 40 bits of IPA to even cover it with the smallest configuration).
highmem really is a control that says 'things may live above 4GB'. It doesn't say *how far* above 4GB it can be placed. Which is what I am trying to address.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Re: [PATCH v2 1/5] hw/arm/virt: Key enablement of highmem PCIe on highmem_ecam
Hi Marc,
Quoted 9 lines · Marc Zyngier
Hi Eric,
Quoted 5 lines · Eric Auger
couldn't you simply introduce
highmem_redistwhich is truly missing. You could set it invirt_set_memmap()in case you skipextended_mapoverlay and use it invirt_gicv3_redist_region_count()as you did? In addition to the device memory top address check against the 4GB limit if !highmem, we should be fine then?
Quoted 8 lines
No, highmem really isn't nearly enough.
Imagine you have (like I do) a system with 36 bits of IPA space. Create a VM with 8GB of RAM (which means the low-end of IPA space is already 9GB). Obviously, highmem=true here. With the current code, we will try to expose this PCI MMIO range, which falls way out of the IPA space (you need at least 40 bits of IPA to even cover it with the smallest configuration).
In that case the he High MMIO region is accounted in the vms->highest_gpa:
for (i = VIRT_LOWMEMMAP_LAST; i < ARRAY_SIZE(extended_memmap); i++) { hwaddr size = extended_memmap[i].size;
base = ROUND_UP(base, size);
vms->memmap[i].base = base;
vms->memmap[i].size = size;
base += size;
}
vms->highest_gpa = base - 1;
and virt_kvm_type() should exit in that case at:
if (requested_pa_size > max_vm_pa_size) { error_report("-m and ,maxmem option values " "require an IPA range (%d bits) larger than " "the one supported by the host (%d bits)", requested_pa_size, max_vm_pa_size); exit(1);
}
?
Quoted 4 lines
highmem really is a control that says 'things may live above 4GB'. It doesn't say *how far* above 4GB it can be placed. Which is what I am trying to address.
OK I will look at your v4 ;-)
Thanks
Eric
Quoted 5 lines
Thanks,
M.
Marc Zyngier
maintainerPATCH v2 2/5 hw/arm/virt: Add a control for the the highmem redistributors
In reply to Marc ZyngierJust like we can control the enablement of the highmem PCIe region using highmem_ecam, let's add a control for the highmem GICv3 redistributor region.
Similarily to highmem_ecam, these redistributors are disabled when highmem is off.
--- hw/arm/virt-acpi-build.c | 2 ++ hw/arm/virt.c | 3 +++ include/hw/arm/virt.h | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-)
hw/arm/virt-acpi-build.c +2 -0
@@ -792,6 +792,8 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); + vms->highmem_redists &= vms->highmem; +acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);
hw/arm/virt.c +3 -0
@@ -2053,6 +2053,8 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); + vms->highmem_redists &= vms->highmem; +create_gic (vms, sysmem);virt_cpu_post_init (vms, sysmem);
@@ -2750,6 +2752,7 @@ static void virt_instance_init(Object *obj) vms->gic_version =VIRT_GIC_VERSION_NOSEL ; vms->highmem_ecam = !vmc->no_highmem_ecam; + vms->highmem_redists = true;if (vmc->no_its) { vms->its = false;
include/hw/arm/virt.h +3 -1
@@ -141,6 +141,7 @@ struct VirtMachineState {bool secure;bool highmem;bool highmem_ecam; +bool highmem_redists;bool its;bool tcg_its;bool virt;
@@ -187,7 +188,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); - return MACHINE(vms)->smp.cpus > redist0_capacity ? 2 : 1; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ? 2 : 1; }#endif /* QEMU_ARM_VIRT_H */
Signature · 1 line
2.30.2
Re: [PATCH v2 2/5] hw/arm/virt: Add a control for the the highmem redistributors
Quoted 61 lines · Marc Zyngier
Just like we can control the enablement of the highmem PCIe region using
highmem_ecam, let's add a control for the highmem GICv3 redistributor region.Similarily to
highmem_ecam, these redistributors are disabled when highmem is off.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 2 ++ hw/arm/virt.c | 3 +++ include/hw/arm/virt.h | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-)hw/arm/virt-acpi-build.c +2
-0@@ -792,6 +792,8 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); + vms->highmem_redists &= vms->highmem; +acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);hw/arm/virt.c +3
-0@@ -2053,6 +2053,8 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); + vms->highmem_redists &= vms->highmem; +create_gic (vms, sysmem);virt_cpu_post_init (vms, sysmem);@@ -2750,6 +2752,7 @@ static void virt_instance_init(Object *obj) vms->gic_version =VIRT_GIC_VERSION_NOSEL ; vms->highmem_ecam = !vmc->no_highmem_ecam; + vms->highmem_redists = true;if (vmc->no_its) { vms->its = false;include/hw/arm/virt.h +1
-0@@ -141,6 +141,7 @@ struct VirtMachineState {bool secure;bool highmem;bool highmem_ecam; +bool highmem_redists;bool its;bool tcg_its;bool virt;
include/hw/arm/virt.h static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) · line 192
Quoted 6 lines
include/hw/arm/virt.h +2
-1@@ -187,7 +188,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); -return MACHINE (vms)->smp.cpus > redist0_capacity ?2 :1 ; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ?2 :1 ;
Wouldn't it be equivalent to just use vms->highmem here?
Quoted 6 lines
include/hw/arm/virt.h +0
-0}#endif /* QEMU_ARM_VIRT_H */ Signature · 1 line
2.30.2
Thanks,
drew
Andrew Jones
Reviewed-byRe: [PATCH v2 2/5] hw/arm/virt: Add a control for the the highmem redistributors
Quoted 1 line · Andrew Jones
On Sun, Oct 03, 2021 at 05:46:02PM +0100, Marc Zyngier wrote:
...
Quoted 6 lines
include/hw/arm/virt.h static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) · line 192
Quoted 4 lines
include/hw/arm/virt.h +2
-1-return MACHINE (vms)->smp.cpus > redist0_capacity ?2 :1 ; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ?2 :1 ;Wouldn't it be equivalent to just use
vms->highmemhere?
OK, I see in the last patch that we may disable highmem_redists but not highmem.
In that case,
Thanks,
drew
Marc Zyngier
maintainerPATCH v2 3/5 hw/arm/virt: Honor highmem setting when computing the memory map
In reply to Marc ZyngierEven when the VM is configured with highmem=off, the highest_gpa field includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.
This leads to failures in virt_kvm_type() on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.
Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
--- hw/arm/virt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
hw/arm/virt.c +8 -1
@@ -1628,6 +1628,11 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +if (!vms->highmem && + vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size
@@ -1657,7 +1662,9 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? + base : + vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ;if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Signature · 1 line
2.30.2
Re: [PATCH v2 3/5] hw/arm/virt: Honor highmem setting when computing the memory map
Quoted 48 lines · Marc Zyngier
Even when the VM is configured with highmem=off, the
highest_gpafield includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.This leads to failures in
virt_kvm_type()on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)hw/arm/virt.c +8
-1@@ -1628,6 +1628,11 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +if (!vms->highmem && + vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size @@ -1657,7 +1662,9 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? + base : + vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ;if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;Signature · 1 line
2.30.2
Re: [PATCH v2 3/5] hw/arm/virt: Honor highmem setting when computing the memory map
In reply to Marc ZyngierHi Marc,
Quoted 35 lines · Marc Zyngier
Even when the VM is configured with highmem=off, the
highest_gpafield includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.This leads to failures in
virt_kvm_type()on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)hw/arm/virt.c +5
-0@@ -1628,6 +1628,11 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +if (!vms->highmem && + vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1667
Quoted 7 lines
hw/arm/virt.c +3
-1@@ -1657,7 +1662,9 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? + base : + vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ;
I think I would have preferred to have
if (vms->highmem) {
for (i = VIRT_LOWMEMMAP_LAST; i < ARRAY_SIZE(extended_memmap); i++) { hwaddr size = extended_memmap[i].size;
base = ROUND_UP(base, size);
vms->memmap[i].base = base;
vms->memmap[i].size = size;
base += size;
}
}
as it is useless to execute that code and create new memmap entries in case of !highmem.
But nevertheless, this looks correct
Eric
Quoted 3 lines
hw/arm/virt.c +0
-0if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Marc Zyngier
maintainerRe: [PATCH v2 3/5] hw/arm/virt: Honor highmem setting when computing the memory map
Quoted 52 lines · Eric Auger
Hi Marc,
Quoted 35 lines · Marc Zyngier
Even when the VM is configured with highmem=off, the
highest_gpafield includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.This leads to failures in
virt_kvm_type()on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)hw/arm/virt.c +5
-0@@ -1628,6 +1628,11 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +if (!vms->highmem && + vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1667
Quoted 7 lines
hw/arm/virt.c +3
-1@@ -1657,7 +1662,9 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? + base : + vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ;I think I would have preferred to have
if (
vms->highmem) {
for (i =VIRT_LOWMEMMAP_LAST; i <ARRAY_SIZE(extended_memmap); i++) { hwaddr size = extended_memmap[i].size;
Quoted 8 lines
base =
ROUND_UP(base, size);
vms->memmap[i].base = base;
vms->memmap[i].size = size;
base += size;
}
}
as it is useless to execute that code and create new memmap entries in case of !highmem.
I agree that it is a bit useless when we only have highmem. But we really want to deal with arbitrary IPA spaces (see how this changes in the follow-up patches), and we need to check that everything fits in the IPA space (and fix things up if they don't).
Quoted 2 lines
But nevertheless, this looks correct
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Marc Zyngier
maintainerPATCH v2 4/5 hw/arm/virt: Use the PA range to compute the memory map
In reply to Marc ZyngierThe highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the virt_set_memmap() function, allowing it to correctly exclude highmem devices if they are outside of the PA range.
--- hw/arm/virt.c | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-)
hw/arm/virt.c +35 -11
@@ -1610,10 +1610,10 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms); -hwaddr base, device_memory_base, device_memory_size; +hwaddr base, device_memory_base, device_memory_size, memtop;int i; vms->memmap = extended_memmap;
@@ -1628,9 +1628,12 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } -if (!vms->highmem && - vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (!vms->highmem) + pa_bits =32 ; + +if (vms->memmap[VIRT_MEM ].base + ms->maxram_size >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, vms->memmap[VIRT_MEM ].base + ms->maxram_size -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }/*
@@ -1645,7 +1648,7 @@ static void virt_set_memmap(VirtMachineState *vms) device_memory_size = ms->maxram_size - ms->ram_size + ms->ram_slots * GiB;/* Base address of the high IO region */ - base = device_memory_base +ROUND_UP (device_memory_size, GiB); + memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB);if (base < device_memory_base) {error_report ("maxmem/slots too huge" );exit (EXIT_FAILURE );
@@ -1662,9 +1665,17 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? - base : - vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. +*/ +if (base <=BIT_ULL (pa_bits)) { + vms->highest_gpa = base -1 ; + }else { + vms->highest_gpa = memtop -1 ; + } +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
@@ -1860,7 +1871,20 @@ static void machvirt_init(MachineState *machine)* to create a VM with the right number of IPA bits. */ if (!vms->memmap) { -virt_set_memmap (vms); +ARMCPU *armcpu =ARM_CPU (first_cpu); +int pa_bits; + +if (object_property_get_bool (OBJECT (first_cpu),"aarch64" ,NULL )) { + pa_bits =arm_pamax (armcpu); + }else if (arm_feature (&armcpu->env,ARM_FEATURE_LPAE )) { +/* v7 with LPAE */ + pa_bits =40 ; + }else { +/* Anything else */ + pa_bits =32 ; + } + +virt_set_memmap (vms, pa_bits); }/* We can probe only here because during property set
@@ -2596,7 +2620,7 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) max_vm_pa_size =kvm_arm_get_max_vm_ipa_size (ms, &fixed_ipa);/* we freeze the memory map to compute the highest gpa */ -virt_set_memmap (vms); +virt_set_memmap (vms, max_vm_pa_size); requested_pa_size =64 -clz64 (vms->highest_gpa);
Signature · 1 line
2.30.2
Re: [PATCH v2 4/5] hw/arm/virt: Use the PA range to compute the memory map
Quoted 54 lines · Marc Zyngier
The highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the
virt_set_memmap()function, allowing it to correctly exclude highmem devices if they are outside of the PA range.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-)hw/arm/virt.c +9
-6@@ -1610,10 +1610,10 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms); -hwaddr base, device_memory_base, device_memory_size; +hwaddr base, device_memory_base, device_memory_size, memtop;int i; vms->memmap = extended_memmap;@@ -1628,9 +1628,12 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } -if (!vms->highmem && - vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (!vms->highmem) + pa_bits =32 ; + +if (vms->memmap[VIRT_MEM ].base + ms->maxram_size >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, vms->memmap[VIRT_MEM ].base + ms->maxram_size -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }/* @@ -1645,7 +1648,7 @@ static void virt_set_memmap(VirtMachineState *vms) device_memory_size = ms->maxram_size - ms->ram_size + ms->ram_slots * GiB;/* Base address of the high IO region */ - base = device_memory_base +ROUND_UP (device_memory_size, GiB); + memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB);if (base < device_memory_base) {error_report ("maxmem/slots too huge" );exit (EXIT_FAILURE );
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1671
Quoted 10 lines
hw/arm/virt.c +4
-3@@ -1662,9 +1665,17 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? - base : - vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits.
We tested that
vms->memmap[VIRT_MEM].base + ms->maxram_size
fits within pa_bits, but here we're setting highest_gpa to
ROUND_UP(vms->memmap[VIRT_MEM].base + ms->ram_size, GiB) + ROUND_UP(ms->maxram_size - ms->ram_size + ms->ram_slots * GiB, GiB)
which will be larger. Shouldn't we test memtop instead to make this guarantee?
Quoted 11 lines
hw/arm/virt.c +7
-0+*/ +if (base <=BIT_ULL (pa_bits)) { + vms->highest_gpa = base -1 ; + }else { + vms->highest_gpa = memtop -1 ; + } +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
hw/arm/virt.c static void machvirt_init(MachineState *machine) · line 1874
Quoted 5 lines
hw/arm/virt.c +1
-1@@ -1860,7 +1871,20 @@ static void machvirt_init(MachineState *machine)* to create a VM with the right number of IPA bits. */ if (!vms->memmap) { -virt_set_memmap (vms); +ARMCPU *armcpu =ARM_CPU (first_cpu);
I think it's too early to use first_cpu here (although, I'll admit I'm always confused as to what gets initialized when...) Assuming we need to realize the cpus first, then we don't do that until a bit further down in this function. I wonder if it's possible to delay this memmap setup until after cpu realization. I see the memmap getting used prior when calculating virt_max_cpus, but that looks like it needs to be updated anyway to take highmem into account as to whether or not we should consider the high gicv3 redist region in the calculation.
Quoted 28 lines
hw/arm/virt.c +14
-1+int pa_bits; + +if (object_property_get_bool (OBJECT (first_cpu),"aarch64" ,NULL )) { + pa_bits =arm_pamax (armcpu); + }else if (arm_feature (&armcpu->env,ARM_FEATURE_LPAE )) { +/* v7 with LPAE */ + pa_bits =40 ; + }else { +/* Anything else */ + pa_bits =32 ; + } + +virt_set_memmap (vms, pa_bits); }/* We can probe only here because during property set @@ -2596,7 +2620,7 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) max_vm_pa_size =kvm_arm_get_max_vm_ipa_size (ms, &fixed_ipa);/* we freeze the memory map to compute the highest gpa */ -virt_set_memmap (vms); +virt_set_memmap (vms, max_vm_pa_size); requested_pa_size =64 -clz64 (vms->highest_gpa);Signature · 1 line
2.30.2
Thanks,
drew
Marc Zyngier
maintainerRe: [PATCH v2 4/5] hw/arm/virt: Use the PA range to compute the memory map
Quoted 71 lines · Andrew Jones
Quoted 54 lines · Marc Zyngier
The highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the
virt_set_memmap()function, allowing it to correctly exclude highmem devices if they are outside of the PA range.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-)hw/arm/virt.c +9
-6@@ -1610,10 +1610,10 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms); -hwaddr base, device_memory_base, device_memory_size; +hwaddr base, device_memory_base, device_memory_size, memtop;int i; vms->memmap = extended_memmap;@@ -1628,9 +1628,12 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } -if (!vms->highmem && - vms->memmap[VIRT_MEM ].base + ms->maxram_size >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (!vms->highmem) + pa_bits =32 ; + +if (vms->memmap[VIRT_MEM ].base + ms->maxram_size >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, vms->memmap[VIRT_MEM ].base + ms->maxram_size -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }/* @@ -1645,7 +1648,7 @@ static void virt_set_memmap(VirtMachineState *vms) device_memory_size = ms->maxram_size - ms->ram_size + ms->ram_slots * GiB;/* Base address of the high IO region */ - base = device_memory_base +ROUND_UP (device_memory_size, GiB); + memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB);if (base < device_memory_base) {error_report ("maxmem/slots too huge" );exit (EXIT_FAILURE );hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1671
Quoted 10 lines
hw/arm/virt.c +4
-3@@ -1662,9 +1665,17 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? - base : - vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. We tested that
vms->memmap[VIRT_MEM].base +
ms->maxram_size
Quoted 7 lines
fits within
pa_bits, but here we're settinghighest_gpato
ROUND_UP(vms->memmap[VIRT_MEM].base + ms->ram_size, GiB)+ROUND_UP(ms->maxram_size - ms->ram_size + ms->ram_slots * GiB, GiB)which will be larger. Shouldn't we test memtop instead to make this guarantee?
Yes, well spotted.
Quoted 20 lines
Quoted 13 lines
hw/arm/virt.c +7
-0+*/ +if (base <=BIT_ULL (pa_bits)) { + vms->highest_gpa = base -1 ; + }else { + vms->highest_gpa = memtop -1 ; + } +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;hw/arm/virt.c static void machvirt_init(MachineState *machine) · line 1874
Quoted 5 lines
hw/arm/virt.c +1
-1@@ -1860,7 +1871,20 @@ static void machvirt_init(MachineState *machine)* to create a VM with the right number of IPA bits. */ if (!vms->memmap) { -virt_set_memmap (vms); +ARMCPU *armcpu =ARM_CPU (first_cpu);
Quoted 8 lines
I think it's too early to use
first_cpuhere (although, I'll admit I'm always confused as to what gets initialized when...) Assuming we need to realize the cpus first, then we don't do that until a bit further down in this function. I wonder if it's possible to delay this memmap setup until after cpu realization. I see the memmap getting used prior when calculatingvirt_max_cpus, but that looks like it needs to be updated anyway to take highmem into account as to whether or not we should consider the high gicv3 redist region in the calculation.
OK, this is nothing short of total hell. You can't create the memory map later, as MTE and the secure world both get in the way (they really want a valid memory map). And as you pointed out, using first_cpu is not appropriate here (obviously, I didn't test this nearly enough). I could split the creation of the CPUs in two sequences with the memory map creation in between, but this quickly becomes quite invasive.
My current approach is to keep the current flow, but to create a temporary CPU, find whatever I need to know about it, and free it. Yes, this is a bit overkill, but it solves the chicken and egg issue simply enough.
Thanks,
M.
Signature · 5 lines
Without deviation from the norm, progress is not possible. _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v2 4/5] hw/arm/virt: Use the PA range to compute the memory map
In reply to Marc ZyngierOn Sun, Oct 03, 2021 at 05:46:04PM +0100, Marc Zyngier wrote:
...
Quoted 4 lines
hw/arm/virt.c +0
-0@@ -1662,9 +1665,17 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; }
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1674
Quoted 10 lines
hw/arm/virt.c +7
-3- vms->highest_gpa = (vms->highmem ? - base : - vms->memmap[VIRT_MEM ].base + ms->maxram_size) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. +*/ +if (base <=BIT_ULL (pa_bits)) { + vms->highest_gpa = base -1 ;
^ missing space here
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1681
Quoted 7 lines
hw/arm/virt.c +4
-0+ }else { + vms->highest_gpa = memtop -1 ; + } +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Thanks,
drew
Marc Zyngier
maintainerPATCH v2 5/5 hw/arm/virt: Disable highmem devices that don't fit in the PA range
In reply to Marc ZyngierMake sure both the highmem PCIe and GICv3 regions are disabled when they don't fully fit in the PA range.
--- hw/arm/virt.c | 3 +++ 1 file changed, 3 insertions(+)
hw/arm/virt.c +3 -0
@@ -1673,6 +1673,9 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits)if (base <=BIT_ULL (pa_bits)) { vms->highest_gpa = base -1 ; }else { +/* Advertise that we have disabled the highmem devices */ + vms->highmem_ecam = false; + vms->highmem_redists = false; vms->highest_gpa = memtop -1 ; }
Signature · 1 line
2.30.2
Re: [PATCH v2 5/5] hw/arm/virt: Disable highmem devices that don't fit in the PA range
Quoted 25 lines · Marc Zyngier
Make sure both the highmem PCIe and GICv3 regions are disabled when they don't fully fit in the PA range.
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 3 +++ 1 file changed, 3 insertions(+)hw/arm/virt.c +3
-0@@ -1673,6 +1673,9 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits)if (base <=BIT_ULL (pa_bits)) { vms->highest_gpa = base -1 ; }else { +/* Advertise that we have disabled the highmem devices */ + vms->highmem_ecam = false; + vms->highmem_redists = false; vms->highest_gpa = memtop -1 ; }Signature · 1 line
2.30.2