target/arm: Reduced-IPA space and highmem fixes
Marc Zyngier
maintainerPATCH v4 1/6 hw/arm/virt: Add a control for the the highmem PCIe MMIO
In reply to Marc ZyngierJust like we can control the enablement of the highmem PCIe ECAM region using highmem_ecam, let's add a control for the highmem PCIe MMIO region.
Similarily to highmem_ecam, this region is disabled when highmem is off.
--- hw/arm/virt-acpi-build.c | 10 ++++------ hw/arm/virt.c | 7 +++++-- include/hw/arm/virt.h | 1 + 3 files changed, 10 insertions(+), 8 deletions(-)
hw/arm/virt-acpi-build.c +4 -6
@@ -158,10 +158,9 @@ static void acpi_dsdt_add_virtio(Aml *scope, }static void acpi_dsdt_add_pci (Aml *scope,const MemMapEntry *memmap, -uint32_t irq,bool use_highmem,bool highmem_ecam, -VirtMachineState *vms) +uint32_t irq,VirtMachineState *vms) { -int ecam_id =VIRT_ECAM_ID (highmem_ecam); +int ecam_id =VIRT_ECAM_ID (vms->highmem_ecam);struct GPEXConfig cfg = { .mmio32 = memmap[VIRT_PCIE_MMIO ], .pio = memmap[VIRT_PCIE_PIO ],
@@ -170,7 +169,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, .bus = vms->bus, }; -if (use_highmem) { +if (vms->highmem_mmio) { cfg.mmio64 = memmap[VIRT_HIGH_PCIE_MMIO ]; }
@@ -868,8 +867,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)acpi_dsdt_add_fw_cfg (scope, &memmap[VIRT_FW_CFG ]);acpi_dsdt_add_virtio (scope, &memmap[VIRT_MMIO ], (irqmap[VIRT_MMIO ] +ARM_SPI_BASE ),NUM_VIRTIO_TRANSPORTS ); -acpi_dsdt_add_pci (scope, memmap, (irqmap[VIRT_PCIE ] +ARM_SPI_BASE ), - vms->highmem, vms->highmem_ecam, vms); +acpi_dsdt_add_pci (scope, memmap, irqmap[VIRT_PCIE ] +ARM_SPI_BASE , vms);if (vms->acpi_dev) { build_ged_aml(scope, "\\_SB."GED_DEVICE, HOTPLUG_HANDLER(vms->acpi_dev),
hw/arm/virt.c +5 -2
@@ -1412,7 +1412,7 @@ static void create_pcie(VirtMachineState *vms) mmio_reg, base_mmio, size_mmio);memory_region_add_subregion (get_system_memory (), base_mmio, mmio_alias); -if (vms->highmem) { +if (vms->highmem_mmio) {/* Map high MMIO space */ MemoryRegion *high_mmio_alias =g_new0 (MemoryRegion,1 );
@@ -1466,7 +1466,7 @@ static void create_pcie(VirtMachineState *vms)qemu_fdt_setprop_sized_cells (ms->fdt, nodename,"reg" ,2 , base_ecam,2 , size_ecam); -if (vms->highmem) { +if (vms->highmem_mmio) { qemu_fdt_setprop_sized_cells(ms->fdt, nodename, "ranges", 1, FDT_PCI_RANGE_IOPORT, 2, 0, 2, base_pio, 2, size_pio,
@@ -2105,6 +2105,8 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); + vms->highmem_mmio &= vms->highmem; +create_gic (vms, sysmem);virt_cpu_post_init (vms, sysmem);
@@ -2802,6 +2804,7 @@ static void virt_instance_init(Object *obj) vms->gic_version =VIRT_GIC_VERSION_NOSEL ; vms->highmem_ecam = !vmc->no_highmem_ecam; + vms->highmem_mmio = true;if (vmc->no_its) { vms->its = false;
include/hw/arm/virt.h +1 -0
@@ -143,6 +143,7 @@ struct VirtMachineState {bool secure;bool highmem;bool highmem_ecam; +bool highmem_mmio;bool its;bool tcg_its;bool virt;
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v4 2/6 hw/arm/virt: Add a control for the the highmem redistributors
In reply to Marc ZyngierJust like we can control the enablement of the highmem PCIe region using highmem_ecam, let's add a control for the highmem GICv3 redistributor region.
Similarily to highmem_ecam, these redistributors are disabled when highmem is off.
--- hw/arm/virt-acpi-build.c | 2 ++ hw/arm/virt.c | 2 ++ include/hw/arm/virt.h | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-)
hw/arm/virt-acpi-build.c +2 -0
@@ -946,6 +946,8 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); + vms->highmem_redists &= vms->highmem; +acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);
hw/arm/virt.c +2 -0
@@ -2106,6 +2106,7 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); vms->highmem_mmio &= vms->highmem; + vms->highmem_redists &= vms->highmem;create_gic (vms, sysmem);
@@ -2805,6 +2806,7 @@ static void virt_instance_init(Object *obj) vms->highmem_ecam = !vmc->no_highmem_ecam; vms->highmem_mmio = true; + vms->highmem_redists = true;if (vmc->no_its) { vms->its = false;
include/hw/arm/virt.h +3 -1
@@ -144,6 +144,7 @@ struct VirtMachineState {bool highmem;bool highmem_ecam;bool highmem_mmio; +bool highmem_redists;bool its;bool tcg_its;bool virt;
@@ -190,7 +191,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); - return MACHINE(vms)->smp.cpus > redist0_capacity ? 2 : 1; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ? 2 : 1; }#endif /* QEMU_ARM_VIRT_H */
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v4 3/6 hw/arm/virt: Honor highmem setting when computing the memory map
In reply to Marc ZyngierEven when the VM is configured with highmem=off, the highest_gpa field includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.
This leads to failures in virt_kvm_type() on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.
Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
--- hw/arm/virt.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
hw/arm/virt.c +7 -3
@@ -1663,7 +1663,7 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)static void virt_set_memmap (VirtMachineState *vms) {MachineState *ms =MACHINE (vms); -hwaddr base, device_memory_base, device_memory_size; +hwaddr base, device_memory_base, device_memory_size, memtop;int i; vms->memmap = extended_memmap;
@@ -1690,7 +1690,11 @@ static void virt_set_memmap(VirtMachineState *vms) device_memory_size = ms->maxram_size - ms->ram_size + ms->ram_slots * GiB;/* Base address of the high IO region */ - base = device_memory_base +ROUND_UP (device_memory_size, GiB); + memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB); +if (!vms->highmem && memtop >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }if (base < device_memory_base) {error_report ("maxmem/slots too huge" );exit (EXIT_FAILURE );
@@ -1707,7 +1711,7 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? base : memtop) -1 ;if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v4 6/6 hw/arm/virt: Drop superfluous checks against highmem
In reply to Marc ZyngierNow that the devices present in the extended memory map are checked against the available PA space and disabled when they don't fit, there is no need to keep the same checks against highmem, as highmem really is a shortcut for the PA space being 32bit.
--- hw/arm/virt-acpi-build.c | 2 -- hw/arm/virt.c | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-)
hw/arm/virt-acpi-build.c +0 -2
@@ -946,8 +946,6 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); - vms->highmem_redists &= vms->highmem; -acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);
hw/arm/virt.c +1 -4
@@ -2170,9 +2170,6 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); - vms->highmem_mmio &= vms->highmem; - vms->highmem_redists &= vms->highmem; -create_gic (vms, sysmem);virt_cpu_post_init (vms, sysmem);
@@ -2191,7 +2188,7 @@ static void machvirt_init(MachineState *machine) machine->ram_size, "mach-virt.tag"); } - vms->highmem_ecam &= vms->highmem && (!firmware_loaded || aarch64); + vms->highmem_ecam &= (!firmware_loaded || aarch64);create_rtc (vms);
Signature · 1 line
2.30.2
Marc Zyngier
maintainerv4 cover target/arm: Reduced-IPA space and highmem fixes
Here's another stab at enabling QEMU on systems with pathologically reduced IPA ranges such as the Apple M1 (previous version at [1]). Eventually, we're able to run a KVM guest with more than just 3GB of RAM on a system with a 36bit IPA space, and at most 123 vCPUs.
This also addresses some pathological QEMU behaviours, where the highmem property is used as a flag allowing exposure of devices that can't possibly fit in the PA space of the VM, resulting in a guest failure.
In the end, we generalise the notion of PA space when exposing individual devices in the expanded memory map, and treat highmem as another flavour or PA space restriction.
This series does a few things:
- introduce new attributes to control the enabling of the highmem GICv3 redistributors and the highmem PCIe MMIO range
- correctly cap the PA range with highmem is off
- generalise the highmem behaviour to any PA range
- disable each highmem device region that doesn't fit in the PA range
- cleanup uses of highmem outside of virt_set_memmap()
This has been tested on an M1-based Mac-mini running Linux v5.16-rc6 with both KVM and TCG.
* From v3 [1]:
- Introduced highmem_mmio as the MMIO pendant to highmem_ecam after Eric made it plain that I was misguided in using highmem_ecam to gate the MMIO region.
- Fixed the way the top of RAM is enforced (using the device memory size, rounded up to the nearest GB). I long debated *not* using the rounded up version, but finally decided that it would be the least surprising, given that each slot is supposed to hold a full GB.
- Now allowing some of the highmem devices to be individually enabled if they fit in the PA range. For example, a system with a 39bit PA range and at most 255GB of RAM can use the highmem redist and PCIe ECAM ranges, but not the high PCIe range.
- Dropped some of Andrew's RBs, as the code significantly changed.
[1] https://lore.kernel.org/r/[email protected]
Marc Zyngier (6):
hw/arm/virt: Add a control for the the highmem PCIe MMIO hw/arm/virt: Add a control for the the highmem redistributors hw/arm/virt: Honor highmem setting when computing the memory map hw/arm/virt: Use the PA range to compute the memory map hw/arm/virt: Disable highmem devices that don't fit in the PA range hw/arm/virt: Drop superfluous checks against highmem
hw/arm/virt-acpi-build.c | 10 ++--- hw/arm/virt.c | 87 +++++++++++++++++++++++++++++++++++----- include/hw/arm/virt.h | 5 ++- 3 files changed, 85 insertions(+), 17 deletions(-)
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v4 4/6 hw/arm/virt: Use the PA range to compute the memory map
The highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the virt_set_memmap() function, allowing it to correctly exclude highmem devices if they are outside of the PA range.
--- hw/arm/virt.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-)
hw/arm/virt.c +46 -7
@@ -1660,7 +1660,7 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms);hwaddr base, device_memory_base, device_memory_size, memtop;
@@ -1678,6 +1678,13 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +/* +* !highmem is exactly the same as limiting the PA space to 32bit, +* irrespective of the underlying capabilities of the HW. +*/ +if (!vms->highmem) + pa_bits =32 ; +/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size
@@ -1691,8 +1698,9 @@ static void virt_set_memmap(VirtMachineState *vms)/* Base address of the high IO region */ memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB); -if (!vms->highmem && memtop >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (memtop >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, memtop -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }if (base < device_memory_base) {
@@ -1711,7 +1719,13 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? base : memtop) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. +*/ + vms->highest_gpa = (base <=BIT_ULL (pa_bits) ? base : memtop) -1 ; +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
@@ -1902,12 +1916,38 @@ static void machvirt_init(MachineState *machine)unsigned int smp_cpus = machine->smp.cpus;unsigned int max_cpus = machine->smp.max_cpus; + possible_cpus = mc->possible_cpu_arch_ids (machine); +/* * In accelerated mode, the memory map is computed earlier in kvm_type() * to create a VM with the right number of IPA bits. */ if (!vms->memmap) { -virt_set_memmap (vms); +Object *cpuobj; +ARMCPU *armcpu; +int pa_bits; + +/* +* Instanciate a temporary CPU object to find out about what +* we are about to deal with. Once this is done, get rid of +* the object. +*/ + cpuobj =object_new (possible_cpus->cpus[0 ].type); + armcpu =ARM_CPU (cpuobj); + +if (object_property_get_bool (cpuobj,"aarch64" ,NULL )) { + pa_bits =arm_pamax (armcpu); + }else if (arm_feature (&armcpu->env,ARM_FEATURE_LPAE )) { +/* v7 with LPAE */ + pa_bits =40 ; + }else { +/* Anything else */ + pa_bits =32 ; + } + +object_unref (cpuobj); + +virt_set_memmap (vms, pa_bits); }/* We can probe only here because during property set
@@ -1989,7 +2029,6 @@ static void machvirt_init(MachineState *machine)create_fdt (vms); - possible_cpus = mc->possible_cpu_arch_ids (machine);assert (possible_cpus->len == max_cpus);for (n =0 ; n < possible_cpus->len; n++) {Object *cpuobj;
@@ -2646,7 +2685,7 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) max_vm_pa_size =kvm_arm_get_max_vm_ipa_size (ms, &fixed_ipa);/* we freeze the memory map to compute the highest gpa */ -virt_set_memmap (vms); +virt_set_memmap (vms, max_vm_pa_size); requested_pa_size =64 -clz64 (vms->highest_gpa);
Signature · 1 line
2.30.2
Marc Zyngier
maintainerPATCH v4 5/6 hw/arm/virt: Disable highmem devices that don't fit in the PA range
In reply to Marc ZyngierIn order to only keep the highmem devices that actually fit in the PA range, check their location against the range and update highest_gpa if they fit. If they don't, mark them them as disabled.
--- hw/arm/virt.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-)
hw/arm/virt.c +28 -6
@@ -1711,21 +1711,43 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits) base = vms->memmap[VIRT_MEM ].base +LEGACY_RAMLIMIT_BYTES ; } +/* We know for sure that at least the memory fits in the PA space */ + vms->highest_gpa = memtop -1 ; +for (i =VIRT_LOWMEMMAP_LAST ; i <ARRAY_SIZE (extended_memmap); i++) {hwaddr size = extended_memmap[i].size; +bool fits; base =ROUND_UP (base, size); vms->memmap[i].base = base; vms->memmap[i].size = size; + +/* +* Check each device to see if they fit in the PA space, +* moving highest_gpa as we go. +* +* For each device that doesn't fit, disable it. +*/ + fits = (base + size) <=BIT_ULL (pa_bits); +if (fits) { + vms->highest_gpa =MAX (vms->highest_gpa, base + size -1 ); + } + +switch (i) { +case VIRT_HIGH_GIC_REDIST2 : + vms->highmem_redists &= fits; +break ; +case VIRT_HIGH_PCIE_ECAM : + vms->highmem_ecam &= fits; +break ; +case VIRT_HIGH_PCIE_MMIO : + vms->highmem_mmio &= fits; +break ; + } + base += size; } -/* -* If base fits within pa_bits, all good. If it doesn't, limit it -* to the end of RAM, which is guaranteed to fit within pa_bits. -*/ - vms->highest_gpa = (base <=BIT_ULL (pa_bits) ? base : memtop) -1 ; -if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Signature · 1 line
2.30.2
Re: [PATCH v4 3/6] hw/arm/virt: Honor highmem setting when computing the memory map
In reply to Marc ZyngierHi Marc,
Quoted 9 lines · Marc Zyngier
Even when the VM is configured with highmem=off, the
highest_gpafield includes devices that are above the 4GiB limit. Similarily, nothing seem to check that the memory is within the limit set by the highmem=off option.This leads to failures in
virt_kvm_type()on systems that have a crippled IPA range, as the reported IPA space is larger than what it should be.
Quoted 6 lines
Instead, honor the user-specified limit to only use the devices at the lowest end of the spectrum, and fail if we have memory crossing the 4GiB limit.
Reviewed-by Andrew Jones <[email protected]>Signed-off-by Marc Zyngier <[email protected]>
Eric
Quoted 39 lines
--- hw/arm/virt.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)hw/arm/virt.c +7
-3@@ -1663,7 +1663,7 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)static void virt_set_memmap (VirtMachineState *vms) {MachineState *ms =MACHINE (vms); -hwaddr base, device_memory_base, device_memory_size; +hwaddr base, device_memory_base, device_memory_size, memtop;int i; vms->memmap = extended_memmap;@@ -1690,7 +1690,11 @@ static void virt_set_memmap(VirtMachineState *vms) device_memory_size = ms->maxram_size - ms->ram_size + ms->ram_slots * GiB;/* Base address of the high IO region */ - base = device_memory_base +ROUND_UP (device_memory_size, GiB); + memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB); +if (!vms->highmem && memtop >4 * GiB) { +error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +exit (EXIT_FAILURE ); + }if (base < device_memory_base) {error_report ("maxmem/slots too huge" );exit (EXIT_FAILURE );@@ -1707,7 +1711,7 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = base -1 ; + vms->highest_gpa = (vms->highmem ? base : memtop) -1 ;if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Re: [PATCH v4 2/6] hw/arm/virt: Add a control for the the highmem redistributors
In reply to Marc ZyngierHi Marc,
Quoted 61 lines · Marc Zyngier
Just like we can control the enablement of the highmem PCIe region using
highmem_ecam, let's add a control for the highmem GICv3 redistributor region.Similarily to
highmem_ecam, these redistributors are disabled when highmem is off.Reviewed-by Andrew Jones <[email protected]>Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt-acpi-build.c | 2 ++ hw/arm/virt.c | 2 ++ include/hw/arm/virt.h | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-)hw/arm/virt-acpi-build.c +2
-0@@ -946,6 +946,8 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); + vms->highmem_redists &= vms->highmem; +acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);hw/arm/virt.c +2
-0@@ -2106,6 +2106,7 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); vms->highmem_mmio &= vms->highmem; + vms->highmem_redists &= vms->highmem;create_gic (vms, sysmem);@@ -2805,6 +2806,7 @@ static void virt_instance_init(Object *obj) vms->highmem_ecam = !vmc->no_highmem_ecam; vms->highmem_mmio = true; + vms->highmem_redists = true;if (vmc->no_its) { vms->its = false;include/hw/arm/virt.h +1
-0@@ -144,6 +144,7 @@ struct VirtMachineState {bool highmem;bool highmem_ecam;bool highmem_mmio; +bool highmem_redists;bool its;bool tcg_its;bool virt;
include/hw/arm/virt.h static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) · line 195
Quoted 6 lines
include/hw/arm/virt.h +2
-1@@ -190,7 +191,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); -return MACHINE (vms)->smp.cpus > redist0_capacity ?2 :1 ; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ?2 :1 ;
If we fail to use the high redist region, is there any check that the number of vcpus does not exceed the first redist region capacity. Did you check that config, does it nicely fail?
Eric
Quoted 3 lines
include/hw/arm/virt.h +0
-0}#endif /* QEMU_ARM_VIRT_H */
Re: [PATCH v4 4/6] hw/arm/virt: Use the PA range to compute the memory map
In reply to Marc ZyngierHi Marc,
Quoted 25 lines · Marc Zyngier
The highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the
virt_set_memmap()function, allowing it to correctly exclude highmem devices if they are outside of the PA range.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-)hw/arm/virt.c +1
-1@@ -1660,7 +1660,7 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms);hwaddr base, device_memory_base, device_memory_size, memtop;
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1686
Quoted 9 lines
hw/arm/virt.c +6
-0@@ -1678,6 +1678,13 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); } +/* +* !highmem is exactly the same as limiting the PA space to 32bit, +* irrespective of the underlying capabilities of the HW. +*/ +if (!vms->highmem) + pa_bits =32 ;
you need {} according to the QEMU coding style. Welcome to a new shiny world :-)
Quoted 32 lines
hw/arm/virt.c +11
-3+/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size @@ -1691,8 +1698,9 @@ static void virt_set_memmap(VirtMachineState *vms)/* Base address of the high IO region */ memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB); -if (!vms->highmem && memtop >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (memtop >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, memtop -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }if (base < device_memory_base) {@@ -1711,7 +1719,13 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? base : memtop) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. +*/ + vms->highest_gpa = (base <=BIT_ULL (pa_bits) ? base : memtop) -1 ; +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
hw/arm/virt.c static void machvirt_init(MachineState *machine) · line 1917
Quoted 2 lines
hw/arm/virt.c +0
-0@@ -1902,12 +1916,38 @@ static void machvirt_init(MachineState *machine)unsigned int smp_cpus = machine->smp.cpus;unsigned int max_cpus = machine->smp.max_cpus;
Move the cpu_type check before?
if (!cpu_type_valid(machine->cpu_type)) { error_report("mach-virt: CPU type %s not supported", machine->cpu_type);
exit(1);
}
Quoted 46 lines
hw/arm/virt.c +27
-2+ possible_cpus = mc->possible_cpu_arch_ids (machine); +/* * In accelerated mode, the memory map is computed earlier in kvm_type() * to create a VM with the right number of IPA bits. */ if (!vms->memmap) { -virt_set_memmap (vms); +Object *cpuobj; +ARMCPU *armcpu; +int pa_bits; + +/* +* Instanciate a temporary CPU object to find out about what +* we are about to deal with. Once this is done, get rid of +* the object. +*/ + cpuobj =object_new (possible_cpus->cpus[0 ].type); + armcpu =ARM_CPU (cpuobj); + +if (object_property_get_bool (cpuobj,"aarch64" ,NULL )) { + pa_bits =arm_pamax (armcpu); + }else if (arm_feature (&armcpu->env,ARM_FEATURE_LPAE )) { +/* v7 with LPAE */ + pa_bits =40 ; + }else { +/* Anything else */ + pa_bits =32 ; + } + +object_unref (cpuobj); + +virt_set_memmap (vms, pa_bits); }/* We can probe only here because during property set @@ -1989,7 +2029,6 @@ static void machvirt_init(MachineState *machine)create_fdt (vms); - possible_cpus = mc->possible_cpu_arch_ids (machine);assert (possible_cpus->len == max_cpus);for (n =0 ; n < possible_cpus->len; n++) {Object *cpuobj;
hw/arm/virt.c static int virt_kvm_type(MachineState *ms, const char *type_str) · line 2691
Quoted 8 lines
hw/arm/virt.c +1
-1@@ -2646,7 +2685,7 @@ static int virt_kvm_type(MachineState *ms, const char *type_str) max_vm_pa_size =kvm_arm_get_max_vm_ipa_size (ms, &fixed_ipa);/* we freeze the memory map to compute the highest gpa */ -virt_set_memmap (vms); +virt_set_memmap (vms, max_vm_pa_size); requested_pa_size =64 -clz64 (vms->highest_gpa);
Thanks
Eric
Marc Zyngier
maintainerRe: [PATCH v4 2/6] hw/arm/virt: Add a control for the the highmem redistributors
In reply to Eric AugerHi Eric,
Quoted 4 lines · Eric Auger
Hi Marc,
On 1/7/22 5:33 PM, Marc Zyngier wrote:
[...]
Quoted 10 lines
include/hw/arm/virt.h static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) · line 195
Quoted 7 lines
include/hw/arm/virt.h +2
-1@@ -190,7 +191,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); -return MACHINE (vms)->smp.cpus > redist0_capacity ?2 :1 ; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ?2 :1 ;If we fail to use the high redist region, is there any check that the number of vcpus does not exceed the first redist region capacity. Did you check that config, does it nicely fail?
I did, and it does (example on M1 with KVM):
$ /home/maz/vminstall/qemu-hack -m 1G -smp 256 -cpu host -machine virt,accel=kvm,gic-version=3,highmem=on -nographic -drive if=pflash,format=raw,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd qemu-hack: warning: Number of SMP cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: warning: Number of hotpluggable cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: Capacity of the redist regions(123) is less than number of vcpus(256)
Thanks,
M.
Signature · 5 lines
Without deviation from the norm, progress is not possible. _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v4 2/6] hw/arm/virt: Add a control for the the highmem redistributors
Quoted 4 lines · Marc Zyngier
$ /home/maz/vminstall/qemu-hack -m 1G -smp 256 -cpu host -machine virt,accel=kvm,gic-version=3,highmem=on -nographic -drive if=pflash,format=raw,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd qemu-hack: warning: Number of SMP cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: warning: Number of hotpluggable cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: Capacity of the redist regions(123) is less than number of vcpus(256)
Side question: why is KVM_CAP_NR_VCPUS returning 8 for "recommended cpus supported by KVM" ? Is something still assuming GICv2 CPU limits?
-- PMM
Re: [PATCH v4 2/6] hw/arm/virt: Add a control for the the highmem redistributors
In reply to Marc ZyngierHi Marc,
Quoted 19 lines · Marc Zyngier
Hi Eric,
Quoted 3 lines · Eric Auger
Hi Marc,
On 1/7/22 5:33 PM, Marc Zyngier wrote:
[...]
Quoted 10 lines
include/hw/arm/virt.h static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) · line 195
Quoted 7 lines
include/hw/arm/virt.h +2
-1@@ -190,7 +191,8 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)assert (vms->gic_version ==VIRT_GIC_VERSION_3 ); -return MACHINE (vms)->smp.cpus > redist0_capacity ?2 :1 ; +return (MACHINE (vms)->smp.cpus > redist0_capacity && + vms->highmem_redists) ?2 :1 ;If we fail to use the high redist region, is there any check that the number of vcpus does not exceed the first redist region capacity. Did you check that config, does it nicely fail?
Quoted 6 lines
I did, and it does (example on M1 with KVM):
$ /home/maz/vminstall/qemu-hack -m 1G -smp 256 -cpu host -machine virt,accel=kvm,gic-version=3,highmem=on -nographic -drive if=pflash,format=raw,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd qemu-hack: warning: Number of SMP cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: warning: Number of hotpluggable cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: Capacity of the redist regions(123) is less than number of vcpus(256)
OK perfect!
Eric
Quoted 5 lines
Thanks,
M.
Marc Zyngier
maintainerRe: [PATCH v4 4/6] hw/arm/virt: Use the PA range to compute the memory map
In reply to Eric AugerQuoted 32 lines · Eric Auger
Hi Marc,
Quoted 28 lines · Marc Zyngier
The highmem attribute is nothing but another way to express the PA range of a VM. To support HW that has a smaller PA range then what QEMU assumes, pass this PA range to the
virt_set_memmap()function, allowing it to correctly exclude highmem devices if they are outside of the PA range.Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-)hw/arm/virt.c +1
-1@@ -1660,7 +1660,7 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx)return arm_cpu_mp_affinity (idx, clustersz); } -static void virt_set_memmap (VirtMachineState *vms) +static void virt_set_memmap (VirtMachineState *vms,int pa_bits) {MachineState *ms =MACHINE (vms);hwaddr base, device_memory_base, device_memory_size, memtop;@@ -1678,6 +1678,13 @@ static void virt_set_memmap(VirtMachineState *vms)exit (EXIT_FAILURE ); }
Quoted 8 lines
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms) · line 1686
Quoted 6 lines
hw/arm/virt.c +6
-0+/* +* !highmem is exactly the same as limiting the PA space to 32bit, +* irrespective of the underlying capabilities of the HW. +*/ +if (!vms->highmem) + pa_bits =32 ;you need {} according to the QEMU coding style. Welcome to a new shiny world :-)
Yeah. Between the reduced indentation and the avalanche of braces, my brain fails to pattern-match blocks of code. Amusing how inflexible you become after a couple of decades...
Quoted 36 lines
Quoted 32 lines
hw/arm/virt.c +11
-3+/* * We compute the base of the high IO region depending on the * amount of initial and device memory. The device memory start/size @@ -1691,8 +1698,9 @@ static void virt_set_memmap(VirtMachineState *vms)/* Base address of the high IO region */ memtop = base = device_memory_base +ROUND_UP (device_memory_size, GiB); -if (!vms->highmem && memtop >4 * GiB) { -error_report ("highmem=off, but memory crosses the 4GiB limit\n" ); +if (memtop >BIT_ULL (pa_bits)) { +error_report ("Addressing limited to %d bits, but memory exceeds it by %llu bytes\n" , + pa_bits, memtop -BIT_ULL (pa_bits));exit (EXIT_FAILURE ); }if (base < device_memory_base) {@@ -1711,7 +1719,13 @@ static void virt_set_memmap(VirtMachineState *vms) vms->memmap[i].size = size; base += size; } - vms->highest_gpa = (vms->highmem ? base : memtop) -1 ; + +/* +* If base fits within pa_bits, all good. If it doesn't, limit it +* to the end of RAM, which is guaranteed to fit within pa_bits. +*/ + vms->highest_gpa = (base <=BIT_ULL (pa_bits) ? base : memtop) -1 ; +if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;hw/arm/virt.c static void machvirt_init(MachineState *machine) · line 1917
Quoted 2 lines
hw/arm/virt.c +0
-0@@ -1902,12 +1916,38 @@ static void machvirt_init(MachineState *machine)unsigned int smp_cpus = machine->smp.cpus;unsigned int max_cpus = machine->smp.max_cpus;Move the
cpu_typecheck before?
Quoted 6 lines
if (!cpu_type_valid(machine->cpu_type)) {
error_report("mach-virt: CPU type %s not supported", machine->cpu_type);
exit(1);
}Quoted 1 line
Yes, very good point. I wonder why this was tucked away past computing the memory map and the GIC configuration... Anyway, I'll move it up.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Marc Zyngier
maintainerRe: [PATCH v4 2/6] hw/arm/virt: Add a control for the the highmem redistributors
In reply to Peter MaydellQuoted 10 lines · Peter Maydell
Quoted 4 lines · Marc Zyngier
$ /home/maz/vminstall/qemu-hack -m 1G -smp 256 -cpu host -machine virt,accel=kvm,gic-version=3,highmem=on -nographic -drive if=pflash,format=raw,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd qemu-hack: warning: Number of SMP cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: warning: Number of hotpluggable cpus requested (256) exceeds the recommended cpus supported by KVM (8) qemu-hack: Capacity of the redist regions(123) is less than number of vcpus(256)Side question: why is
KVM_CAP_NR_VCPUSreturning 8 for "recommended cpus supported by KVM" ? Is something still assuming GICv2 CPU limits?
No, it is only that KVM_CAP_NR_VCPUS is defined as returning the number of physical CPUs (and this test machine has only 8 of them).
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Re: [PATCH v4 5/6] hw/arm/virt: Disable highmem devices that don't fit in the PA range
In reply to Marc ZyngierHi Marc,
Quoted 3 lines · Marc Zyngier
In order to only keep the highmem devices that actually fit in the PA range, check their location against the range and update
highest_gpaif they fit. If they don't, mark them them as disabled.
s/them them/them
Quoted 24 lines
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-)hw/arm/virt.c +4
-0@@ -1711,21 +1711,43 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits) base = vms->memmap[VIRT_MEM ].base +LEGACY_RAMLIMIT_BYTES ; } +/* We know for sure that at least the memory fits in the PA space */ + vms->highest_gpa = memtop -1 ; +for (i =VIRT_LOWMEMMAP_LAST ; i <ARRAY_SIZE (extended_memmap); i++) {hwaddr size = extended_memmap[i].size; +bool fits; base =ROUND_UP (base, size); vms->memmap[i].base = base; vms->memmap[i].size = size;
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms, int pa_bits) · line 1733
Quoted 10 lines
hw/arm/virt.c +10
-0+ +/* +* Check each device to see if they fit in the PA space, +* moving highest_gpa as we go. +* +* For each device that doesn't fit, disable it. +*/ + fits = (base + size) <=BIT_ULL (pa_bits); +if (fits) { + vms->highest_gpa =MAX (vms->highest_gpa, base + size -1 );
why do you need the MAX()?
Quoted 15 lines
hw/arm/virt.c +13
-0+ } +switch (i) { +case VIRT_HIGH_GIC_REDIST2 : + vms->highmem_redists &= fits; +break ; +case VIRT_HIGH_PCIE_ECAM : + vms->highmem_ecam &= fits; +break ; +case VIRT_HIGH_PCIE_MMIO : + vms->highmem_mmio &= fits; +break ; + } + base += size; }
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms, int pa_bits) · line 1753
Quoted 10 lines
hw/arm/virt.c +0
-6-/* -* If base fits within pa_bits, all good. If it doesn't, limit it -* to the end of RAM, which is guaranteed to fit within pa_bits. -*/ - vms->highest_gpa = (base <=BIT_ULL (pa_bits) ? base : memtop) -1 ; -if (device_memory_size >0 ) { ms->device_memory =g_malloc0 (sizeof (*ms->device_memory)); ms->device_memory->base = device_memory_base;
Eric
Re: [PATCH v4 6/6] hw/arm/virt: Drop superfluous checks against highmem
In reply to Marc ZyngierHi Marc,
Quoted 6 lines · Marc Zyngier
Now that the devices present in the extended memory map are checked against the available PA space and disabled when they don't fit, there is no need to keep the same checks against highmem, as highmem really is a shortcut for the PA space being 32bit.
Signed-off-by Marc Zyngier <[email protected]>
Thanks
Eric
Quoted 41 lines
--- hw/arm/virt-acpi-build.c | 2 -- hw/arm/virt.c | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-)hw/arm/virt-acpi-build.c +0
-2@@ -946,8 +946,6 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables)acpi_add_table (table_offsets, tables_blob);build_fadt_rev5 (tables_blob, tables->linker, vms, dsdt); - vms->highmem_redists &= vms->highmem; -acpi_add_table (table_offsets, tables_blob);build_madt (tables_blob, tables->linker, vms);hw/arm/virt.c +1
-4@@ -2170,9 +2170,6 @@ static void machvirt_init(MachineState *machine)virt_flash_fdt (vms, sysmem, secure_sysmem ?: sysmem); - vms->highmem_mmio &= vms->highmem; - vms->highmem_redists &= vms->highmem; -create_gic (vms, sysmem);virt_cpu_post_init (vms, sysmem);@@ -2191,7 +2188,7 @@ static void machvirt_init(MachineState *machine) machine->ram_size, "mach-virt.tag"); } - vms->highmem_ecam &= vms->highmem && (!firmware_loaded || aarch64); + vms->highmem_ecam &= (!firmware_loaded || aarch64);create_rtc (vms);
Marc Zyngier
maintainerRe: [PATCH v4 5/6] hw/arm/virt: Disable highmem devices that don't fit in the PA range
In reply to Eric AugerQuoted 29 lines · Eric Auger
Hi Marc,
Quoted 3 lines · Marc Zyngier
In order to only keep the highmem devices that actually fit in the PA range, check their location against the range and update
highest_gpaif they fit. If they don't, mark them them as disabled.s/them them/them
Quoted 21 lines
Signed-off-by Marc Zyngier <[email protected]>--- hw/arm/virt.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-)hw/arm/virt.c +4
-0@@ -1711,21 +1711,43 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits) base = vms->memmap[VIRT_MEM ].base +LEGACY_RAMLIMIT_BYTES ; } +/* We know for sure that at least the memory fits in the PA space */ + vms->highest_gpa = memtop -1 ; +for (i =VIRT_LOWMEMMAP_LAST ; i <ARRAY_SIZE (extended_memmap); i++) {hwaddr size = extended_memmap[i].size; +bool fits;
Quoted 14 lines
hw/arm/virt.c static void virt_set_memmap(VirtMachineState *vms, int pa_bits) · line 1733
Quoted 13 lines
hw/arm/virt.c +10
-0base =ROUND_UP (base, size); vms->memmap[i].base = base; vms->memmap[i].size = size; + +/* +* Check each device to see if they fit in the PA space, +* moving highest_gpa as we go. +* +* For each device that doesn't fit, disable it. +*/ + fits = (base + size) <=BIT_ULL (pa_bits); +if (fits) { + vms->highest_gpa =MAX (vms->highest_gpa, base + size -1 );why do you need the
MAX()?
Well spotted, I don't. Since we build the memmap by moving base upward, I can directly use 'base + size - 1' as the new highest_gpa value.
Thanks,
M.
Signature · 5 lines
Without deviation from the norm, progress is not possible. _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm