Skip to contentSkip to conversation

pc: memhotplug: rise minimum DIMM addr/size alignment to 128Mb

PATCH In review Igor Mammedov 4 participants 11 messages Sep 21, 2015 – Sep 23, 2015

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 pc: memhotplug: rise minimum DIMM addr/size alignment to 128Mb

it's attempt to workaround virtio bug reported earlier:
http://lists.nongnu.org/archive/html/qemu-devel/2015-08/msg00522.html
where virtio can't handle buffer that crosses border between 2 DIMM's (i.e. 2 MemoryRegions).

Testing showed that virtio doesn't hit above bug with 128Mb DIMM's granularity. Also linux memory hotplug can handle hotplugged memory starting with 128Mb memory sections so lets rise minimum size limit to 128Mb and align starting DIMM address on 128Mb.

It's certainly not the fix but it reduces risk of crashing VM till virtio is fixed.
It also could be improved in guest's virtio side if it would align buffers on 128Mb border and limit max buffer size to the same value.

---
Based on PCI tree as it has patches that add
2.5 machine type.

---
 hw/i386/pc.c         |  8 +++++---
 hw/i386/pc_piix.c    | 12 ++++++++++--
 hw/i386/pc_q35.c     | 12 ++++++++++--
 include/hw/i386/pc.h |  5 ++---
 4 files changed, 27 insertions(+), 10 deletions(-)
hw/i386/pc.c +5 -3
index b5107f7..ddb6710 100644
@@ -1645,8 +1645,9 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     MemoryRegion *mr = ddc->get_memory_region(dimm);
     uint64_t align = TARGET_PAGE_SIZE;
 
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
-        align = memory_region_get_alignment(mr);
+    if (pcms->enforce_aligned_dimm) {
+        align = MAX(memory_region_get_alignment(mr),
+                    pcms->enforce_aligned_dimm);
     }
 
     if (!pcms->acpi_dev) {
@@ -1936,7 +1937,8 @@ static void pc_machine_initfn(Object *obj)
                                     "Enable vmport (pc & q35)",
                                     &error_abort);
 
-    pcms->enforce_aligned_dimm = true;
+    /* align DIMM starting address/size by 128Mb */
+    pcms->enforce_aligned_dimm = 1ULL << 27;
     object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
                              pc_machine_get_aligned_dimm,
                              NULL, &error_abort);
hw/i386/pc_piix.c +10 -2
index caa4edc..7671905 100644
@@ -301,9 +301,17 @@ static void pc_init1(MachineState *machine,
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -326,7 +334,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
 }
 
 static void pc_compat_2_0(MachineState *machine)
@@ -485,7 +493,7 @@ static void pc_i440fx_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", pc_compat_2_3,
                       pc_i440fx_2_4_machine_options)
 
 
hw/i386/pc_q35.c +10 -2
index 506b6bf..72b479f 100644
@@ -284,9 +284,17 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -307,7 +315,7 @@ static void pc_compat_2_1(MachineState *machine)
     PCMachineState *pcms = PC_MACHINE(machine);
 
     pc_compat_2_2(machine);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
 }
@@ -388,7 +396,7 @@ static void pc_q35_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", pc_compat_2_4,
                    pc_q35_2_4_machine_options);
 
 
include/hw/i386/pc.h +2 -3
index 6896328..fdcf0ec 100644
@@ -23,8 +23,7 @@
 /**
  * PCMachineState:
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
+ * @enforce_aligned_dimm: minimal DIMM's address/size alignment
  */
 struct PCMachineState {
     /*< private >*/
@@ -37,9 +36,9 @@ struct PCMachineState {
     ISADevice *rtc;
 
     uint64_t max_ram_below_4g;
+    uint64_t enforce_aligned_dimm;
     OnOffAuto vmport;
     OnOffAuto smm;
-    bool enforce_aligned_dimm;
     ram_addr_t below_4g_mem_size, above_4g_mem_size;
 };
 
Signature · 1 line
1.8.3.1
#1 lore raw
Quoted 11 lines · Igor Mammedov

it's attempt to workaround virtio bug reported earlier:
http://lists.nongnu.org/archive/html/qemu-devel/2015-08/msg00522.html
where virtio can't handle buffer that crosses border between 2 DIMM's (i.e. 2 MemoryRegions).

Testing showed that virtio doesn't hit above bug with 128Mb DIMM's granularity. Also linux memory hotplug can handle hotplugged memory starting with 128Mb memory sections so lets rise minimum size limit to 128Mb and align starting DIMM address on 128Mb.

Quoted 7 lines

It's certainly not the fix but it reduces risk of crashing VM till virtio is fixed.
It also could be improved in guest's virtio side if it would align buffers on 128Mb border and limit max buffer size to the same value.

This seems to be easily handled at a level above QEMU---and the fix would be available to older machine types as well. This patch would also make it quite a bit harder to test the real fix with QEMU. It is not alone a reason to NACK it but should also be kept in mind.

Aligning to 4K makes some sense, since 4K is the page size, but enforcing an arbitrary alignment above 4K is policy that does not belong in QEMU.

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

Paolo

Quoted 142 lines

---
Based on PCI tree as it has patches that add
2.5 machine type.

---
 hw/i386/pc.c         |  8 +++++---
 hw/i386/pc_piix.c    | 12 ++++++++++--
 hw/i386/pc_q35.c     | 12 ++++++++++--
 include/hw/i386/pc.h |  5 ++---
 4 files changed, 27 insertions(+), 10 deletions(-)
hw/i386/pc.c +5 -3
index b5107f7..ddb6710 100644
@@ -1645,8 +1645,9 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     MemoryRegion *mr = ddc->get_memory_region(dimm);
     uint64_t align = TARGET_PAGE_SIZE;
 
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
-        align = memory_region_get_alignment(mr);
+    if (pcms->enforce_aligned_dimm) {
+        align = MAX(memory_region_get_alignment(mr),
+                    pcms->enforce_aligned_dimm);
     }
 
     if (!pcms->acpi_dev) {
@@ -1936,7 +1937,8 @@ static void pc_machine_initfn(Object *obj)
                                     "Enable vmport (pc & q35)",
                                     &error_abort);
 
-    pcms->enforce_aligned_dimm = true;
+    /* align DIMM starting address/size by 128Mb */
+    pcms->enforce_aligned_dimm = 1ULL << 27;
     object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
                              pc_machine_get_aligned_dimm,
                              NULL, &error_abort);
hw/i386/pc_piix.c +10 -2
index caa4edc..7671905 100644
@@ -301,9 +301,17 @@ static void pc_init1(MachineState *machine,
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -326,7 +334,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
 }
 
 static void pc_compat_2_0(MachineState *machine)
@@ -485,7 +493,7 @@ static void pc_i440fx_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", pc_compat_2_3,
                       pc_i440fx_2_4_machine_options)
 
 
hw/i386/pc_q35.c +10 -2
index 506b6bf..72b479f 100644
@@ -284,9 +284,17 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -307,7 +315,7 @@ static void pc_compat_2_1(MachineState *machine)
     PCMachineState *pcms = PC_MACHINE(machine);
 
     pc_compat_2_2(machine);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
 }
@@ -388,7 +396,7 @@ static void pc_q35_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", pc_compat_2_4,
                    pc_q35_2_4_machine_options);
 
 
include/hw/i386/pc.h +2 -3
index 6896328..fdcf0ec 100644
@@ -23,8 +23,7 @@
 /**
  * PCMachineState:
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
+ * @enforce_aligned_dimm: minimal DIMM's address/size alignment
  */
 struct PCMachineState {
     /*< private >*/
@@ -37,9 +36,9 @@ struct PCMachineState {
     ISADevice *rtc;
 
     uint64_t max_ram_below_4g;
+    uint64_t enforce_aligned_dimm;
     OnOffAuto vmport;
     OnOffAuto smm;
-    bool enforce_aligned_dimm;
     ram_addr_t below_4g_mem_size, above_4g_mem_size;
 };
 
#2 lore raw
Quoted 27 lines · On Mon, 21 Sep 2015 14:22:23 +0200 Paolo Bonzini <[email protected]> wrote:
Quoted 11 lines · Igor Mammedov

it's attempt to workaround virtio bug reported earlier:
http://lists.nongnu.org/archive/html/qemu-devel/2015-08/msg00522.html
where virtio can't handle buffer that crosses border between 2 DIMM's (i.e. 2 MemoryRegions).

Testing showed that virtio doesn't hit above bug with 128Mb DIMM's granularity. Also linux memory hotplug can handle hotplugged memory starting with 128Mb memory sections so lets rise minimum size limit to 128Mb and align starting DIMM address on 128Mb.

Quoted 7 lines

It's certainly not the fix but it reduces risk of crashing VM till virtio is fixed.
It also could be improved in guest's virtio side if it would align buffers on 128Mb border and limit max buffer size to the same value.

This seems to be easily handled at a level above QEMU---and the fix would be available to older machine types as well. This patch would also make it quite a bit harder to test the real fix with QEMU. It is not alone a reason to NACK it but should also be kept in mind.

Quoted 8 lines

Aligning to 4K makes some sense, since 4K is the page size, but enforcing an arbitrary alignment above 4K is policy that does not belong in QEMU.

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

in current machine types we already enforce backend-s address/size alignment, which is file's page size for hugetlbfs-backed memory and 2Mb for RAM backend.

So I guess we could try to apply workaround to virtio on guest side, aligning and limiting max buffer size to 2Mb, it should work for 'old' machine types as well.

Quoted 145 lines

Paolo

Quoted 142 lines

---
Based on PCI tree as it has patches that add
2.5 machine type.

---
 hw/i386/pc.c         |  8 +++++---
 hw/i386/pc_piix.c    | 12 ++++++++++--
 hw/i386/pc_q35.c     | 12 ++++++++++--
 include/hw/i386/pc.h |  5 ++---
 4 files changed, 27 insertions(+), 10 deletions(-)
hw/i386/pc.c +5 -3
index b5107f7..ddb6710 100644
@@ -1645,8 +1645,9 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     MemoryRegion *mr = ddc->get_memory_region(dimm);
     uint64_t align = TARGET_PAGE_SIZE;
 
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
-        align = memory_region_get_alignment(mr);
+    if (pcms->enforce_aligned_dimm) {
+        align = MAX(memory_region_get_alignment(mr),
+                    pcms->enforce_aligned_dimm);
     }
 
     if (!pcms->acpi_dev) {
@@ -1936,7 +1937,8 @@ static void pc_machine_initfn(Object *obj)
                                     "Enable vmport (pc & q35)",
                                     &error_abort);
 
-    pcms->enforce_aligned_dimm = true;
+    /* align DIMM starting address/size by 128Mb */
+    pcms->enforce_aligned_dimm = 1ULL << 27;
     object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
                              pc_machine_get_aligned_dimm,
                              NULL, &error_abort);
hw/i386/pc_piix.c +10 -2
index caa4edc..7671905 100644
@@ -301,9 +301,17 @@ static void pc_init1(MachineState *machine,
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -326,7 +334,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
 }
 
 static void pc_compat_2_0(MachineState *machine)
@@ -485,7 +493,7 @@ static void pc_i440fx_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", pc_compat_2_3,
                       pc_i440fx_2_4_machine_options)
 
 
hw/i386/pc_q35.c +10 -2
index 506b6bf..72b479f 100644
@@ -284,9 +284,17 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -307,7 +315,7 @@ static void pc_compat_2_1(MachineState *machine)
     PCMachineState *pcms = PC_MACHINE(machine);
 
     pc_compat_2_2(machine);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
 }
@@ -388,7 +396,7 @@ static void pc_q35_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", pc_compat_2_4,
                    pc_q35_2_4_machine_options);
 
 
include/hw/i386/pc.h +2 -3
index 6896328..fdcf0ec 100644
@@ -23,8 +23,7 @@
 /**
  * PCMachineState:
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
+ * @enforce_aligned_dimm: minimal DIMM's address/size alignment
  */
 struct PCMachineState {
     /*< private >*/
@@ -37,9 +36,9 @@ struct PCMachineState {
     ISADevice *rtc;
 
     uint64_t max_ram_below_4g;
+    uint64_t enforce_aligned_dimm;
     OnOffAuto vmport;
     OnOffAuto smm;
-    bool enforce_aligned_dimm;
     ram_addr_t below_4g_mem_size, above_4g_mem_size;
 };
 
#3 lore raw
Quoted 6 lines · Igor Mammedov
Quoted 4 lines
Quoted 4 lines

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

in current machine types we already enforce backend-s address/size alignment, which is file's page size for hugetlbfs-backed memory and 2Mb for RAM backend.

Right, but it's not enough if the guest's physical address is not aligned to 2Mb/1Gb too. This is why we changed i440FX and q35 to have only 3 and 2 gigabytes of low memory (down from 3.5 and 2 IIRC).

Quoted 3 lines

So I guess we could try to apply workaround to virtio on guest side, aligning and limiting max buffer size to 2Mb, it should work for 'old' machine types as well.

That would make sense and it would be complementary to natural alignment of DIMMs in the host. This would give:

host guest
old old fails
old new works (virtio workaround)
new old works (natural alignment)
new new works (choose your favorite workaround)

Paolo

#4 lore raw
Quoted 7 lines · On Mon, 21 Sep 2015 15:13:17 +0200 Paolo Bonzini <[email protected]> wrote:
Quoted 4 lines · Igor Mammedov
Quoted 4 lines
Quoted 4 lines

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

Quoted 6 lines
Quoted 2 lines

in current machine types we already enforce backend-s address/size alignment, which is file's page size for hugetlbfs-backed memory and 2Mb for RAM backend.

Right, but it's not enough if the guest's physical address is not aligned to 2Mb/1Gb too. This is why we changed i440FX and q35 to have only 3 and 2 gigabytes of low memory (down from 3.5 and 2 IIRC).

DIMM's GPA is aligned to backend's alignment since 2.2,
it should be aligned 2Mb/1Gb depending on what hugetlbfs file is used so that already works as expected

Quoted 5 lines
Quoted 4 lines

So I guess we could try to apply workaround to virtio on guest side, aligning and limiting max buffer size to 2Mb, it should work for 'old' machine types as well.

Quoted 7 lines

That would make sense and it would be complementary to natural alignment of DIMMs in the host. This would give:

host guest
old old fails
old new works (virtio workaround)
new old works (natural alignment)

+ not sure if it would work,
I've though that virtio refactoring, that drops requirement for buffer to be inside of only one MemoryRegion, would touch virtio in QEMU and on guest side too.

Quoted 3 lines

new new works (choose your favorite workaround)

Paolo

#5 lore raw
Quoted 13 lines · Igor Mammedov
Quoted 10 lines · On Mon, 21 Sep 2015 15:13:17 +0200 Paolo Bonzini <[email protected]> wrote:
Quoted 6 lines · Igor Mammedov
Quoted 4 lines
Quoted 4 lines

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

in current machine types we already enforce backend-s address/size alignment, which is file's page size for hugetlbfs-backed memory and 2Mb for RAM backend.

Quoted 7 lines
Quoted 3 lines

Right, but it's not enough if the guest's physical address is not aligned to 2Mb/1Gb too. This is why we changed i440FX and q35 to have only 3 and 2 gigabytes of low memory (down from 3.5 and 2 IIRC).

DIMM's GPA is aligned to backend's alignment since 2.2,
it should be aligned 2Mb/1Gb depending on what hugetlbfs file is used so that already works as expected

Oh, ok. This is what pcms->enforce_aligned_dimm does when true, and this is also what I was missing. Great!

Quoted 7 lines
Quoted 7 lines
Quoted 3 lines

So I guess we could try to apply workaround to virtio on guest side, aligning and limiting max buffer size to 2Mb, it should work for 'old' machine types as well.

That would make sense and it would be complementary to natural alignment of DIMMs in the host. This would give:

Quoted 8 lines
Quoted 4 lines

host guest
old old fails
old new works (virtio workaround)
new old works (natural alignment)

+ not sure if it would work,
I've though that virtio refactoring, that drops requirement for buffer to be inside of only one MemoryRegion, would touch virtio in QEMU and on guest side too.

Right, it would theoretically not be enough. However, I think it would work in practice because hot-plugged DIMMs should be bigger than 128 MiB.

Paolo

#6 lore raw
In reply to Paolo Bonzini
Quoted 20 lines · On Mon, 21 Sep 2015 14:22:23 +0200 Paolo Bonzini <[email protected]> wrote:
Quoted 17 lines · Igor Mammedov

it's attempt to workaround virtio bug reported earlier:
http://lists.nongnu.org/archive/html/qemu-devel/2015-08/msg00522.html
where virtio can't handle buffer that crosses border between 2 DIMM's (i.e. 2 MemoryRegions).

Testing showed that virtio doesn't hit above bug with 128Mb DIMM's granularity. Also linux memory hotplug can handle hotplugged memory starting with 128Mb memory sections so lets rise minimum size limit to 128Mb and align starting DIMM address on 128Mb.

It's certainly not the fix but it reduces risk of crashing VM till virtio is fixed.
It also could be improved in guest's virtio side if it would align buffers on 128Mb border and limit max buffer size to the same value.

Quoted 6 lines
Quoted 1 line

This seems to be easily handled at a level above QEMU---and the fix would be available to older machine types as well. This patch would also make it quite a bit harder to test the real fix with QEMU. It is not alone a reason to NACK it but should also be kept in mind.

Patch makes it easy to change enforced alignment for future machine types, so lowering alignment for testing isn't hard when virtio is fixed.

Handling it at libvirt level is a bit hard since currently it doesn't deal with DIMM.addr allocation and there isn't any interface to communicate hotplug address range to libvirt. Libvirt doesn't need to know anything about DIMM.addr except of migration when it needs to replicate state on target side.

Also it's QEMU bug/fault and pushing workaround to upper layers doesn't seem right when it's much easier to do it in QEMU itself.

Quoted 154 lines

Aligning to 4K makes some sense, since 4K is the page size, but enforcing an arbitrary alignment above 4K is policy that does not belong in QEMU.

To some extend, enforcing natural alignment would be okay as a workaround for the virtio bug as well. It would also make it easier to ensure that hotplugged hugetlbfs-backed memory can use hugepages in the guest. Does it make sense to you?

Paolo

Quoted 142 lines

---
Based on PCI tree as it has patches that add
2.5 machine type.

---
 hw/i386/pc.c         |  8 +++++---
 hw/i386/pc_piix.c    | 12 ++++++++++--
 hw/i386/pc_q35.c     | 12 ++++++++++--
 include/hw/i386/pc.h |  5 ++---
 4 files changed, 27 insertions(+), 10 deletions(-)
hw/i386/pc.c +5 -3
index b5107f7..ddb6710 100644
@@ -1645,8 +1645,9 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     MemoryRegion *mr = ddc->get_memory_region(dimm);
     uint64_t align = TARGET_PAGE_SIZE;
 
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
-        align = memory_region_get_alignment(mr);
+    if (pcms->enforce_aligned_dimm) {
+        align = MAX(memory_region_get_alignment(mr),
+                    pcms->enforce_aligned_dimm);
     }
 
     if (!pcms->acpi_dev) {
@@ -1936,7 +1937,8 @@ static void pc_machine_initfn(Object *obj)
                                     "Enable vmport (pc & q35)",
                                     &error_abort);
 
-    pcms->enforce_aligned_dimm = true;
+    /* align DIMM starting address/size by 128Mb */
+    pcms->enforce_aligned_dimm = 1ULL << 27;
     object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
                              pc_machine_get_aligned_dimm,
                              NULL, &error_abort);
hw/i386/pc_piix.c +10 -2
index caa4edc..7671905 100644
@@ -301,9 +301,17 @@ static void pc_init1(MachineState *machine,
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -326,7 +334,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
 }
 
 static void pc_compat_2_0(MachineState *machine)
@@ -485,7 +493,7 @@ static void pc_i440fx_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", pc_compat_2_3,
                       pc_i440fx_2_4_machine_options)
 
 
hw/i386/pc_q35.c +10 -2
index 506b6bf..72b479f 100644
@@ -284,9 +284,17 @@ static void pc_q35_init(MachineState *machine)
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -307,7 +315,7 @@ static void pc_compat_2_1(MachineState *machine)
     PCMachineState *pcms = PC_MACHINE(machine);
 
     pc_compat_2_2(machine);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
 }
@@ -388,7 +396,7 @@ static void pc_q35_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", pc_compat_2_4,
                    pc_q35_2_4_machine_options);
 
 
include/hw/i386/pc.h +2 -3
index 6896328..fdcf0ec 100644
@@ -23,8 +23,7 @@
 /**
  * PCMachineState:
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
+ * @enforce_aligned_dimm: minimal DIMM's address/size alignment
  */
 struct PCMachineState {
     /*< private >*/
@@ -37,9 +36,9 @@ struct PCMachineState {
     ISADevice *rtc;
 
     uint64_t max_ram_below_4g;
+    uint64_t enforce_aligned_dimm;
     OnOffAuto vmport;
     OnOffAuto smm;
-    bool enforce_aligned_dimm;
     ram_addr_t below_4g_mem_size, above_4g_mem_size;
 };
 
#7 lore raw
Quoted 2 lines · Igor Mammedov

Also it's QEMU bug/fault and pushing workaround to upper layers doesn't seem right when it's much easier to do it in QEMU itself.

No, it's virtio's bug. It makes sense to push workaround for guest bugs as far from the hypervisor as possible.

If we want to increase the alignment in QEMU, I would prefer to have natural alignment which makes some sense and happens to work around the bug as well. Michael, Eduardo, any third opinions?

Paolo

#8 lore raw
Quoted 6 lines · On Wed, 23 Sep 2015 11:38:56 +0200 Paolo Bonzini <[email protected]> wrote:
Quoted 2 lines · Igor Mammedov

Also it's QEMU bug/fault and pushing workaround to upper layers doesn't seem right when it's much easier to do it in QEMU itself.

Quoted 6 lines

No, it's virtio's bug. It makes sense to push workaround for guest bugs as far from the hypervisor as possible.

If we want to increase the alignment in QEMU, I would prefer to have natural alignment which makes some sense and happens to work around the bug as well. Michael, Eduardo, any third opinions?

Natural alignment we have now is 2Mb, we can leave hipervisor side as is and just try to align on 2Mb in guest + limit buffer size to 2Mb as we that would make sure that buffer never crosses DIMM borders.

Quoted 3 lines

Paolo

#9 lore raw
In reply to Paolo Bonzini
Quoted 8 lines · * Paolo Bonzini ([email protected]) wrote:
Quoted 2 lines · Igor Mammedov

Also it's QEMU bug/fault and pushing workaround to upper layers doesn't seem right when it's much easier to do it in QEMU itself.

No, it's virtio's bug. It makes sense to push workaround for guest bugs as far from the hypervisor as possible.

But you really don't want to have higher level things having to align addresses themselves. I could see adding an option for the required alignment would be OK.

Quoted 4 lines

If we want to increase the alignment in QEMU, I would prefer to have natural alignment which makes some sense and happens to work around the bug as well. Michael, Eduardo, any third opinions?

By natural alignment do you mean that an 'n MB' DIMM gets aligned to 'n MB' ?

Dave

Quoted 1 line

Paolo

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#10 lore raw
In reply to Igor Mammedov

On Mon, Sep 21, 2015 at 01:50:23PM +0200, Igor Mammedov wrote:
[...]

Quoted 5 lines
hw/i386/pc_piix.c +0 -0
index caa4edc..7671905 100644

hw/i386/pc_piix.c static void pc_init1(MachineState *machine, · line 310

Quoted 10 lines
hw/i386/pc_piix.c +7 -0
@@ -301,9 +301,17 @@ static void pc_init1(MachineState *machine,
     }
 }
 
+static void pc_compat_2_4(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    pcms->enforce_aligned_dimm = TARGET_PAGE_SIZE;
+}
+

I want to stop creating new machine-init-time pc_compat_*() functions because they make it harder to do any analysis or introspection of machine class data in the future.

If enforce_aligned_dimm never changes during the lifetime of the machine object, it can be moved to PCMachineClass. If it can change, you can allow it to be set using PCMachineClass::default_machine_opts by providing a property setter.

Quoted 17 lines
hw/i386/pc_piix.c +2 -1
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
+    pc_compat_2_4(machine);
     savevm_skip_section_footers();
     if (kvm_enabled()) {
         pcms->smm = ON_OFF_AUTO_OFF;
@@ -326,7 +334,7 @@ static void pc_compat_2_1(MachineState *machine)
     pc_compat_2_2(machine);
     smbios_uuid_encoded = false;
     x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pcms->enforce_aligned_dimm = 0;
 }
 
 static void pc_compat_2_0(MachineState *machine)

hw/i386/pc_piix.c static void pc_i440fx_2_4_machine_options(MachineClass *m) · line 496

Quoted 5 lines
hw/i386/pc_piix.c +1 -1
@@ -485,7 +493,7 @@ static void pc_i440fx_2_4_machine_options(MachineClass *m)
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", pc_compat_2_3,

Did you mean pc_compat_2_4?

Quoted 3 lines
hw/i386/pc_piix.c +0 -0
                       pc_i440fx_2_4_machine_options)
 
 
Signature · 1 line
Eduardo