Skip to contentSkip to conversation

spapr: Add associativity reference point count to machine info

PATCH In review Reza Arbab 3 participants 8 messages May 18, 2020 – May 21, 2020

lore
Share & tools
lore

Reza Arbab

author
#0 lore raw

PATCH v2 1/2 spapr: Add associativity reference point count to machine info

Make the number of NUMA associativity reference points a machine-specific value, using the currently assumed default (two reference points). This preps the next patch to conditionally change it.

---
 hw/ppc/spapr.c         | 6 +++++-
 include/hw/ppc/spapr.h | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +5 -1
index c18eab0a2305..88b4a1f17716 100644
@@ -889,10 +889,12 @@ static int spapr_dt_rng(void *fdt)
 static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
 {
     MachineState *ms = MACHINE(spapr);
+    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(ms);
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
     uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
     uint32_t lrdr_capacity[] = {
@@ -944,8 +946,9 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
                      qemu_hypertas->str, qemu_hypertas->len));
     g_string_free(qemu_hypertas, TRUE);
 
+    nr_refpoints = MIN(smc->nr_assoc_refpoints, ARRAY_SIZE(refpoints));
     _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
-                     refpoints, sizeof(refpoints)));
+                     refpoints, nr_refpoints * sizeof(uint32_t)));
 
     _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains",
                      maxdomains, sizeof(maxdomains)));
@@ -4541,6 +4544,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
+    smc->nr_assoc_refpoints = 2;
     xfc->match_nvt = spapr_match_nvt;
 }
 
include/hw/ppc/spapr.h +1 -0
index e579eaf28c05..abaf9a92adc0 100644
@@ -129,6 +129,7 @@ struct SpaprMachineClass {
     bool linux_pci_probe;
     bool smp_threads_vsmt; /* set VSMT to smp_threads by default */
     hwaddr rma_limit;          /* clamp the RMA to this size */
+    uint32_t nr_assoc_refpoints;
 
     void (*phb_placement)(SpaprMachineState *spapr, uint32_t index,
                           uint64_t *buid, hwaddr *pio, 
Signature · 1 line
2.18.2
author
#1 lore raw

PATCH v2 2/2 spapr: Add a new level of NUMA for GPUs

NUMA nodes corresponding to GPU memory currently have the same affinity/distance as normal memory nodes. Add a third NUMA associativity reference point enabling us to give GPU nodes more distance.

This is guest visible information, which shouldn't change under a running guest across migration between different qemu versions, so make the change effective only in new (pseries > 5.0) machine types.

Before, numactl -H output in a guest with 4 GPUs (nodes 2-5):

node distances:
node 0 1 2 3 4 5

  0:  10  40  40  40  40  40
  1:  40  10  40  40  40  40
  2:  40  40  10  40  40  40
  3:  40  40  40  10  40  40
  4:  40  40  40  40  10  40
  5:  40  40  40  40  40  10

After:

node distances:
node 0 1 2 3 4 5

  0:  10  40  80  80  80  80
  1:  40  10  80  80  80  80
  2:  80  80  10  80  80  80
  3:  80  80  80  10  80  80
  4:  80  80  80  80  10  80
  5:  80  80  80  80  80  10

These are the same distances as on the host, mirroring the change made to host firmware in skiboot commit f845a648b8cb ("numa/associativity:
Add a new level of NUMA for GPU's").

---
 hw/ppc/spapr.c             | 11 +++++++++--
 hw/ppc/spapr_pci_nvlink2.c |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)
hw/ppc/spapr.c +9 -2
index 88b4a1f17716..1d9193d5ee49 100644
@@ -893,7 +893,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
-    uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t refpoints[] = {
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x2),
+    };
     uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
@@ -4544,7 +4548,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
-    smc->nr_assoc_refpoints = 2;
+    smc->nr_assoc_refpoints = 3;
     xfc->match_nvt = spapr_match_nvt;
 }
 
@@ -4611,8 +4615,11 @@ DEFINE_SPAPR_MACHINE(5_1, "5.1", true);
  */
 static void spapr_machine_5_0_class_options(MachineClass *mc)
 {
+    SpaprMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_5_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_5_0, hw_compat_5_0_len);
+    smc->nr_assoc_refpoints = 2;
 }
 
 DEFINE_SPAPR_MACHINE(5_0, "5.0", false);
hw/ppc/spapr_pci_nvlink2.c +1 -1
index 8332d5694e46..247fd48731e2 100644
@@ -362,7 +362,7 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
         uint32_t associativity[] = {
             cpu_to_be32(0x4),
             SPAPR_GPU_NUMA_ID,
-            SPAPR_GPU_NUMA_ID,
+            cpu_to_be32(nvslot->numa_id),
             SPAPR_GPU_NUMA_ID,
             cpu_to_be32(nvslot->numa_id)
         };
Signature · 1 line
2.18.2
#2 lore raw

Re: [PATCH v2 2/2] spapr: Add a new level of NUMA for GPUs

Quoted 83 lines · On Mon, 18 May 2020 16:44:18 -0500 Reza Arbab <[email protected]> wrote:

NUMA nodes corresponding to GPU memory currently have the same affinity/distance as normal memory nodes. Add a third NUMA associativity reference point enabling us to give GPU nodes more distance.

This is guest visible information, which shouldn't change under a running guest across migration between different qemu versions, so make the change effective only in new (pseries > 5.0) machine types.

Before, numactl -H output in a guest with 4 GPUs (nodes 2-5):

node distances:
node 0 1 2 3 4 5

  0:  10  40  40  40  40  40
  1:  40  10  40  40  40  40
  2:  40  40  10  40  40  40
  3:  40  40  40  10  40  40
  4:  40  40  40  40  10  40
  5:  40  40  40  40  40  10

After:

node distances:
node 0 1 2 3 4 5

  0:  10  40  80  80  80  80
  1:  40  10  80  80  80  80
  2:  80  80  10  80  80  80
  3:  80  80  80  10  80  80
  4:  80  80  80  80  10  80
  5:  80  80  80  80  80  10

These are the same distances as on the host, mirroring the change made to host firmware in skiboot commit f845a648b8cb ("numa/associativity:
Add a new level of NUMA for GPU's").

---
 hw/ppc/spapr.c             | 11 +++++++++--
 hw/ppc/spapr_pci_nvlink2.c |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)
hw/ppc/spapr.c +9 -2
index 88b4a1f17716..1d9193d5ee49 100644
@@ -893,7 +893,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
-    uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t refpoints[] = {
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x2),
+    };
     uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
@@ -4544,7 +4548,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
-    smc->nr_assoc_refpoints = 2;
+    smc->nr_assoc_refpoints = 3;
     xfc->match_nvt = spapr_match_nvt;
 }
 
@@ -4611,8 +4615,11 @@ DEFINE_SPAPR_MACHINE(5_1, "5.1", true);
  */
 static void spapr_machine_5_0_class_options(MachineClass *mc)
 {
+    SpaprMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_5_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_5_0, hw_compat_5_0_len);
+    smc->nr_assoc_refpoints = 2;
 }
 
 DEFINE_SPAPR_MACHINE(5_0, "5.0", false);
hw/ppc/spapr_pci_nvlink2.c +0 -0
index 8332d5694e46..247fd48731e2 100644

hw/ppc/spapr_pci_nvlink2.c void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt) · line 365

Quoted 5 lines
hw/ppc/spapr_pci_nvlink2.c +1 -1
@@ -362,7 +362,7 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
         uint32_t associativity[] = {
             cpu_to_be32(0x4),
             SPAPR_GPU_NUMA_ID,
-            SPAPR_GPU_NUMA_ID,
+            cpu_to_be32(nvslot->numa_id),

This is a guest visible change. It should theoretically be controlled with a compat property of the PHB (look for "static GlobalProperty" in spapr.c). But since this code is only used for GPU passthrough and we don't support migration of such devices, I guess it's okay. Maybe just mention it in the changelog.

Quoted 3 lines
hw/ppc/spapr_pci_nvlink2.c +0 -0
             SPAPR_GPU_NUMA_ID,
             cpu_to_be32(nvslot->numa_id)
         };
#3 lore raw

Re: [PATCH v2 2/2] spapr: Add a new level of NUMA for GPUs

Quoted 89 lines · Greg Kurz
Quoted 86 lines · On Mon, 18 May 2020 16:44:18 -0500 Reza Arbab <[email protected]> wrote:

NUMA nodes corresponding to GPU memory currently have the same affinity/distance as normal memory nodes. Add a third NUMA associativity reference point enabling us to give GPU nodes more distance.

This is guest visible information, which shouldn't change under a running guest across migration between different qemu versions, so make the change effective only in new (pseries > 5.0) machine types.

Before, numactl -H output in a guest with 4 GPUs (nodes 2-5):

node distances:
node 0 1 2 3 4 5

  0:  10  40  40  40  40  40
  1:  40  10  40  40  40  40
  2:  40  40  10  40  40  40
  3:  40  40  40  10  40  40
  4:  40  40  40  40  10  40
  5:  40  40  40  40  40  10

After:

node distances:
node 0 1 2 3 4 5

  0:  10  40  80  80  80  80
  1:  40  10  80  80  80  80
  2:  80  80  10  80  80  80
  3:  80  80  80  10  80  80
  4:  80  80  80  80  10  80
  5:  80  80  80  80  80  10

These are the same distances as on the host, mirroring the change made to host firmware in skiboot commit f845a648b8cb ("numa/associativity:
Add a new level of NUMA for GPU's").

---
 hw/ppc/spapr.c             | 11 +++++++++--
 hw/ppc/spapr_pci_nvlink2.c |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)
hw/ppc/spapr.c +9 -2
index 88b4a1f17716..1d9193d5ee49 100644
@@ -893,7 +893,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
-    uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t refpoints[] = {
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x2),
+    };
     uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
@@ -4544,7 +4548,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
-    smc->nr_assoc_refpoints = 2;
+    smc->nr_assoc_refpoints = 3;
     xfc->match_nvt = spapr_match_nvt;
 }
 
@@ -4611,8 +4615,11 @@ DEFINE_SPAPR_MACHINE(5_1, "5.1", true);
  */
 static void spapr_machine_5_0_class_options(MachineClass *mc)
 {
+    SpaprMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_5_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_5_0, hw_compat_5_0_len);
+    smc->nr_assoc_refpoints = 2;
 }
 
 DEFINE_SPAPR_MACHINE(5_0, "5.0", false);
hw/ppc/spapr_pci_nvlink2.c +0 -0
index 8332d5694e46..247fd48731e2 100644
@@ -362,7 +362,7 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
         uint32_t associativity[] = {
             cpu_to_be32(0x4),
             SPAPR_GPU_NUMA_ID,
Quoted 8 lines

hw/ppc/spapr_pci_nvlink2.c void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt) · line 365

Quoted 2 lines
hw/ppc/spapr_pci_nvlink2.c +1 -1
-            SPAPR_GPU_NUMA_ID,
+            cpu_to_be32(nvslot->numa_id),

This is a guest visible change. It should theoretically be controlled with a compat property of the PHB (look for "static GlobalProperty" in spapr.c). But since this code is only used for GPU passthrough and we don't support migration of such devices, I guess it's okay. Maybe just mention it in the changelog.

Yeah, we might get away with it, but it should be too hard to get this right, so let's do it.

Quoted 5 lines
Quoted 4 lines
hw/ppc/spapr_pci_nvlink2.c +0 -0
             SPAPR_GPU_NUMA_ID,
             cpu_to_be32(nvslot->numa_id)
         };
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#4 lore raw

Re: [PATCH v2 2/2] spapr: Add a new level of NUMA for GPUs

Quoted 99 lines · On Thu, 21 May 2020 15:13:45 +1000 David Gibson <[email protected]>…
Quoted 89 lines · Greg Kurz
Quoted 86 lines · On Mon, 18 May 2020 16:44:18 -0500 Reza Arbab <[email protected]> wrote:

NUMA nodes corresponding to GPU memory currently have the same affinity/distance as normal memory nodes. Add a third NUMA associativity reference point enabling us to give GPU nodes more distance.

This is guest visible information, which shouldn't change under a running guest across migration between different qemu versions, so make the change effective only in new (pseries > 5.0) machine types.

Before, numactl -H output in a guest with 4 GPUs (nodes 2-5):

node distances:
node 0 1 2 3 4 5

  0:  10  40  40  40  40  40
  1:  40  10  40  40  40  40
  2:  40  40  10  40  40  40
  3:  40  40  40  10  40  40
  4:  40  40  40  40  10  40
  5:  40  40  40  40  40  10

After:

node distances:
node 0 1 2 3 4 5

  0:  10  40  80  80  80  80
  1:  40  10  80  80  80  80
  2:  80  80  10  80  80  80
  3:  80  80  80  10  80  80
  4:  80  80  80  80  10  80
  5:  80  80  80  80  80  10

These are the same distances as on the host, mirroring the change made to host firmware in skiboot commit f845a648b8cb ("numa/associativity:
Add a new level of NUMA for GPU's").

---
 hw/ppc/spapr.c             | 11 +++++++++--
 hw/ppc/spapr_pci_nvlink2.c |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)
hw/ppc/spapr.c +9 -2
index 88b4a1f17716..1d9193d5ee49 100644
@@ -893,7 +893,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
-    uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t refpoints[] = {
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x4),
+        cpu_to_be32(0x2),
+    };
     uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
@@ -4544,7 +4548,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
-    smc->nr_assoc_refpoints = 2;
+    smc->nr_assoc_refpoints = 3;
     xfc->match_nvt = spapr_match_nvt;
 }
 
@@ -4611,8 +4615,11 @@ DEFINE_SPAPR_MACHINE(5_1, "5.1", true);
  */
 static void spapr_machine_5_0_class_options(MachineClass *mc)
 {
+    SpaprMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_5_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_5_0, hw_compat_5_0_len);
+    smc->nr_assoc_refpoints = 2;
 }
 
 DEFINE_SPAPR_MACHINE(5_0, "5.0", false);
hw/ppc/spapr_pci_nvlink2.c +0 -0
index 8332d5694e46..247fd48731e2 100644
@@ -362,7 +362,7 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
         uint32_t associativity[] = {
             cpu_to_be32(0x4),
             SPAPR_GPU_NUMA_ID,
Quoted 8 lines

hw/ppc/spapr_pci_nvlink2.c void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt) · line 365

Quoted 2 lines
hw/ppc/spapr_pci_nvlink2.c +1 -1
-            SPAPR_GPU_NUMA_ID,
+            cpu_to_be32(nvslot->numa_id),

This is a guest visible change. It should theoretically be controlled with a compat property of the PHB (look for "static GlobalProperty" in spapr.c). But since this code is only used for GPU passthrough and we don't support migration of such devices, I guess it's okay. Maybe just mention it in the changelog.

Quoted 1 line

Yeah, we might get away with it, but it should be too hard to get this

I guess you mean "it shouldn't be too hard" ?

Quoted 8 lines

right, so let's do it.

Quoted 5 lines
Quoted 4 lines
hw/ppc/spapr_pci_nvlink2.c +0 -0
             SPAPR_GPU_NUMA_ID,
             cpu_to_be32(nvslot->numa_id)
         };
#5 lore raw
In reply to Reza Arbab
Quoted 28 lines · On Mon, 18 May 2020 16:44:17 -0500 Reza Arbab <[email protected]> wrote:

Make the number of NUMA associativity reference points a machine-specific value, using the currently assumed default (two reference points). This preps the next patch to conditionally change it.

---
 hw/ppc/spapr.c         | 6 +++++-
 include/hw/ppc/spapr.h | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +2 -0
index c18eab0a2305..88b4a1f17716 100644
@@ -889,10 +889,12 @@ static int spapr_dt_rng(void *fdt)
 static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
 {
     MachineState *ms = MACHINE(spapr);
+    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(ms);
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
     uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
     uint32_t lrdr_capacity[] = {

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 949

Quoted 4 lines
hw/ppc/spapr.c +1 -0
@@ -944,8 +946,9 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
                      qemu_hypertas->str, qemu_hypertas->len));
     g_string_free(qemu_hypertas, TRUE);
 
+    nr_refpoints = MIN(smc->nr_assoc_refpoints, ARRAY_SIZE(refpoints));

Having the machine requesting more reference points than available would clearly be a bug. I'd rather add an assert() than silently clipping to the size of refpoints[].

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 952

Quoted 4 lines
hw/ppc/spapr.c +1 -1
     _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
-                     refpoints, sizeof(refpoints)));
+                     refpoints, nr_refpoints * sizeof(uint32_t)));
 

Size can be expressed without yet another explicit reference to the uint32_t type:

nr_refpoints * sizeof(refpoints[0])

hw/ppc/spapr.c static void spapr_machine_class_init(ObjectClass *oc, void *data) · line 4547

Quoted 7 lines
hw/ppc/spapr.c +1 -0
     _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains",
                      maxdomains, sizeof(maxdomains)));
@@ -4541,6 +4544,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
+    smc->nr_assoc_refpoints = 2;

When adding a new setting for the default machine type, we usually take care of older machine types at the same time, ie. folding this patch into the next one. Both patches are simple enough that it should be okay and this would avoid this line to be touched again.

Quoted 15 lines
hw/ppc/spapr.c +0 -0
     xfc->match_nvt = spapr_match_nvt;
 }
 
include/hw/ppc/spapr.h +1 -0
index e579eaf28c05..abaf9a92adc0 100644
@@ -129,6 +129,7 @@ struct SpaprMachineClass {
     bool linux_pci_probe;
     bool smp_threads_vsmt; /* set VSMT to smp_threads by default */
     hwaddr rma_limit;          /* clamp the RMA to this size */
+    uint32_t nr_assoc_refpoints;
 
     void (*phb_placement)(SpaprMachineState *spapr, uint32_t index,
                           uint64_t *buid, hwaddr *pio, 
#6 lore raw
Quoted 34 lines · Greg Kurz
Quoted 31 lines · On Mon, 18 May 2020 16:44:17 -0500 Reza Arbab <[email protected]> wrote:

Make the number of NUMA associativity reference points a machine-specific value, using the currently assumed default (two reference points). This preps the next patch to conditionally change it.

---
 hw/ppc/spapr.c         | 6 +++++-
 include/hw/ppc/spapr.h | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +2 -0
index c18eab0a2305..88b4a1f17716 100644
@@ -889,10 +889,12 @@ static int spapr_dt_rng(void *fdt)
 static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
 {
     MachineState *ms = MACHINE(spapr);
+    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(ms);
     int rtas;
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
     uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x4) };
+    uint32_t nr_refpoints;
     uint64_t max_device_addr = MACHINE(spapr)->device_memory->base +
         memory_region_size(&MACHINE(spapr)->device_memory->mr);
     uint32_t lrdr_capacity[] = {
@@ -944,8 +946,9 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
                      qemu_hypertas->str, qemu_hypertas->len));
     g_string_free(qemu_hypertas, TRUE);
 
Quoted 5 lines

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 949

Quoted 1 line
hw/ppc/spapr.c +1 -0
+    nr_refpoints = MIN(smc->nr_assoc_refpoints, ARRAY_SIZE(refpoints));

Having the machine requesting more reference points than available would clearly be a bug. I'd rather add an assert() than silently clipping to the size of refpoints[].

Actually, I think this "num reference points" thing is a false abstraction. It's selecting a number of entries from a list of reference points that's fixed. The number of things we could do simply by changing the machine property and not the array is pretty small.

I think it would be simpler to just have a boolean in the machine class.

Quoted 39 lines

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 952

Quoted 4 lines
hw/ppc/spapr.c +1 -1
     _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
-                     refpoints, sizeof(refpoints)));
+                     refpoints, nr_refpoints * sizeof(uint32_t)));
 

Size can be expressed without yet another explicit reference to the uint32_t type:

nr_refpoints * sizeof(refpoints[0])

hw/ppc/spapr.c static void spapr_machine_class_init(ObjectClass *oc, void *data) · line 4547

Quoted 7 lines
hw/ppc/spapr.c +1 -0
     _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains",
                      maxdomains, sizeof(maxdomains)));
@@ -4541,6 +4544,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
+    smc->nr_assoc_refpoints = 2;

When adding a new setting for the default machine type, we usually take care of older machine types at the same time, ie. folding this patch into the next one. Both patches are simple enough that it should be okay and this would avoid this line to be touched again.

Quoted 15 lines
hw/ppc/spapr.c +0 -0
     xfc->match_nvt = spapr_match_nvt;
 }
 
include/hw/ppc/spapr.h +1 -0
index e579eaf28c05..abaf9a92adc0 100644
@@ -129,6 +129,7 @@ struct SpaprMachineClass {
     bool linux_pci_probe;
     bool smp_threads_vsmt; /* set VSMT to smp_threads by default */
     hwaddr rma_limit;          /* clamp the RMA to this size */
+    uint32_t nr_assoc_refpoints;
 
     void (*phb_placement)(SpaprMachineState *spapr, uint32_t index,
                           uint64_t *buid, hwaddr *pio, 
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
author
#7 lore raw
In reply to Greg Kurz
Quoted 11 lines · Greg Kurz

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 949

Quoted 5 lines · On Mon, 18 May 2020 16:44:17 -0500 Reza Arbab <[email protected]> wrote:
hw/ppc/spapr.c +1 -0
@@ -944,8 +946,9 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt)
                      qemu_hypertas->str, qemu_hypertas->len));
     g_string_free(qemu_hypertas, TRUE);
 
+    nr_refpoints = MIN(smc->nr_assoc_refpoints, ARRAY_SIZE(refpoints));

Having the machine requesting more reference points than available would clearly be a bug. I'd rather add an assert() than silently clipping to the size of refpoints[].

I'll rework nr_assoc_refpoints into a bool as David suggested. Struggling for a name at the moment, but I'll think on it.

Quoted 9 lines

hw/ppc/spapr.c static void spapr_dt_rtas(SpaprMachineState *spapr, void *fdt) · line 952

Quoted 4 lines
hw/ppc/spapr.c +1 -1
     _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
-                     refpoints, sizeof(refpoints)));
+                     refpoints, nr_refpoints * sizeof(uint32_t)));
 

Size can be expressed without yet another explicit reference to the uint32_t type:

nr_refpoints * sizeof(refpoints[0])

Will do.

Quoted 10 lines

hw/ppc/spapr.c static void spapr_machine_class_init(ObjectClass *oc, void *data) · line 4547

Quoted 5 lines
hw/ppc/spapr.c +1 -0
@@ -4541,6 +4544,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     smc->linux_pci_probe = true;
     smc->smp_threads_vsmt = true;
     smc->nr_xirqs = SPAPR_NR_XIRQS;
+    smc->nr_assoc_refpoints = 2;

When adding a new setting for the default machine type, we usually take care of older machine types at the same time, ie. folding this patch into the next one. Both patches are simple enough that it should be okay and this would avoid this line to be touched again.

No problem. I'll squash the patches together and work in that PHB compat property as well.

Thanks for your feedback!

Signature · 1 line
Reza Arbab