Skip to contentSkip to conversation

numa, spapr: add thread-id in the possible_cpus list

RFC In review Laurent Vivier 3 participants 10 messages Feb 12, 2019 – Feb 13, 2019

lore
Share & tools
lore
#0 lore raw

cover numa, spapr: add thread-id in the possible_cpus list

There are inconsistencies between the command line using "-numa node,cpus=XX" and what is checked internally:
the XX is supposed to be a CPU number, but for SPAPR it's taken as a core number, ignoring the threads. (See the description message of PATCH 1 for more details)

This series fixes this problem by introducing the threads in the possible_cpus list instead of only the cores. To avoid inconsistent topology, it doesn't allow anymore to have an incomplete CPU NUMA config on the command line (there was already a message announcing it will be absoleted for 2 years).

Laurent Vivier (4):
numa,spapr: add thread-id in the possible_cpus list
numa: exit on incomplete CPU mapping
numa: move cpu_slot_to_string() upper in the function
numa: check threads of the same core are on the same node

 hw/core/machine.c | 115 ++++++++++++++++++++++++++--------------------
 hw/ppc/spapr.c    |  33 ++++++-------
 tests/numa-test.c |  24 +---------
 3 files changed, 81 insertions(+), 91 deletions(-)
Signature · 1 line
2.20.1
#1 lore raw

RFC 1/4 numa, spapr: add thread-id in the possible_cpus list

spapr_possible_cpu_arch_ids() counts only cores, and so the number of available CPUs is the number of vCPU divided by smp_threads.

... -smp 4,maxcpus=8,cores=2,threads=2,sockets=2 -numa node,cpus=0,cpus=1 \
-numa node,cpus=3,cpus=4 \
-numa node -numa node

This generates (info hotpluggable-cpus)

node-id: 0 core-id: 0 thread-id: 0 [thread-id: 1]
node-id: 0 core-id: 6 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 2 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 4 thread-id: 0 [thread-id: 1]

And this command line generates the following error:

CPU(s) not present in any NUMA nodes: CPU 3 [core-id: 6]

That is wrong because CPU 3 [core-id: 6] is assigned to node-id 0 Moreover "cpus=4" is not valid, because it means core-id 8 but maxcpus is 8.

With this patch we have now:

node-id: 0 core-id: 0 thread-id: 0
node-id: 0 core-id: 0 thread-id: 1
node-id: 0 core-id: 1 thread-id: 0
node-id: 1 core-id: 1 thread-id: 1
node-id: 0 core-id: 2 thread-id: 1
node-id: 1 core-id: 2 thread-id: 0
node-id: 0 core-id: 3 thread-id: 1
node-id: 0 core-id: 3 thread-id: 0

CPUs 0 (core-id: 0 thread-id: 0) and 1 (core-id: 0 thread-id: 1) are correctly assigned to node-id 0, CPUs 3 (core-id: 1 thread-id: 1) and
4 (core-id: 2 thread-id: 0) are correctly assigned to node-id 1.
All other CPUs are assigned to node-id 0 by default.

And the error message is also correct:

  CPU(s) not present in any NUMA nodes: CPU 2 [core-id: 1, thread-id: 0], \
                                        CPU 5 [core-id: 2, thread-id: 1], \
                                        CPU 6 [core-id: 3, thread-id: 0], \
                                        CPU 7 [core-id: 3, thread-id: 1]
Fixes ec78f8114bc4 ("numa: use possible_cpus for not mapped CPUs check")

Before commit ec78f8114bc4, output was correct:

CPU(s) not present in any NUMA nodes: 2 5 6 7

---
 hw/ppc/spapr.c | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)
hw/ppc/spapr.c +13 -20
index 332cba89d425..7196ba09da34 100644
@@ -2404,15 +2404,13 @@ static void spapr_validate_node_memory(MachineState *machine, Error **errp)
 /* find cpu slot in machine->possible_cpus by core_id */
 static CPUArchId *spapr_find_cpu_slot(MachineState *ms, uint32_t id, int *idx)
 {
-    int index = id / smp_threads;
-
-    if (index >= ms->possible_cpus->len) {
+    if (id >= ms->possible_cpus->len) {
         return NULL;
     }
     if (idx) {
-        *idx = index;
+        *idx = id;
     }
-    return &ms->possible_cpus->cpus[index];
+    return &ms->possible_cpus->cpus[id];
 }
 
 static void spapr_set_vsmt_mode(sPAPRMachineState *spapr, Error **errp)
@@ -2514,7 +2512,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
             error_report("This machine version does not support CPU hotplug");
             exit(1);
         }
-        boot_cores_nr = possible_cpus->len;
+        boot_cores_nr = possible_cpus->len / smp_threads;
     }
 
     if (smc->pre_2_10_has_unused_icps) {
@@ -2528,7 +2526,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
         }
     }
 
-    for (i = 0; i < possible_cpus->len; i++) {
+    for (i = 0; i < possible_cpus->len / smp_threads; i++) {
         int core_id = i * smp_threads;
 
         if (mc->has_hotpluggable_cpus) {
@@ -3795,21 +3793,16 @@ spapr_cpu_index_to_props(MachineState *machine, unsigned cpu_index)
 
 static int64_t spapr_get_default_cpu_node_id(const MachineState *ms, int idx)
 {
-    return idx / smp_cores % nb_numa_nodes;
+    return idx / (smp_cores * smp_threads) % nb_numa_nodes;
 }
 
 static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
 {
     int i;
     const char *core_type;
-    int spapr_max_cores = max_cpus / smp_threads;
-    MachineClass *mc = MACHINE_GET_CLASS(machine);
 
-    if (!mc->has_hotpluggable_cpus) {
-        spapr_max_cores = QEMU_ALIGN_UP(smp_cpus, smp_threads) / smp_threads;
-    }
     if (machine->possible_cpus) {
-        assert(machine->possible_cpus->len == spapr_max_cores);
+        assert(machine->possible_cpus->len == max_cpus);
         return machine->possible_cpus;
     }
 
@@ -3820,16 +3813,16 @@ static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
     }
 
     machine->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
-                             sizeof(CPUArchId) * spapr_max_cores);
-    machine->possible_cpus->len = spapr_max_cores;
+                             sizeof(CPUArchId) * max_cpus);
+    machine->possible_cpus->len = max_cpus;
     for (i = 0; i < machine->possible_cpus->len; i++) {
-        int core_id = i * smp_threads;
-
         machine->possible_cpus->cpus[i].type = core_type;
         machine->possible_cpus->cpus[i].vcpus_count = smp_threads;
-        machine->possible_cpus->cpus[i].arch_id = core_id;
+        machine->possible_cpus->cpus[i].arch_id = i;
         machine->possible_cpus->cpus[i].props.has_core_id = true;
-        machine->possible_cpus->cpus[i].props.core_id = core_id;
+        machine->possible_cpus->cpus[i].props.core_id = i / smp_threads;
+        machine->possible_cpus->cpus[i].props.has_thread_id = true;
+        machine->possible_cpus->cpus[i].props.thread_id = i % smp_threads;
     }
     return machine->possible_cpus;
 }
Signature · 1 line
2.20.1
#2 lore raw
Quoted 25 lines · Laurent Vivier

spapr_possible_cpu_arch_ids() counts only cores, and so the number of available CPUs is the number of vCPU divided by smp_threads.

... -smp 4,maxcpus=8,cores=2,threads=2,sockets=2 -numa node,cpus=0,cpus=1 \
-numa node,cpus=3,cpus=4 \
-numa node -numa node

This generates (info hotpluggable-cpus)

node-id: 0 core-id: 0 thread-id: 0 [thread-id: 1]
node-id: 0 core-id: 6 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 2 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 4 thread-id: 0 [thread-id: 1]

And this command line generates the following error:

CPU(s) not present in any NUMA nodes: CPU 3 [core-id: 6]

That is wrong because CPU 3 [core-id: 6] is assigned to node-id 0 Moreover "cpus=4" is not valid, because it means core-id 8 but maxcpus is 8.

With this patch we have now:

Quoted 8 lines

node-id: 0 core-id: 0 thread-id: 0
node-id: 0 core-id: 0 thread-id: 1
node-id: 0 core-id: 1 thread-id: 0
node-id: 1 core-id: 1 thread-id: 1
node-id: 0 core-id: 2 thread-id: 1
node-id: 1 core-id: 2 thread-id: 0
node-id: 0 core-id: 3 thread-id: 1
node-id: 0 core-id: 3 thread-id: 0

I'm afraid this is not the right solution. The point of the hotpluggable cpus table is that it has exactly one entry for each hotpluggable unit. For PAPR that's a core, not a thread.

So, the problem is with how the NUMA configuration code is interpreting possible-cpus, not how the machine is building the table.

Quoted 112 lines

CPUs 0 (core-id: 0 thread-id: 0) and 1 (core-id: 0 thread-id: 1) are correctly assigned to node-id 0, CPUs 3 (core-id: 1 thread-id: 1) and
4 (core-id: 2 thread-id: 0) are correctly assigned to node-id 1.
All other CPUs are assigned to node-id 0 by default.

And the error message is also correct:

  CPU(s) not present in any NUMA nodes: CPU 2 [core-id: 1, thread-id: 0], \
                                        CPU 5 [core-id: 2, thread-id: 1], \
                                        CPU 6 [core-id: 3, thread-id: 0], \
                                        CPU 7 [core-id: 3, thread-id: 1]
Fixes ec78f8114bc4 ("numa: use possible_cpus for not mapped CPUs check")

Before commit ec78f8114bc4, output was correct:

CPU(s) not present in any NUMA nodes: 2 5 6 7

---
 hw/ppc/spapr.c | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)
hw/ppc/spapr.c +13 -20
index 332cba89d425..7196ba09da34 100644
@@ -2404,15 +2404,13 @@ static void spapr_validate_node_memory(MachineState *machine, Error **errp)
 /* find cpu slot in machine->possible_cpus by core_id */
 static CPUArchId *spapr_find_cpu_slot(MachineState *ms, uint32_t id, int *idx)
 {
-    int index = id / smp_threads;
-
-    if (index >= ms->possible_cpus->len) {
+    if (id >= ms->possible_cpus->len) {
         return NULL;
     }
     if (idx) {
-        *idx = index;
+        *idx = id;
     }
-    return &ms->possible_cpus->cpus[index];
+    return &ms->possible_cpus->cpus[id];
 }
 
 static void spapr_set_vsmt_mode(sPAPRMachineState *spapr, Error **errp)
@@ -2514,7 +2512,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
             error_report("This machine version does not support CPU hotplug");
             exit(1);
         }
-        boot_cores_nr = possible_cpus->len;
+        boot_cores_nr = possible_cpus->len / smp_threads;
     }
 
     if (smc->pre_2_10_has_unused_icps) {
@@ -2528,7 +2526,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
         }
     }
 
-    for (i = 0; i < possible_cpus->len; i++) {
+    for (i = 0; i < possible_cpus->len / smp_threads; i++) {
         int core_id = i * smp_threads;
 
         if (mc->has_hotpluggable_cpus) {
@@ -3795,21 +3793,16 @@ spapr_cpu_index_to_props(MachineState *machine, unsigned cpu_index)
 
 static int64_t spapr_get_default_cpu_node_id(const MachineState *ms, int idx)
 {
-    return idx / smp_cores % nb_numa_nodes;
+    return idx / (smp_cores * smp_threads) % nb_numa_nodes;
 }
 
 static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
 {
     int i;
     const char *core_type;
-    int spapr_max_cores = max_cpus / smp_threads;
-    MachineClass *mc = MACHINE_GET_CLASS(machine);
 
-    if (!mc->has_hotpluggable_cpus) {
-        spapr_max_cores = QEMU_ALIGN_UP(smp_cpus, smp_threads) / smp_threads;
-    }
     if (machine->possible_cpus) {
-        assert(machine->possible_cpus->len == spapr_max_cores);
+        assert(machine->possible_cpus->len == max_cpus);
         return machine->possible_cpus;
     }
 
@@ -3820,16 +3813,16 @@ static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
     }
 
     machine->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
-                             sizeof(CPUArchId) * spapr_max_cores);
-    machine->possible_cpus->len = spapr_max_cores;
+                             sizeof(CPUArchId) * max_cpus);
+    machine->possible_cpus->len = max_cpus;
     for (i = 0; i < machine->possible_cpus->len; i++) {
-        int core_id = i * smp_threads;
-
         machine->possible_cpus->cpus[i].type = core_type;
         machine->possible_cpus->cpus[i].vcpus_count = smp_threads;
-        machine->possible_cpus->cpus[i].arch_id = core_id;
+        machine->possible_cpus->cpus[i].arch_id = i;
         machine->possible_cpus->cpus[i].props.has_core_id = true;
-        machine->possible_cpus->cpus[i].props.core_id = core_id;
+        machine->possible_cpus->cpus[i].props.core_id = i / smp_threads;
+        machine->possible_cpus->cpus[i].props.has_thread_id = true;
+        machine->possible_cpus->cpus[i].props.thread_id = i % smp_threads;
     }
     return machine->possible_cpus;
 }
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#3 lore raw
Quoted 35 lines · On Wed, 13 Feb 2019 12:25:45 +1100 David Gibson <[email protected]>…
Quoted 25 lines · Laurent Vivier

spapr_possible_cpu_arch_ids() counts only cores, and so the number of available CPUs is the number of vCPU divided by smp_threads.

... -smp 4,maxcpus=8,cores=2,threads=2,sockets=2 -numa node,cpus=0,cpus=1 \
-numa node,cpus=3,cpus=4 \
-numa node -numa node

This generates (info hotpluggable-cpus)

node-id: 0 core-id: 0 thread-id: 0 [thread-id: 1]
node-id: 0 core-id: 6 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 2 thread-id: 0 [thread-id: 1]
node-id: 1 core-id: 4 thread-id: 0 [thread-id: 1]

And this command line generates the following error:

CPU(s) not present in any NUMA nodes: CPU 3 [core-id: 6]

That is wrong because CPU 3 [core-id: 6] is assigned to node-id 0 Moreover "cpus=4" is not valid, because it means core-id 8 but maxcpus is 8.

With this patch we have now:

Quoted 8 lines

node-id: 0 core-id: 0 thread-id: 0
node-id: 0 core-id: 0 thread-id: 1
node-id: 0 core-id: 1 thread-id: 0
node-id: 1 core-id: 1 thread-id: 1
node-id: 0 core-id: 2 thread-id: 1
node-id: 1 core-id: 2 thread-id: 0
node-id: 0 core-id: 3 thread-id: 1
node-id: 0 core-id: 3 thread-id: 0

Quoted 6 lines

I'm afraid this is not the right solution. The point of the hotpluggable cpus table is that it has exactly one entry for each hotpluggable unit. For PAPR that's a core, not a thread.

So, the problem is with how the NUMA configuration code is interpreting possible-cpus, not how the machine is building the table.

I'd suggest to deprecate/remove 'cpus' suboption in -numa. One should use '-numa cpu' instead, which is written with possible_cpus in mind.

Quoted 114 lines
Quoted 113 lines

CPUs 0 (core-id: 0 thread-id: 0) and 1 (core-id: 0 thread-id: 1) are correctly assigned to node-id 0, CPUs 3 (core-id: 1 thread-id: 1) and
4 (core-id: 2 thread-id: 0) are correctly assigned to node-id 1.
All other CPUs are assigned to node-id 0 by default.

And the error message is also correct:

  CPU(s) not present in any NUMA nodes: CPU 2 [core-id: 1, thread-id: 0], \
                                        CPU 5 [core-id: 2, thread-id: 1], \
                                        CPU 6 [core-id: 3, thread-id: 0], \
                                        CPU 7 [core-id: 3, thread-id: 1]
Fixes ec78f8114bc4 ("numa: use possible_cpus for not mapped CPUs check")

Before commit ec78f8114bc4, output was correct:

CPU(s) not present in any NUMA nodes: 2 5 6 7

---
 hw/ppc/spapr.c | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)
hw/ppc/spapr.c +13 -20
index 332cba89d425..7196ba09da34 100644
@@ -2404,15 +2404,13 @@ static void spapr_validate_node_memory(MachineState *machine, Error **errp)
 /* find cpu slot in machine->possible_cpus by core_id */
 static CPUArchId *spapr_find_cpu_slot(MachineState *ms, uint32_t id, int *idx)
 {
-    int index = id / smp_threads;
-
-    if (index >= ms->possible_cpus->len) {
+    if (id >= ms->possible_cpus->len) {
         return NULL;
     }
     if (idx) {
-        *idx = index;
+        *idx = id;
     }
-    return &ms->possible_cpus->cpus[index];
+    return &ms->possible_cpus->cpus[id];
 }
 
 static void spapr_set_vsmt_mode(sPAPRMachineState *spapr, Error **errp)
@@ -2514,7 +2512,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
             error_report("This machine version does not support CPU hotplug");
             exit(1);
         }
-        boot_cores_nr = possible_cpus->len;
+        boot_cores_nr = possible_cpus->len / smp_threads;
     }
 
     if (smc->pre_2_10_has_unused_icps) {
@@ -2528,7 +2526,7 @@ static void spapr_init_cpus(sPAPRMachineState *spapr)
         }
     }
 
-    for (i = 0; i < possible_cpus->len; i++) {
+    for (i = 0; i < possible_cpus->len / smp_threads; i++) {
         int core_id = i * smp_threads;
 
         if (mc->has_hotpluggable_cpus) {
@@ -3795,21 +3793,16 @@ spapr_cpu_index_to_props(MachineState *machine, unsigned cpu_index)
 
 static int64_t spapr_get_default_cpu_node_id(const MachineState *ms, int idx)
 {
-    return idx / smp_cores % nb_numa_nodes;
+    return idx / (smp_cores * smp_threads) % nb_numa_nodes;
 }
 
 static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
 {
     int i;
     const char *core_type;
-    int spapr_max_cores = max_cpus / smp_threads;
-    MachineClass *mc = MACHINE_GET_CLASS(machine);
 
-    if (!mc->has_hotpluggable_cpus) {
-        spapr_max_cores = QEMU_ALIGN_UP(smp_cpus, smp_threads) / smp_threads;
-    }
     if (machine->possible_cpus) {
-        assert(machine->possible_cpus->len == spapr_max_cores);
+        assert(machine->possible_cpus->len == max_cpus);
         return machine->possible_cpus;
     }
 
@@ -3820,16 +3813,16 @@ static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
     }
 
     machine->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
-                             sizeof(CPUArchId) * spapr_max_cores);
-    machine->possible_cpus->len = spapr_max_cores;
+                             sizeof(CPUArchId) * max_cpus);
+    machine->possible_cpus->len = max_cpus;
     for (i = 0; i < machine->possible_cpus->len; i++) {
-        int core_id = i * smp_threads;
-
         machine->possible_cpus->cpus[i].type = core_type;
         machine->possible_cpus->cpus[i].vcpus_count = smp_threads;
-        machine->possible_cpus->cpus[i].arch_id = core_id;
+        machine->possible_cpus->cpus[i].arch_id = i;
         machine->possible_cpus->cpus[i].props.has_core_id = true;
-        machine->possible_cpus->cpus[i].props.core_id = core_id;
+        machine->possible_cpus->cpus[i].props.core_id = i / smp_threads;
+        machine->possible_cpus->cpus[i].props.has_thread_id = true;
+        machine->possible_cpus->cpus[i].props.thread_id = i % smp_threads;
     }
     return machine->possible_cpus;
 }  
#4 lore raw
Quoted 42 lines · Igor Mammedov
Quoted 39 lines · On Wed, 13 Feb 2019 12:25:45 +1100 David Gibson <[email protected]>…
Quoted 25 lines · Laurent Vivier

spapr_possible_cpu_arch_ids() counts only cores, and so the number of available CPUs is the number of vCPU divided by smp_threads.

... -smp 4,maxcpus=8,cores=2,threads=2,sockets=2 -numa node,cpus=0,cpus=1 \
-numa node,cpus=3,cpus=4 \
-numa node -numa node

This generates (info hotpluggable-cpus)

   node-id: 0 core-id: 0 thread-id: 0 [thread-id: 1]
   node-id: 0 core-id: 6 thread-id: 0 [thread-id: 1]
   node-id: 1 core-id: 2 thread-id: 0 [thread-id: 1]
   node-id: 1 core-id: 4 thread-id: 0 [thread-id: 1]

And this command line generates the following error:

CPU(s) not present in any NUMA nodes: CPU 3 [core-id: 6]

That is wrong because CPU 3 [core-id: 6] is assigned to node-id 0 Moreover "cpus=4" is not valid, because it means core-id 8 but maxcpus is 8.

With this patch we have now:

Quoted 8 lines
   node-id: 0 core-id: 0 thread-id: 0
   node-id: 0 core-id: 0 thread-id: 1
   node-id: 0 core-id: 1 thread-id: 0
   node-id: 1 core-id: 1 thread-id: 1
   node-id: 0 core-id: 2 thread-id: 1
   node-id: 1 core-id: 2 thread-id: 0
   node-id: 0 core-id: 3 thread-id: 1
   node-id: 0 core-id: 3 thread-id: 0

I'm afraid this is not the right solution. The point of the hotpluggable cpus table is that it has exactly one entry for each hotpluggable unit. For PAPR that's a core, not a thread.

Quoted 6 lines
Quoted 2 lines

So, the problem is with how the NUMA configuration code is interpreting possible-cpus, not how the machine is building the table.

I'd suggest to deprecate/remove 'cpus' suboption in -numa. One should use '-numa cpu' instead, which is written with possible_cpus in mind.

I agree.

Should I keep the patch to remove the incomplete CPU mapping support?

Thanks,
Laurent

#5 lore raw
Quoted 46 lines · On Wed, 13 Feb 2019 10:08:24 +0100 Laurent Vivier <[email protected]> wrote:
Quoted 45 lines · Igor Mammedov
Quoted 41 lines · On Wed, 13 Feb 2019 12:25:45 +1100 David Gibson <[email protected]>…
Quoted 25 lines · Laurent Vivier

spapr_possible_cpu_arch_ids() counts only cores, and so the number of available CPUs is the number of vCPU divided by smp_threads.

... -smp 4,maxcpus=8,cores=2,threads=2,sockets=2 -numa node,cpus=0,cpus=1 \
-numa node,cpus=3,cpus=4 \
-numa node -numa node

This generates (info hotpluggable-cpus)

   node-id: 0 core-id: 0 thread-id: 0 [thread-id: 1]
   node-id: 0 core-id: 6 thread-id: 0 [thread-id: 1]
   node-id: 1 core-id: 2 thread-id: 0 [thread-id: 1]
   node-id: 1 core-id: 4 thread-id: 0 [thread-id: 1]

And this command line generates the following error:

CPU(s) not present in any NUMA nodes: CPU 3 [core-id: 6]

That is wrong because CPU 3 [core-id: 6] is assigned to node-id 0 Moreover "cpus=4" is not valid, because it means core-id 8 but maxcpus is 8.

With this patch we have now:

Quoted 8 lines
   node-id: 0 core-id: 0 thread-id: 0
   node-id: 0 core-id: 0 thread-id: 1
   node-id: 0 core-id: 1 thread-id: 0
   node-id: 1 core-id: 1 thread-id: 1
   node-id: 0 core-id: 2 thread-id: 1
   node-id: 1 core-id: 2 thread-id: 0
   node-id: 0 core-id: 3 thread-id: 1
   node-id: 0 core-id: 3 thread-id: 0  

I'm afraid this is not the right solution. The point of the hotpluggable cpus table is that it has exactly one entry for each hotpluggable unit. For PAPR that's a core, not a thread.

So, the problem is with how the NUMA configuration code is interpreting possible-cpus, not how the machine is building the table.

Quoted 7 lines
Quoted 3 lines

I'd suggest to deprecate/remove 'cpus' suboption in -numa. One should use '-numa cpu' instead, which is written with possible_cpus in mind.

I agree.

Should I keep the patch to remove the incomplete CPU mapping support?

Considering code path didn't go through deprecation process (we didn't have it back then), lets keep it for now and deprecate option properly first. Once deprecation period ends we well remove all related code at once.

Maybe amend affected/add error messages that it's been deprecated and will be removed soon and point out to replacement option.

Quoted 2 lines

Thanks,
Laurent

#6 lore raw

RFC 2/4 numa: exit on incomplete CPU mapping

In reply to Laurent Vivier

Change the existing message to an error and exit.

This message was a warning and comes with the information it will be removed in the future since May 10 2017 (ec78f8114bc4 "numa: use possible_cpus for not mapped CPUs check").

Update tests/numa-test to remove the incomplete CPU mapping test.

---
 hw/core/machine.c | 46 +++++++++++++++++++++-------------------------
 tests/numa-test.c | 20 --------------------
 2 files changed, 21 insertions(+), 45 deletions(-)
hw/core/machine.c +21 -25
index 077fbd182adf..7c74b318f635 100644
@@ -873,51 +873,47 @@ static char *cpu_slot_to_string(const CPUArchId *cpu)
 static void machine_numa_finish_cpu_init(MachineState *machine)
 {
     int i;
-    bool default_mapping;
+    bool default_mapping = true;
     GString *s = g_string_new(NULL);
     MachineClass *mc = MACHINE_GET_CLASS(machine);
     const CPUArchIdList *possible_cpus = mc->possible_cpu_arch_ids(machine);
+    const CPUArchId *cpu_slot;
 
     assert(nb_numa_nodes);
     for (i = 0; i < possible_cpus->len; i++) {
-        if (possible_cpus->cpus[i].props.has_node_id) {
-            break;
+        cpu_slot = &possible_cpus->cpus[i];
+
+        if (cpu_slot->props.has_node_id) {
+            default_mapping = false;
+        } else {
+            /* record slots with not set mapping, */
+            char *cpu_str = cpu_slot_to_string(cpu_slot);
+            g_string_append_printf(s, "%sCPU %d [%s]",
+                                   s->len ? ", " : "", i, cpu_str);
+            g_free(cpu_str);
         }
     }
-    default_mapping = (i == possible_cpus->len);
+    if (!default_mapping && s->len && !qtest_enabled()) {
+        error_report("CPU(s) not present in any NUMA nodes: %s", s->str);
+        error_report("All CPU(s) up to maxcpus must be described "
+                    "in NUMA config");
+        g_string_free(s, true);
+        exit(1);
+    }
+    g_string_free(s, true);
 
     for (i = 0; i < possible_cpus->len; i++) {
-        const CPUArchId *cpu_slot = &possible_cpus->cpus[i];
+        cpu_slot = &possible_cpus->cpus[i];
 
         if (!cpu_slot->props.has_node_id) {
             /* fetch default mapping from board and enable it */
             CpuInstanceProperties props = cpu_slot->props;
 
             props.node_id = mc->get_default_cpu_node_id(machine, i);
-            if (!default_mapping) {
-                /* record slots with not set mapping,
-                 * TODO: make it hard error in future */
-                char *cpu_str = cpu_slot_to_string(cpu_slot);
-                g_string_append_printf(s, "%sCPU %d [%s]",
-                                       s->len ? ", " : "", i, cpu_str);
-                g_free(cpu_str);
-
-                /* non mapped cpus used to fallback to node 0 */
-                props.node_id = 0;
-            }
-
             props.has_node_id = true;
             machine_set_cpu_numa_node(machine, &props, &error_fatal);
         }
     }
-    if (s->len && !qtest_enabled()) {
-        warn_report("CPU(s) not present in any NUMA nodes: %s",
-                    s->str);
-        warn_report("All CPU(s) up to maxcpus should be described "
-                    "in NUMA config, ability to start up with partial NUMA "
-                    "mappings is obsoleted and will be removed in future");
-    }
-    g_string_free(s, true);
 }
 
 void machine_run_board_init(MachineState *machine)
tests/numa-test.c +0 -20
index 9824fdd5875e..5280573fc992 100644
@@ -55,25 +55,6 @@ static void test_mon_default(const void *data)
     g_free(cli);
 }
 
-static void test_mon_partial(const void *data)
-{
-    char *s;
-    char *cli;
-
-    cli = make_cli(data, "-smp 8 "
-                   "-numa node,nodeid=0,cpus=0-1 "
-                   "-numa node,nodeid=1,cpus=4-5 ");
-    qtest_start(cli);
-
-    s = hmp("info numa");
-    g_assert(strstr(s, "node 0 cpus: 0 1 2 3 6 7"));
-    g_assert(strstr(s, "node 1 cpus: 4 5"));
-    g_free(s);
-
-    qtest_end();
-    g_free(cli);
-}
-
 static QList *get_cpus(QDict **resp)
 {
     *resp = qmp("{ 'execute': 'query-cpus' }");
@@ -333,7 +314,6 @@ int main(int argc, char **argv)
 
     qtest_add_data_func("/numa/mon/default", args, test_mon_default);
     qtest_add_data_func("/numa/mon/cpus/explicit", args, test_mon_explicit);
-    qtest_add_data_func("/numa/mon/cpus/partial", args, test_mon_partial);
     qtest_add_data_func("/numa/qmp/cpus/query-cpus", args, test_query_cpus);
 
     if (!strcmp(arch, "i386") || !strcmp(arch, "x86_64")) {
Signature · 1 line
2.20.1
#7 lore raw

RFC 3/4 numa: move cpu_slot_to_string() upper in the function

In reply to Laurent Vivier

This will allow to use it in more functions in the future.

As we change the prototype to take directly CpuInstanceProperties instead of CPUArchId, rename the function to cpu_props_to_string().

---
 hw/core/machine.c | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)
hw/core/machine.c +22 -22
index 7c74b318f635..a2c29692b55e 100644
@@ -550,6 +550,27 @@ HotpluggableCPUList *machine_query_hotpluggable_cpus(MachineState *machine)
     return head;
 }
 
+static char *cpu_props_to_string(const CpuInstanceProperties *props)
+{
+    GString *s = g_string_new(NULL);
+    if (props->has_socket_id) {
+        g_string_append_printf(s, "socket-id: %"PRId64, props->socket_id);
+    }
+    if (props->has_core_id) {
+        if (s->len) {
+            g_string_append_printf(s, ", ");
+        }
+        g_string_append_printf(s, "core-id: %"PRId64, props->core_id);
+    }
+    if (props->has_thread_id) {
+        if (s->len) {
+            g_string_append_printf(s, ", ");
+        }
+        g_string_append_printf(s, "thread-id: %"PRId64, props->thread_id);
+    }
+    return g_string_free(s, false);
+}
+
 /**
  * machine_set_cpu_numa_node:
  * @machine: machine object to modify
@@ -849,27 +870,6 @@ bool machine_mem_merge(MachineState *machine)
     return machine->mem_merge;
 }
 
-static char *cpu_slot_to_string(const CPUArchId *cpu)
-{
-    GString *s = g_string_new(NULL);
-    if (cpu->props.has_socket_id) {
-        g_string_append_printf(s, "socket-id: %"PRId64, cpu->props.socket_id);
-    }
-    if (cpu->props.has_core_id) {
-        if (s->len) {
-            g_string_append_printf(s, ", ");
-        }
-        g_string_append_printf(s, "core-id: %"PRId64, cpu->props.core_id);
-    }
-    if (cpu->props.has_thread_id) {
-        if (s->len) {
-            g_string_append_printf(s, ", ");
-        }
-        g_string_append_printf(s, "thread-id: %"PRId64, cpu->props.thread_id);
-    }
-    return g_string_free(s, false);
-}
-
 static void machine_numa_finish_cpu_init(MachineState *machine)
 {
     int i;
@@ -887,7 +887,7 @@ static void machine_numa_finish_cpu_init(MachineState *machine)
             default_mapping = false;
         } else {
             /* record slots with not set mapping, */
-            char *cpu_str = cpu_slot_to_string(cpu_slot);
+            char *cpu_str = cpu_props_to_string(&cpu_slot->props);
             g_string_append_printf(s, "%sCPU %d [%s]",
                                    s->len ? ", " : "", i, cpu_str);
             g_free(cpu_str);
Signature · 1 line
2.20.1
#8 lore raw

RFC 4/4 numa: check threads of the same core are on the same node

In reply to Laurent Vivier

A core cannot be split between two nodes.
To check if a thread of the same core has already been assigned to a node, this patch reverses the numa topology checking order and exits if the topology is not valid.

Update test/numa-test accordingly.

Fixes 722387e78daf ("spapr: get numa node mapping from possible_cpus instead of numa_get_node_for_cpu()")
---
 hw/core/machine.c | 27 ++++++++++++++++++++++++---
 tests/numa-test.c |  4 ++--
 2 files changed, 26 insertions(+), 5 deletions(-)
hw/core/machine.c +24 -3
index a2c29692b55e..c0a556b0dce7 100644
@@ -602,6 +602,7 @@ void machine_set_cpu_numa_node(MachineState *machine,
     MachineClass *mc = MACHINE_GET_CLASS(machine);
     bool match = false;
     int i;
+    const CpuInstanceProperties *previous_props = NULL;
 
     if (!mc->possible_cpu_arch_ids) {
         error_setg(errp, "mapping of CPUs to NUMA node is not supported");
@@ -634,18 +635,38 @@ void machine_set_cpu_numa_node(MachineState *machine,
         }
 
         /* skip slots with explicit mismatch */
-        if (props->has_thread_id && props->thread_id != slot->props.thread_id) {
+        if (props->has_socket_id && props->socket_id != slot->props.socket_id) {
                 continue;
         }
 
-        if (props->has_core_id && props->core_id != slot->props.core_id) {
+        if (props->has_core_id) {
+            if (props->core_id != slot->props.core_id) {
                 continue;
+            }
+            if (slot->props.has_node_id) {
+                /* we have a node where our core is already assigned */
+                previous_props = &slot->props;
+            }
         }
 
-        if (props->has_socket_id && props->socket_id != slot->props.socket_id) {
+        if (props->has_thread_id && props->thread_id != slot->props.thread_id) {
                 continue;
         }
 
+        /* check current thread matches node of the thread of the same core */
+        if (previous_props && previous_props->has_node_id &&
+            previous_props->node_id != props->node_id) {
+            char *cpu_str = cpu_props_to_string(props);
+            char *node_str = cpu_props_to_string(previous_props);
+            error_setg(errp,  "Invalid node-id=%"PRIu64" of [%s]: core-id "
+                              "[%s] is already assigned to node-id %"PRIu64,
+                              props->node_id, cpu_str,
+                              node_str, previous_props->node_id);
+            g_free(cpu_str);
+            g_free(node_str);
+            return;
+        }
+
         /* reject assignment if slot is already assigned, for compatibility
          * of legacy cpu_index mapping with SPAPR core based mapping do not
          * error out if cpu thread and matched core have the same node-id */
tests/numa-test.c +2 -2
index 5280573fc992..a7c3c5b4dee8 100644
@@ -112,7 +112,7 @@ static void pc_numa_cpu(const void *data)
         "-numa cpu,node-id=1,socket-id=0 "
         "-numa cpu,node-id=0,socket-id=1,core-id=0 "
         "-numa cpu,node-id=0,socket-id=1,core-id=1,thread-id=0 "
-        "-numa cpu,node-id=1,socket-id=1,core-id=1,thread-id=1");
+        "-numa cpu,node-id=0,socket-id=1,core-id=1,thread-id=1");
     qtest_start(cli);
     cpus = get_cpus(&resp);
     g_assert(cpus);
@@ -141,7 +141,7 @@ static void pc_numa_cpu(const void *data)
         } else if (socket == 1 && core == 1 && thread == 0) {
             g_assert_cmpint(node, ==, 0);
         } else if (socket == 1 && core == 1 && thread == 1) {
-            g_assert_cmpint(node, ==, 1);
+            g_assert_cmpint(node, ==, 0);
         } else {
             g_assert(false);
         }
Signature · 1 line
2.20.1
#9 lore raw

Re: [Qemu-devel] [RFC 4/4] numa: check threads of the same core are on the same node

Quoted 4 lines · Laurent Vivier

A core cannot be split between two nodes.
To check if a thread of the same core has already been assigned to a node, this patch reverses the numa topology checking order and exits if the topology is not valid.

I'm not entirely sure if this makes sense to enforce generically.

It's certainly true for PAPR - we have no way to represent threads with different NUMA nodes to the guest.

It probably makes sense for everything - the whole point of threading is to take better advantage of latencies accessing memory, so it seems implausible that the threads would have different paths to memory.

But... there are some pretty weird setups out there, so I'm not sure it's a good idea to enforce a restriction generically that's not actually inherent in the structure of the problem.

Quoted 87 lines

Update test/numa-test accordingly.

Fixes 722387e78daf ("spapr: get numa node mapping from possible_cpus instead of numa_get_node_for_cpu()")
---
 hw/core/machine.c | 27 ++++++++++++++++++++++++---
 tests/numa-test.c |  4 ++--
 2 files changed, 26 insertions(+), 5 deletions(-)
hw/core/machine.c +24 -3
index a2c29692b55e..c0a556b0dce7 100644
@@ -602,6 +602,7 @@ void machine_set_cpu_numa_node(MachineState *machine,
     MachineClass *mc = MACHINE_GET_CLASS(machine);
     bool match = false;
     int i;
+    const CpuInstanceProperties *previous_props = NULL;
 
     if (!mc->possible_cpu_arch_ids) {
         error_setg(errp, "mapping of CPUs to NUMA node is not supported");
@@ -634,18 +635,38 @@ void machine_set_cpu_numa_node(MachineState *machine,
         }
 
         /* skip slots with explicit mismatch */
-        if (props->has_thread_id && props->thread_id != slot->props.thread_id) {
+        if (props->has_socket_id && props->socket_id != slot->props.socket_id) {
                 continue;
         }
 
-        if (props->has_core_id && props->core_id != slot->props.core_id) {
+        if (props->has_core_id) {
+            if (props->core_id != slot->props.core_id) {
                 continue;
+            }
+            if (slot->props.has_node_id) {
+                /* we have a node where our core is already assigned */
+                previous_props = &slot->props;
+            }
         }
 
-        if (props->has_socket_id && props->socket_id != slot->props.socket_id) {
+        if (props->has_thread_id && props->thread_id != slot->props.thread_id) {
                 continue;
         }
 
+        /* check current thread matches node of the thread of the same core */
+        if (previous_props && previous_props->has_node_id &&
+            previous_props->node_id != props->node_id) {
+            char *cpu_str = cpu_props_to_string(props);
+            char *node_str = cpu_props_to_string(previous_props);
+            error_setg(errp,  "Invalid node-id=%"PRIu64" of [%s]: core-id "
+                              "[%s] is already assigned to node-id %"PRIu64,
+                              props->node_id, cpu_str,
+                              node_str, previous_props->node_id);
+            g_free(cpu_str);
+            g_free(node_str);
+            return;
+        }
+
         /* reject assignment if slot is already assigned, for compatibility
          * of legacy cpu_index mapping with SPAPR core based mapping do not
          * error out if cpu thread and matched core have the same node-id */
tests/numa-test.c +2 -2
index 5280573fc992..a7c3c5b4dee8 100644
@@ -112,7 +112,7 @@ static void pc_numa_cpu(const void *data)
         "-numa cpu,node-id=1,socket-id=0 "
         "-numa cpu,node-id=0,socket-id=1,core-id=0 "
         "-numa cpu,node-id=0,socket-id=1,core-id=1,thread-id=0 "
-        "-numa cpu,node-id=1,socket-id=1,core-id=1,thread-id=1");
+        "-numa cpu,node-id=0,socket-id=1,core-id=1,thread-id=1");
     qtest_start(cli);
     cpus = get_cpus(&resp);
     g_assert(cpus);
@@ -141,7 +141,7 @@ static void pc_numa_cpu(const void *data)
         } else if (socket == 1 && core == 1 && thread == 0) {
             g_assert_cmpint(node, ==, 0);
         } else if (socket == 1 && core == 1 && thread == 1) {
-            g_assert_cmpint(node, ==, 1);
+            g_assert_cmpint(node, ==, 0);
         } else {
             g_assert(false);
         }
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson