x86: Allow to set NUMA distance for different NUMA nodes

Current, QEMU does not provide a clear command to set vNUMA distance for guest although we already have -numa command to set vNUMA nodes.

vNUMA distance makes sense in certain scenario.
But now, if we create a guest that has 4 vNUMA nodes, when we check NUMA info via numactl -H, we will see:

node distance:
node 0 1 2 3

  0:   10   20   20   20
  1:   20   10   20   20
  2:   20   20   10   20
  3:   20   20   20   10

Guest kernel regards all local node as distance 10, and all remote node as distance 20 when there is no SLIT table since QEMU doesn't build it. It looks like a little strange when you have seen the distance in an actual physical machine that contains 4 NUMA nodes. My machine shows:

node distance:
node 0 1 2 3

  0:   10   21   31   41
  1:   21   10   21   31
  2:   31   21   10   21
  3:   41   31   21   10

This patch is going to add SLIT table support in QEMU, and provides addtional option dist for command -numa to allow user set vNUMA distance by QEMU command.

With this patch, when a user wants to create a guest that contains several vNUMA nodes and also wants to set distance among those nodes, the QEMU command would like:

```

(quoted diff) +0 -12

-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=0,policy=bind,id=node0 \
-numa node,nodeid=0,cpus=0,memdev=node0 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=1,policy=bind,id=node1 \
-numa node,nodeid=1,cpus=1,memdev=node1 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=2,policy=bind,id=node2 \
-numa node,nodeid=2,cpus=2,memdev=node2 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=3,policy=bind,id=node3 \
-numa node,nodeid=3,cpus=3,memdev=node3 \
-numa dist,a=0,b=1,val=21 \
-numa dist,a=0,b=2,val=31 \
-numa dist,a=0,b=3,val=41 \
-numa dist,a=1,b=0,val=21 \
...

```

Thanks,
-He

Signed-off-by He Chen <[email protected]>

---
 hw/i386/acpi-build.c        | 28 ++++++++++++++++++++++++++
 include/hw/acpi/acpi-defs.h |  9 +++++++++
 include/sysemu/numa.h       |  1 +
 numa.c                      | 48 +++++++++++++++++++++++++++++++++++++++++++++
 qapi-schema.json            | 24 +++++++++++++++++++++--
 qemu-options.hx             |  5 ++++-
 6 files changed, 112 insertions(+), 3 deletions(-)

hw/i386/acpi-build.c +0 -0

index 2073108..7ced37d 100644

hw/i386/acpi-build.c +6 -0

+    uint8_t *entry;
+    int slit_start, slit_data_len, i, j;
+    slit_start = table_data->len;
+
+    slit = acpi_data_push(table_data, sizeof(*slit));
+    slit->nb_localities = nb_numa_nodes;

hw/i386/acpi-build.c +9 -0

+    build_header(linker, table_data,
+                 (void *)(table_data->data + slit_start),
+                 "SLIT",
+                 table_data->len - slit_start, 1, NULL, NULL);
+}
+
+static void
 build_mcfg_q35(GArray *table_data, BIOSLinker *linker, AcpiMcfgInfo *info)
 {
     AcpiTableMcfg *mcfg;

@@ -2678,6 +2704,8 @@ void acpi_build(AcpiBuildTables *tables, MachineState *machine)
     if (pcms->numa_nodes) {
         acpi_add_table(table_offsets, tables_blob);
         build_srat(tables_blob, tables->linker, machine);
+        acpi_add_table(table_offsets, tables_blob);
+        build_slit(tables_blob, tables->linker, machine);
     }
     if (acpi_get_mcfg(&mcfg)) {
         acpi_add_table(table_offsets, tables_blob);

include/hw/acpi/acpi-defs.h +2 -0

index 4cc3630..b183a8f 100644
@@ -527,6 +527,15 @@ struct AcpiSratProcessorGiccAffinity
 
 typedef struct AcpiSratProcessorGiccAffinity AcpiSratProcessorGiccAffinity;
 
+/*
+ * SLIT (NUMA distance description) table

include/sysemu/numa.h +1 -0

index 8f09dcf..2f7a941 100644
@@ -21,6 +21,7 @@ typedef struct node_info {
     struct HostMemoryBackend *node_memdev;
     bool present;
     QLIST_HEAD(, numa_addr_range) addr; /* List to store address ranges */
+    uint8_t distance[MAX_NODES];
 } NodeInfo;
 
 extern NodeInfo numa_info[MAX_NODES];

numa.c +0 -0

index e01cb54..897657a 100644

numa.c +45 -0

@@ -208,10 +211,33 @@ static void numa_node_parse(NumaNodeOptions *node, QemuOpts *opts, Error **errp)
         numa_info[nodenr].node_mem = object_property_get_int(o, "size", NULL);
         numa_info[nodenr].node_memdev = MEMORY_BACKEND(o);
     }
+
     numa_info[nodenr].present = true;
     max_numa_nodeid = MAX(max_numa_nodeid, nodenr + 1);
 }
 
+static void numa_distance_parse(NumaDistOptions *dist, QemuOpts *opts, Error **errp)
+{
+    uint8_t a = dist->a;
+    uint8_t b = dist->b;
+    uint8_t val = dist->val;
+
+    if (a >= MAX_NODES || b >= MAX_NODES) {
+        error_setg(errp, "Max number of NUMA nodes reached: %"
+                   PRIu16 "", a > b ? a : b);
+        return;
+    }
+
+    if (val > max_numa_distance || val < min_numa_distance) {
+        error_setg(errp,
+                "NUMA distance (%" PRIu8 ") out of range (%d)~(%d)",
+                dist->val, max_numa_distance, min_numa_distance);
+        return;
+    }
+
+    numa_info[a].distance[b] = val;
+}
+
 static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
 {
     NumaOptions *object = NULL;

@@ -235,6 +261,12 @@ static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
         }
         nb_numa_nodes++;
         break;
+    case NUMA_OPTIONS_TYPE_DIST:
+        numa_distance_parse(&object->u.dist, opts, &err);
+        if (err) {
+            goto end;
+        }
+        break;
     default:
         abort();
     }

@@ -294,6 +326,21 @@ static void validate_numa_cpus(void)
     g_free(seen_cpus);
 }
 
+static void default_numa_distance(void)
+{
+    int i, j;
+
+    for (i = 0; i < nb_numa_nodes; i++) {
+        for (j = 0; j < nb_numa_nodes; j++) {
+            if (i == j && numa_info[i].distance[j] != min_numa_distance) {
+                numa_info[i].distance[j] = min_numa_distance;
+            } else if (numa_info[i].distance[j] <= min_numa_distance) {
+                numa_info[i].distance[j] = def_numa_distance;
+            }
+        }
+    }
+}
+
 void parse_numa_opts(MachineClass *mc)
 {
     int i;

@@ -390,6 +437,7 @@ void parse_numa_opts(MachineClass *mc)
         }
 
         validate_numa_cpus();
+        default_numa_distance();
     } else {
         numa_set_mem_node_id(0, ram_size, 0);
     }

qapi-schema.json +22 -2

index 32b4a4b..2988304 100644
@@ -5647,7 +5647,7 @@
 # Since: 2.1
 ##
 { 'enum': 'NumaOptionsType',
-  'data': [ 'node' ] }
+  'data': [ 'node', 'dist' ] }
 
 ##
 # @NumaOptions:

@@ -5660,7 +5660,8 @@
   'base': { 'type': 'NumaOptionsType' },
   'discriminator': 'type',
   'data': {
-    'node': 'NumaNodeOptions' }}
+    'node': 'NumaNodeOptions',
+    'dist': 'NumaDistOptions' }}
 
 ##
 # @NumaNodeOptions:

@@ -5689,6 +5690,25 @@
    '*memdev': 'str' }}
 
 ##
+# @NumaDistOptions:
+#
+# Set distance between 2 NUMA nodes. (for OptsVisitor)
+#
+# @a: first NUMA node.
+#
+# @b: second NUMA node.
+#
+# @val: NUMA distance between 2 given NUMA nodes.
+#
+# Since: 2.9
+##
+{ 'struct': 'NumaDistOptions',
+  'data': {
+   'a':   'uint8',
+   'b':   'uint8',
+   'val': 'uint8' }}
+
+##
 # @HostMemPolicy:
 #
 # Host memory policy types

qemu-options.hx +2 -1

index 8dd8ee3..0de5cf8 100644
@@ -139,12 +139,15 @@ ETEXI
 
 DEF("numa", HAS_ARG, QEMU_OPTION_numa,
     "-numa node[,mem=size][,cpus=firstcpu[-lastcpu]][,nodeid=node]\n"
-    "-numa node[,memdev=id][,cpus=firstcpu[-lastcpu]][,nodeid=node]\n", QEMU_ARCH_ALL)
+    "-numa node[,memdev=id][,cpus=firstcpu[-lastcpu]][,nodeid=node]\n"
+    "-numa dist,a=firstnode,b=secondnode,val=distance\n", QEMU_ARCH_ALL)

Signature · 1 line

2.7.4

Current, QEMU does not provide a clear command to set vNUMA distance for guest although we already have -numa command to set vNUMA nodes.

This patch is going to add SLIT table support in QEMU, and provides addtional option dist for command -numa to allow user set vNUMA

distance by QEMU command.

With this patch, when a user wants to create a guest that contains several vNUMA nodes and also wants to set distance among those nodes, the QEMU command would like:

```

(quoted diff) +0 -12

-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=0,policy=bind,id=node0 \
-numa node,nodeid=0,cpus=0,memdev=node0 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=1,policy=bind,id=node1 \
-numa node,nodeid=1,cpus=1,memdev=node1 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=2,policy=bind,id=node2 \
-numa node,nodeid=2,cpus=2,memdev=node2 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=3,policy=bind,id=node3 \
-numa node,nodeid=3,cpus=3,memdev=node3 \
-numa dist,a=0,b=1,val=21 \
-numa dist,a=0,b=2,val=31 \
-numa dist,a=0,b=3,val=41 \
-numa dist,a=1,b=0,val=21 \
...

```

+++ b/qapi-schema.json

Quoted 7 lines · He Chen

Current, QEMU does not provide a clear command to set vNUMA distance for guest although we already have -numa command to set vNUMA nodes.

This patch is going to add SLIT table support in QEMU, and provides addtional option dist for command -numa to allow user set vNUMA

s/addtional/additional/

Quoted 25 lines

distance by QEMU command.

With this patch, when a user wants to create a guest that contains several vNUMA nodes and also wants to set distance among those nodes, the QEMU command would like:

```

(quoted diff) +0 -12

-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=0,policy=bind,id=node0 \
-numa node,nodeid=0,cpus=0,memdev=node0 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=1,policy=bind,id=node1 \
-numa node,nodeid=1,cpus=1,memdev=node1 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=2,policy=bind,id=node2 \
-numa node,nodeid=2,cpus=2,memdev=node2 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=3,policy=bind,id=node3 \
-numa node,nodeid=3,cpus=3,memdev=node3 \
-numa dist,a=0,b=1,val=21 \
-numa dist,a=0,b=2,val=31 \
-numa dist,a=0,b=3,val=41 \
-numa dist,a=1,b=0,val=21 \
...

```

+++ b/qapi-schema.json

qapi-schema.json line 5650

Quoted 5 lines

qapi-schema.json +1 -1

@@ -5647,7 +5647,7 @@
 # Since: 2.1
 ##
 { 'enum': 'NumaOptionsType',
-  'data': [ 'node' ] }
+  'data': [ 'node', 'dist' ] }

Missing documentation, including the fact that 'dist' is added in 2.10 (you've missed 2.9).

qapi-schema.json line 5703

Quoted 13 lines

qapi-schema.json +11 -0

 ##
+# @NumaDistOptions:
+#
+# Set distance between 2 NUMA nodes. (for OptsVisitor)
+#
+# @a: first NUMA node.
+#
+# @b: second NUMA node.
+#
+# @val: NUMA distance between 2 given NUMA nodes.
+#
+# Since: 2.9

This is a new feature, but you've missed soft freeze; this will have to be 2.10.

qapi-schema.json line 5709

Quoted 6 lines

qapi-schema.json +6 -0

+##
+{ 'struct': 'NumaDistOptions',
+  'data': {
+   'a':   'uint8',
+   'b':   'uint8',
+   'val': 'uint8' }}

Using uint8 limits us to at most 256 numa nodes. Is that going to bite us in the future?

Is 'a' and 'b' really the best naming convention to use (it's concise, but maybe 'first' and 'second', or 'source' and 'destination' is better)?

Quoted 72 lines · Eric Blake

Quoted 7 lines · He Chen

Current, QEMU does not provide a clear command to set vNUMA distance for guest although we already have -numa command to set vNUMA nodes.

This patch is going to add SLIT table support in QEMU, and provides addtional option dist for command -numa to allow user set vNUMA

s/addtional/additional/

Quoted 25 lines

distance by QEMU command.

With this patch, when a user wants to create a guest that contains several vNUMA nodes and also wants to set distance among those nodes, the QEMU command would like:

```

(quoted diff) +0 -12

-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=0,policy=bind,id=node0 \
-numa node,nodeid=0,cpus=0,memdev=node0 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=1,policy=bind,id=node1 \
-numa node,nodeid=1,cpus=1,memdev=node1 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=2,policy=bind,id=node2 \
-numa node,nodeid=2,cpus=2,memdev=node2 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=3,policy=bind,id=node3 \
-numa node,nodeid=3,cpus=3,memdev=node3 \
-numa dist,a=0,b=1,val=21 \
-numa dist,a=0,b=2,val=31 \
-numa dist,a=0,b=3,val=41 \
-numa dist,a=1,b=0,val=21 \
...

```

+++ b/qapi-schema.json

qapi-schema.json line 5650

Quoted 5 lines

qapi-schema.json +1 -1

@@ -5647,7 +5647,7 @@
 # Since: 2.1
 ##
 { 'enum': 'NumaOptionsType',
-  'data': [ 'node' ] }
+  'data': [ 'node', 'dist' ] }

Missing documentation, including the fact that 'dist' is added in 2.10 (you've missed 2.9).

qapi-schema.json line 5703

Quoted 13 lines

qapi-schema.json +11 -0

 ##
+# @NumaDistOptions:
+#
+# Set distance between 2 NUMA nodes. (for OptsVisitor)
+#
+# @a: first NUMA node.
+#
+# @b: second NUMA node.
+#
+# @val: NUMA distance between 2 given NUMA nodes.
+#
+# Since: 2.9

This is a new feature, but you've missed soft freeze; this will have to be 2.10.

qapi-schema.json line 5709

Quoted 6 lines

qapi-schema.json +6 -0

+##
+{ 'struct': 'NumaDistOptions',
+  'data': {
+   'a':   'uint8',
+   'b':   'uint8',
+   'val': 'uint8' }}

Using uint8 limits us to at most 256 numa nodes. Is that going to bite us in the future?

Quoted 2 lines

Is 'a' and 'b' really the best naming convention to use (it's concise, but maybe 'first' and 'second', or 'source' and 'destination' is better)?

'src' and 'dst' are probably a reasnoable compromise between meaningful and short.

Regards,
Daniel

Signature · 3 lines

|: http://berrange.com      -o-    http://www.flickr.com/photos/dberrange/ :|
|: http://libvirt.org              -o-             http://virt-manager.org :|
|: http://entangle-photo.org       -o-    http://search.cpan.org/~danberr/ :|

Quoted 73 lines · Daniel P. Berrange

Quoted 72 lines · Eric Blake

Quoted 7 lines · He Chen

Current, QEMU does not provide a clear command to set vNUMA distance for guest although we already have -numa command to set vNUMA nodes.

This patch is going to add SLIT table support in QEMU, and provides addtional option dist for command -numa to allow user set vNUMA

s/addtional/additional/

Quoted 25 lines

distance by QEMU command.

With this patch, when a user wants to create a guest that contains several vNUMA nodes and also wants to set distance among those nodes, the QEMU command would like:

```

(quoted diff) +0 -12

-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=0,policy=bind,id=node0 \
-numa node,nodeid=0,cpus=0,memdev=node0 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=1,policy=bind,id=node1 \
-numa node,nodeid=1,cpus=1,memdev=node1 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=2,policy=bind,id=node2 \
-numa node,nodeid=2,cpus=2,memdev=node2 \
-object memory-backend-ram,size=1G,prealloc=yes,host-nodes=3,policy=bind,id=node3 \
-numa node,nodeid=3,cpus=3,memdev=node3 \
-numa dist,a=0,b=1,val=21 \
-numa dist,a=0,b=2,val=31 \
-numa dist,a=0,b=3,val=41 \
-numa dist,a=1,b=0,val=21 \
...

```

+++ b/qapi-schema.json

qapi-schema.json line 5650

Quoted 5 lines

qapi-schema.json +1 -1

@@ -5647,7 +5647,7 @@
 # Since: 2.1
 ##
 { 'enum': 'NumaOptionsType',
-  'data': [ 'node' ] }
+  'data': [ 'node', 'dist' ] }

Missing documentation, including the fact that 'dist' is added in 2.10 (you've missed 2.9).

qapi-schema.json line 5703

Quoted 13 lines

qapi-schema.json +11 -0

 ##
+# @NumaDistOptions:
+#
+# Set distance between 2 NUMA nodes. (for OptsVisitor)
+#
+# @a: first NUMA node.
+#
+# @b: second NUMA node.
+#
+# @val: NUMA distance between 2 given NUMA nodes.
+#
+# Since: 2.9

This is a new feature, but you've missed soft freeze; this will have to be 2.10.

qapi-schema.json line 5709

Quoted 6 lines

qapi-schema.json +6 -0

+##
+{ 'struct': 'NumaDistOptions',
+  'data': {
+   'a':   'uint8',
+   'b':   'uint8',
+   'val': 'uint8' }}

Using uint8 limits us to at most 256 numa nodes. Is that going to bite us in the future?

Quoted 5 lines

Quoted 2 lines

Is 'a' and 'b' really the best naming convention to use (it's concise, but maybe 'first' and 'second', or 'source' and 'destination' is better)?

'src' and 'dst' are probably a reasnoable compromise between meaningful and short.

AFAIK the distance is assumed to be symmetrical, src/dst seems to imply asymmetry.