ACPI: Support Generic Initiator proximity domains

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Signed-off-by Jonathan Cameron <[email protected]>

---
 drivers/acpi/numa/srat.c       | 62 +++++++++++++++++++++++++++++++++-
 drivers/base/node.c            |  3 ++
 include/asm-generic/topology.h |  3 ++
 include/linux/nodemask.h       |  1 +
 include/linux/topology.h       |  7 ++++
 5 files changed, 75 insertions(+), 1 deletion(-)

drivers/acpi/numa/srat.c +61 -1

index eadbf90e65d1..fe34315a9234 100644
@@ -170,6 +170,38 @@ acpi_table_print_srat_entry(struct acpi_subtable_header *header)
 		}
 		break;
 
+	case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
+	{
+		struct acpi_srat_generic_affinity *p =
+			(struct acpi_srat_generic_affinity *)header;
+		char name[9] = {};
+
+		if (p->device_handle_type == 0) {
+			/*
+			 * For pci devices this may be the only place they
+			 * are assigned a proximity domain
+			 */
+			pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n",
+				 *(u16 *)(&p->device_handle[0]),
+				 *(u16 *)(&p->device_handle[2]),
+				 p->proximity_domain,
+				 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
+				"enabled" : "disabled");
+		} else {
+			/*
+			 * In this case we can rely on the device having a
+			 * proximity domain reference
+			 */
+			memcpy(name, p->device_handle, 8);
+			pr_info("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n",
+				(char *)(&p->device_handle[0]),
+				(char *)(&p->device_handle[8]),
+				p->proximity_domain,
+				(p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
+				"enabled" : "disabled");
+		}
+	}
+	break;
 	default:
 		pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
 			header->type);

@@ -378,6 +410,32 @@ acpi_parse_gicc_affinity(union acpi_subtable_headers *header,
 	return 0;
 }
 
+static int __init
+acpi_parse_gi_affinity(union acpi_subtable_headers *header,
+		       const unsigned long end)
+{
+	struct acpi_srat_generic_affinity *gi_affinity;
+	int node;
+
+	gi_affinity = (struct acpi_srat_generic_affinity *)header;
+	if (!gi_affinity)
+		return -EINVAL;
+	acpi_table_print_srat_entry(&header->common);
+
+	if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
+		return -EINVAL;
+
+	node = acpi_map_pxm_to_node(gi_affinity->proximity_domain);
+	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
+		pr_err("SRAT: Too many proximity domains.\n");
+		return -EINVAL;
+	}
+	node_set(node, numa_nodes_parsed);
+	node_set_state(node, N_GENERIC_INITIATOR);
+
+	return 0;
+}
+
 static int __initdata parsed_numa_memblks;
 
 static int __init

@@ -433,7 +491,7 @@ int __init acpi_numa_init(void)
 
 	/* SRAT: System Resource Affinity Table */
 	if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
-		struct acpi_subtable_proc srat_proc[3];
+		struct acpi_subtable_proc srat_proc[4];
 
 		memset(srat_proc, 0, sizeof(srat_proc));
 		srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;

@@ -442,6 +500,8 @@ int __init acpi_numa_init(void)
 		srat_proc[1].handler = acpi_parse_x2apic_affinity;
 		srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
 		srat_proc[2].handler = acpi_parse_gicc_affinity;
+		srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY;
+		srat_proc[3].handler = acpi_parse_gi_affinity;
 
 		acpi_table_parse_entries_array(ACPI_SIG_SRAT,
 					sizeof(struct acpi_table_srat),

drivers/base/node.c +3 -0

index 296546ffed6c..e5863baa8cb6 100644
@@ -977,6 +977,8 @@ static struct node_attr node_state_attr[] = {
 #endif
 	[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
 	[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
+	[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
+					   N_GENERIC_INITIATOR),
 };
 
 static struct attribute *node_state_attrs[] = {

@@ -988,6 +990,7 @@ static struct attribute *node_state_attrs[] = {
 #endif
 	&node_state_attr[N_MEMORY].attr.attr,
 	&node_state_attr[N_CPU].attr.attr,
+	&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
 	NULL
 };
 

include/asm-generic/topology.h +3 -0

index 238873739550..54d0b4176a45 100644
@@ -71,6 +71,9 @@
 #ifndef set_cpu_numa_mem
 #define set_cpu_numa_mem(cpu, node)
 #endif
+#ifndef set_gi_numa_mem
+#define set_gi_numa_mem(gi, node)
+#endif
 
 #endif	/* !CONFIG_NUMA || !CONFIG_HAVE_MEMORYLESS_NODES */
 

include/linux/nodemask.h +1 -0

index 27e7fa36f707..1aebf766fb52 100644
@@ -399,6 +399,7 @@ enum node_states {
 #endif
 	N_MEMORY,		/* The node has memory(regular, high, movable) */
 	N_CPU,		/* The node has one or more cpus */
+	N_GENERIC_INITIATOR,	/* The node is a GI only node */
 	NR_NODE_STATES
 };
 

include/linux/topology.h +7 -0

index eb2fe6edd73c..05ccf011e489 100644
@@ -140,6 +140,13 @@ static inline void set_numa_mem(int node)
 }
 #endif
 
+#ifndef set_gi_numa_mem
+static inline void set_gi_numa_mem(int gi, int node)
+{
+	_node_numa_mem_[gi] = node;
+}
+#endif
+
 #ifndef node_to_mem_node
 static inline int node_to_mem_node(int node)
 {

Quoted 9 lines · CEST Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 1 line

Signed-off-by Jonathan Cameron <[email protected]>

This depends on the series from Dan at:

https://lore.kernel.org/linux-acpi/CAPcyv4gBSX58CWH4HZ28w0_cZRzJrhgdEFHa2g8KDqyv8aFqZQ@mail.gmail.com/T/#m1acce3ae8f29f680c0d95fd1e840e703949fbc48

AFAICS, so please respin when that one hits the Linus' tree.

Quoted 169 lines

---
 drivers/acpi/numa/srat.c       | 62 +++++++++++++++++++++++++++++++++-
 drivers/base/node.c            |  3 ++
 include/asm-generic/topology.h |  3 ++
 include/linux/nodemask.h       |  1 +
 include/linux/topology.h       |  7 ++++
 5 files changed, 75 insertions(+), 1 deletion(-)

drivers/acpi/numa/srat.c +61 -1

index eadbf90e65d1..fe34315a9234 100644
@@ -170,6 +170,38 @@ acpi_table_print_srat_entry(struct acpi_subtable_header *header)
 		}
 		break;
 
+	case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
+	{
+		struct acpi_srat_generic_affinity *p =
+			(struct acpi_srat_generic_affinity *)header;
+		char name[9] = {};
+
+		if (p->device_handle_type == 0) {
+			/*
+			 * For pci devices this may be the only place they
+			 * are assigned a proximity domain
+			 */
+			pr_debug("SRAT Generic Initiator(Seg:%u BDF:%u) in proximity domain %d %s\n",
+				 *(u16 *)(&p->device_handle[0]),
+				 *(u16 *)(&p->device_handle[2]),
+				 p->proximity_domain,
+				 (p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
+				"enabled" : "disabled");
+		} else {
+			/*
+			 * In this case we can rely on the device having a
+			 * proximity domain reference
+			 */
+			memcpy(name, p->device_handle, 8);
+			pr_info("SRAT Generic Initiator(HID=%.8s UID=%.4s) in proximity domain %d %s\n",
+				(char *)(&p->device_handle[0]),
+				(char *)(&p->device_handle[8]),
+				p->proximity_domain,
+				(p->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED) ?
+				"enabled" : "disabled");
+		}
+	}
+	break;
 	default:
 		pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
 			header->type);

@@ -378,6 +410,32 @@ acpi_parse_gicc_affinity(union acpi_subtable_headers *header,
 	return 0;
 }
 
+static int __init
+acpi_parse_gi_affinity(union acpi_subtable_headers *header,
+		       const unsigned long end)
+{
+	struct acpi_srat_generic_affinity *gi_affinity;
+	int node;
+
+	gi_affinity = (struct acpi_srat_generic_affinity *)header;
+	if (!gi_affinity)
+		return -EINVAL;
+	acpi_table_print_srat_entry(&header->common);
+
+	if (!(gi_affinity->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
+		return -EINVAL;
+
+	node = acpi_map_pxm_to_node(gi_affinity->proximity_domain);
+	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
+		pr_err("SRAT: Too many proximity domains.\n");
+		return -EINVAL;
+	}
+	node_set(node, numa_nodes_parsed);
+	node_set_state(node, N_GENERIC_INITIATOR);
+
+	return 0;
+}
+
 static int __initdata parsed_numa_memblks;
 
 static int __init

@@ -433,7 +491,7 @@ int __init acpi_numa_init(void)
 
 	/* SRAT: System Resource Affinity Table */
 	if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
-		struct acpi_subtable_proc srat_proc[3];
+		struct acpi_subtable_proc srat_proc[4];
 
 		memset(srat_proc, 0, sizeof(srat_proc));
 		srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;

@@ -442,6 +500,8 @@ int __init acpi_numa_init(void)
 		srat_proc[1].handler = acpi_parse_x2apic_affinity;
 		srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
 		srat_proc[2].handler = acpi_parse_gicc_affinity;
+		srat_proc[3].id = ACPI_SRAT_TYPE_GENERIC_AFFINITY;
+		srat_proc[3].handler = acpi_parse_gi_affinity;
 
 		acpi_table_parse_entries_array(ACPI_SIG_SRAT,
 					sizeof(struct acpi_table_srat),

drivers/base/node.c +3 -0

index 296546ffed6c..e5863baa8cb6 100644
@@ -977,6 +977,8 @@ static struct node_attr node_state_attr[] = {
 #endif
 	[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
 	[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
+	[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
+					   N_GENERIC_INITIATOR),
 };
 
 static struct attribute *node_state_attrs[] = {

@@ -988,6 +990,7 @@ static struct attribute *node_state_attrs[] = {
 #endif
 	&node_state_attr[N_MEMORY].attr.attr,
 	&node_state_attr[N_CPU].attr.attr,
+	&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
 	NULL
 };
 

include/asm-generic/topology.h +3 -0

index 238873739550..54d0b4176a45 100644
@@ -71,6 +71,9 @@
 #ifndef set_cpu_numa_mem
 #define set_cpu_numa_mem(cpu, node)
 #endif
+#ifndef set_gi_numa_mem
+#define set_gi_numa_mem(gi, node)
+#endif
 
 #endif	/* !CONFIG_NUMA || !CONFIG_HAVE_MEMORYLESS_NODES */
 

include/linux/nodemask.h +1 -0

index 27e7fa36f707..1aebf766fb52 100644
@@ -399,6 +399,7 @@ enum node_states {
 #endif
 	N_MEMORY,		/* The node has memory(regular, high, movable) */
 	N_CPU,		/* The node has one or more cpus */
+	N_GENERIC_INITIATOR,	/* The node is a GI only node */
 	NR_NODE_STATES
 };
 

include/linux/topology.h +7 -0

index eb2fe6edd73c..05ccf011e489 100644
@@ -140,6 +140,13 @@ static inline void set_numa_mem(int node)
 }
 #endif
 
+#ifndef set_gi_numa_mem
+static inline void set_gi_numa_mem(int gi, int node)
+{
+	_node_numa_mem_[gi] = node;
+}
+#endif
+
 #ifndef node_to_mem_node
 static inline int node_to_mem_node(int node)
 {

Quoted 10 lines · On Fri, 18 Oct 2019 12:18:33 +0200 "Rafael J. Wysocki" <[email protected]>…

Quoted 9 lines · CEST Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Quoted 1 line

Signed-off-by Jonathan Cameron <[email protected]>

This depends on the series from Dan at:

https://lore.kernel.org/linux-acpi/CAPcyv4gBSX58CWH4HZ28w0_cZRzJrhgdEFHa2g8KDqyv8aFqZQ@mail.gmail.com/T/#m1acce3ae8f29f680c0d95fd1e840e703949fbc48

Hi Rafael,

Yes. Cover letter mentions it was rebased on v4 of that series.

Quoted 1 line

AFAICS, so please respin when that one hits the Linus' tree.

Sure, though that pushes it out another cycle and it's beginning to get a bit silly (just rebases since April).

I guess it can't be helped given the series hits several trees.

Quoted 25 lines · CET Jonathan Cameron

Quoted 10 lines · On Fri, 18 Oct 2019 12:18:33 +0200 "Rafael J. Wysocki" <[email protected]>…

Quoted 9 lines · CEST Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Quoted 1 line

Signed-off-by Jonathan Cameron <[email protected]>

This depends on the series from Dan at:

https://lore.kernel.org/linux-acpi/CAPcyv4gBSX58CWH4HZ28w0_cZRzJrhgdEFHa2g8KDqyv8aFqZQ@mail.gmail.com/T/#m1acce3ae8f29f680c0d95fd1e840e703949fbc48

Hi Rafael,

Yes. Cover letter mentions it was rebased on v4 of that series.

Quoted 1 line

AFAICS, so please respin when that one hits the Linus' tree.

Quoted 4 lines

Sure, though that pushes it out another cycle and it's beginning to get a bit silly (just rebases since April).

I guess it can't be helped given the series hits several trees.

I've just applied the Dan's series and I can take patch [1/4] from this one, but for the [2-3/4] I'd like to get some ACKs from the arm64 and x86 people respectively.

Thanks!

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Jonathan

Quoted 51 lines · Jonathan Cameron

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Quoted 2 lines

Jonathan

Indeed, HMAT can support Generic Initiator, but as far as I know, QEMU only can emulate a node with cpu and memory, or memory-only. Even if we assign a node with cpu only, qemu will raise error. Considering compatibility, there are lots of work to do for QEMU if we change NUMA or SRAT table.

Quoted 53 lines · Tao Xu

Quoted 51 lines · Jonathan Cameron

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Quoted 7 lines

Quoted 2 lines

Jonathan

Indeed, HMAT can support Generic Initiator, but as far as I know, QEMU only can emulate a node with cpu and memory, or memory-only. Even if we assign a node with cpu only, qemu will raise error. Considering compatibility, there are lots of work to do for QEMU if we change NUMA or SRAT table.

Thanks for the background. It would still be a useful feature to be able to define a memory + generic-initiator node in qemu. That will mirror real world accelerators with local memory configurations.

Quoted 51 lines · Jonathan Cameron

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Quoted 2 lines

Jonathan

Indeed, HMAT can support Generic Initiator, but as far as I know, QEMU only can emulate a node with cpu and memory, or memory-only. Even if we assign a node with cpu only, qemu will raise error. Considering compatibility, there are lots of work to do for QEMU if we change NUMA or SRAT table.

Quoted 52 lines · On Wed, 13 Nov 2019 21:57:24 +0800 Tao Xu <[email protected]> wrote:

Quoted 51 lines · Jonathan Cameron

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Quoted 8 lines

Quoted 2 lines

Jonathan

Indeed, HMAT can support Generic Initiator, but as far as I know, QEMU only can emulate a node with cpu and memory, or memory-only. Even if we assign a node with cpu only, qemu will raise error. Considering compatibility, there are lots of work to do for QEMU if we change NUMA or SRAT table.

I faked up a quick HMAT table.

Used a configuration with 3x CPU and memory nodes, 1x memory only node and 1x GI node. Two test cases, one where the GI initiator is further than the CPU containing nodes from the memory only node (realistic case for existing hardware). That behaves as expected and there are no /sys/node/bus/nodeX/access0 entries for the GI node
+ appropriate ones for the memory only node as normal.

The other case is more interesting we have the memory only node nearer to the GI node than to any of the CPUs. In that case for x86 at least the HMAT code is happy to put an access0 directory GI in the GI node with empty access0/initiators and the memory node under access0/targets

The memory only node is node4 and the GI node node3.

So relevant dirs under /sys/bus/nodes/devices

node3/access0/initators/ Empty
node3/access0/targets/node4

node4/access0/initators/[node3 read_bandwidth write_bandwith etc] node4/access0/targets/ Empty

So the result current (I think - the HMAT interface still confuses me :) is that a GI node is treated like a CPU node. This might mean there is no useful information available if you want to figure out which CPU containing node is nearest to Memory when the GI node is nearer still.

Is this a problem? I'm not sure...

If we don't want to include GI nodes then we can possibly use the node_state(N_CPU, x) method to check before considering them, or I guess parse SRAT to extract that info directly.

I tried this and it seems to work so can add patch doing this next version if we think this is the 'right' thing to do.

So what do you think 'should' happen?

Quoted 79 lines · Jonathan Cameron

Quoted 52 lines · On Wed, 13 Nov 2019 21:57:24 +0800 Tao Xu <[email protected]> wrote:

Quoted 51 lines · Jonathan Cameron

Quoted 14 lines · On Tue, 12 Nov 2019 09:55:17 -0800 Dan Williams <[email protected]>…

[ add Tao Xu ]

Quoted 9 lines · Jonathan Cameron

Generic Initiators are a new ACPI concept that allows for the description of proximity domains that contain a device which performs memory access (such as a network card) but neither host CPU nor Memory.

This patch has the parsing code and provides the infrastructure for an architecture to associate these new domains with their nearest memory processing node.

Quoted 6 lines

Thanks for this Jonathan. May I ask how this was tested? Tao has been working on qemu support for HMAT [1]. I have not checked if it already supports generic initiator entries, but it would be helpful to include an example of how the kernel sees these configurations in practice.

[1]: http://patchwork.ozlabs.org/cover/1096737/

Tested against qemu with SRAT and SLIT table overrides from an initrd to actually create the node and give it distances (those all turn up correctly in the normal places). DSDT override used to move an emulated network card into the GI numa node. That currently requires the PCI patch referred to in the cover letter. On arm64 tested both on qemu and real hardware (overrides on tables even for real hardware as I can't persuade our BIOS team to implement Generic Initiators until an OS is actually using them.)

Main real requirement is memory allocations then occur from one of the nodes at the minimal distance when you are do a devm_ allocation from a device assigned. Also need to be able to query the distances to allow load balancing etc. All that works as expected.

It only has a fairly tangential connection to HMAT in that HMAT can provide information on GI nodes. Given HMAT code is quite happy with memoryless nodes anyway it should work. QEMU doesn't currently have support to create GI SRAT entries let alone HMAT using them.

Whilst I could look at adding such support to QEMU, it's not exactly high priority to emulate something we can test easily by overriding the tables before the kernel reads them.

I'll look at how hard it is to build an HMAT tables for my test configs based on the ones I used to test your HMAT patches a while back. Should be easy if tedious.

Quoted 8 lines

Quoted 2 lines

Jonathan

Indeed, HMAT can support Generic Initiator, but as far as I know, QEMU only can emulate a node with cpu and memory, or memory-only. Even if we assign a node with cpu only, qemu will raise error. Considering compatibility, there are lots of work to do for QEMU if we change NUMA or SRAT table.

I faked up a quick HMAT table.

Used a configuration with 3x CPU and memory nodes, 1x memory only node and 1x GI node. Two test cases, one where the GI initiator is further than the CPU containing nodes from the memory only node (realistic case for existing hardware). That behaves as expected and there are no /sys/node/bus/nodeX/access0 entries for the GI node
+ appropriate ones for the memory only node as normal.

The other case is more interesting we have the memory only node nearer to the GI node than to any of the CPUs. In that case for x86 at least the HMAT code is happy to put an access0 directory GI in the GI node with empty access0/initiators and the memory node under access0/targets

Quoted 6 lines

The memory only node is node4 and the GI node node3.

So relevant dirs under /sys/bus/nodes/devices

node3/access0/initators/ Empty
node3/access0/targets/node4

This makes sense node3 is an initiator, no other nodes can initiate to it.

Quoted 18 lines

node4/access0/initators/[node3 read_bandwidth write_bandwith etc] node4/access0/targets/ Empty

So the result current (I think - the HMAT interface still confuses me :) is that a GI node is treated like a CPU node. This might mean there is no useful information available if you want to figure out which CPU containing node is nearest to Memory when the GI node is nearer still.

Is this a problem? I'm not sure...

If we don't want to include GI nodes then we can possibly use the node_state(N_CPU, x) method to check before considering them, or I guess parse SRAT to extract that info directly.

I tried this and it seems to work so can add patch doing this next version if we think this is the 'right' thing to do.

Quoted 1 line

So what do you think 'should' happen?

I think this might be our first case for adding an "access1" instance by default. I.e. in the case when access0 is not a cpu, then access1 is there to at least show the "local" cpu and let userspace see the performance difference of cpu vs a specific-initiator access.

Hi Dan,

Agreed that it makes sense to expand how we describe these cases a bit. To make sure I've understood correctly let me paraphrase what you are proposing (and tweak it a bit ;)

Assuming for this purpose we don't put GIs in CPU nodes as that makes for really fiddly explanation. In reality the code will need to handle that.

1) Leave access0 as it currently is with this series - so continue to not distinguish between CPU nodes and Generic Initator containing ones?

2) Add access 1 which is effectively access0 ignoring Generic Initiators?

My feeling is that any existing users of access0 are definitely not going to be expecting generic initiators, so we might want to do this the other way around. access0 is only CPUs and memory, access1 is including generic initiators. If there are no GIs don't expose access1 at all?

For now we could simply block the GI visibility in access0 and deal with access1 as a separate series. I suspect we will get push back as there are no known users of our new access1 so it may take a while to prove utility and get it accepted.

Quoted 5 lines

My feeling is that any existing users of access0 are definitely not going to be expecting generic initiators, so we might want to do this the other way around. access0 is only CPUs and memory, access1 is including generic initiators. If there are no GIs don't expose access1 at all?

There are no consumers of the information that I know of, so I do not see the risk of regression.

Done in a somewhat different fashion to arm64. Here the infrastructure for memoryless domains was already in place. That infrastruture applies just as well to domains that also don't have a CPU, hence it works for Generic Initiator Domains.

In common with memoryless domains we only register GI domains if the proximity node is not online. If a domain is already a memory containing domain, or a memoryless domain there is nothing to do just because it also contains a Generic Initiator.

Signed-off-by Jonathan Cameron <[email protected]>

---
 arch/x86/include/asm/numa.h |  2 ++
 arch/x86/kernel/setup.c     |  1 +
 arch/x86/mm/numa.c          | 14 ++++++++++++++
 3 files changed, 17 insertions(+)

arch/x86/include/asm/numa.h +2 -0

index bbfde3d2662f..f631467272a3 100644
@@ -62,12 +62,14 @@ extern void numa_clear_node(int cpu);
 extern void __init init_cpu_to_node(void);
 extern void numa_add_cpu(int cpu);
 extern void numa_remove_cpu(int cpu);
+extern void init_gi_nodes(void);
 #else	/* CONFIG_NUMA */
 static inline void numa_set_node(int cpu, int node)	{ }
 static inline void numa_clear_node(int cpu)		{ }
 static inline void init_cpu_to_node(void)		{ }
 static inline void numa_add_cpu(int cpu)		{ }
 static inline void numa_remove_cpu(int cpu)		{ }
+static inline void init_gi_nodes(void)			{ }
 #endif	/* CONFIG_NUMA */
 
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS

arch/x86/kernel/setup.c +1 -0

index cfb533d42371..b6c977907ea5 100644
@@ -1264,6 +1264,7 @@ void __init setup_arch(char **cmdline_p)
 	prefill_possible_map();
 
 	init_cpu_to_node();
+	init_gi_nodes();
 
 	io_apic_init_mappings();
 

arch/x86/mm/numa.c +3 -0

index 4123100e0eaf..50bf724a425e 100644
@@ -733,6 +733,20 @@ static void __init init_memory_less_node(int nid)
 	 */
 }
 
+/*
+ * Generic Initiator Nodes may have neither CPU nor Memory.
+ * At this stage if either of the others were present we would

Quoted 57 lines · Jonathan Cameron

Done in a somewhat different fashion to arm64. Here the infrastructure for memoryless domains was already in place. That infrastruture applies just as well to domains that also don't have a CPU, hence it works for Generic Initiator Domains.

In common with memoryless domains we only register GI domains if the proximity node is not online. If a domain is already a memory containing domain, or a memoryless domain there is nothing to do just because it also contains a Generic Initiator.

Signed-off-by Jonathan Cameron <[email protected]>

---
 arch/x86/include/asm/numa.h |  2 ++
 arch/x86/kernel/setup.c     |  1 +
 arch/x86/mm/numa.c          | 14 ++++++++++++++
 3 files changed, 17 insertions(+)

arch/x86/include/asm/numa.h +2 -0

index bbfde3d2662f..f631467272a3 100644
@@ -62,12 +62,14 @@ extern void numa_clear_node(int cpu);
 extern void __init init_cpu_to_node(void);
 extern void numa_add_cpu(int cpu);
 extern void numa_remove_cpu(int cpu);
+extern void init_gi_nodes(void);
 #else	/* CONFIG_NUMA */
 static inline void numa_set_node(int cpu, int node)	{ }
 static inline void numa_clear_node(int cpu)		{ }
 static inline void init_cpu_to_node(void)		{ }
 static inline void numa_add_cpu(int cpu)		{ }
 static inline void numa_remove_cpu(int cpu)		{ }
+static inline void init_gi_nodes(void)			{ }
 #endif	/* CONFIG_NUMA */
 
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS

arch/x86/kernel/setup.c +1 -0

index cfb533d42371..b6c977907ea5 100644
@@ -1264,6 +1264,7 @@ void __init setup_arch(char **cmdline_p)
 	prefill_possible_map();
 
 	init_cpu_to_node();
+	init_gi_nodes();
 
 	io_apic_init_mappings();
 

arch/x86/mm/numa.c +0 -0

index 4123100e0eaf..50bf724a425e 100644
@@ -733,6 +733,20 @@ static void __init init_memory_less_node(int nid)
 	 */
 }
 

arch/x86/mm/numa.c static void __init init_memory_less_node(int nid) · line 748

Quoted 13 lines

arch/x86/mm/numa.c +13 -0

+/*
+ * Generic Initiator Nodes may have neither CPU nor Memory.
+ * At this stage if either of the others were present we would
+ * already be online.
+ */
+void __init init_gi_nodes(void)
+{
+	int nid;
+
+	for_each_node_state(nid, N_GENERIC_INITIATOR)
+		if (!node_online(nid))
+			init_memory_less_node(nid);
+}  

Nit: missing curly braces.

How do these work in practice, will a system that only had nodes 0-1 today grow a third node '2' that won't have any CPUs on memory on them?

Thanks,

Ingo