Skip to contentSkip to conversation

drm/amdkfd: Create node in kfd sysfs for memory only numa node

PATCH In review Oak Zeng 2 participants 5 messages Jun 1, 2021 – Jun 2, 2021

Subsystems (3)
lore
Share & tools
lore

Oak Zeng

author
#0 lore raw

PATCH 1/4 drm/amdkfd: Create node in kfd sysfs for memory only numa node

Previously kfd driver assumes all CPU numa nodes are associated with system memory and there is no memory only numa node. This is not true anymore for ALDEBARAN A+A set up. This patch creates memory only node in kfd sysfs.

---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 73 ++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 31 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +42 -31
index 3251fe2..56e6dff 100644
@@ -827,8 +827,11 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 	uint32_t flags = 0;
 	uint32_t width;
 
-	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
-			mem->proximity_domain);
+	size_in_bytes =
+		((uint64_t)mem->length_high << 32) +
+		mem->length_low;
+	pr_debug("Found memory entry in CRAT table with proximity_domain=%d, size %lld\n",
+			mem->proximity_domain, size_in_bytes);
 	list_for_each_entry(dev, device_list, list) {
 		if (mem->proximity_domain == dev->proximity_domain) {
 			/* We're on GPU node */
@@ -848,9 +851,6 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
 				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
 
-			size_in_bytes =
-				((uint64_t)mem->length_high << 32) +
-							mem->length_low;
 			width = mem->width;
 
 			/* Multiple banks of the same type are aggregated into
@@ -1718,51 +1718,62 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
 
 	for_each_online_node(numa_node_id) {
+		pr_debug("numa node id %d\n", numa_node_id);
 		if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
-			continue;
-
-		/* Fill in Subtype: Compute Unit */
-		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
-			crat_table->num_domains,
-			(struct crat_subtype_computeunit *)sub_type_hdr);
-		if (ret < 0)
-			return ret;
-		crat_table->length += sub_type_hdr->length;
-		crat_table->total_entries++;
+			pr_debug("Numa node %d is a memory only numa node\n", numa_node_id);
+
+		if (kfd_numa_node_to_apic_id(numa_node_id) != -1) {
+			/* Fill in Subtype: Compute Unit */
+			ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
+					crat_table->num_domains,
+					(struct crat_subtype_computeunit *)sub_type_hdr);
+			if (ret < 0) {
+				pr_err("fill cu for cpu failed\n");
+				return ret;
+			}
+			crat_table->length += sub_type_hdr->length;
+			crat_table->total_entries++;
 
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-			sub_type_hdr->length);
+			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+					sub_type_hdr->length);
+		}
 
 		/* Fill in Subtype: Memory */
 		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
 			crat_table->num_domains,
 			(struct crat_subtype_memory *)sub_type_hdr);
-		if (ret < 0)
+		if (ret < 0) {
+			pr_err("fill mem for cpu failed\n");
 			return ret;
+		}
 		crat_table->length += sub_type_hdr->length;
 		crat_table->total_entries++;
 
 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
 			sub_type_hdr->length);
 
-		/* Fill in Subtype: IO Link */
+		if (kfd_numa_node_to_apic_id(numa_node_id) != -1) {
+			/* Fill in Subtype: IO Link */
 #ifdef CONFIG_X86_64
-		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
-				&entries,
-				(struct crat_subtype_iolink *)sub_type_hdr);
-		if (ret < 0)
-			return ret;
+			ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
+					&entries,
+					(struct crat_subtype_iolink *)sub_type_hdr);
+			if (ret < 0) {
+				pr_err("fill iolink for cpu failed\n");
+				return ret;
+			}
 
-		if (entries) {
-			crat_table->length += (sub_type_hdr->length * entries);
-			crat_table->total_entries += entries;
+			if (entries) {
+				crat_table->length += (sub_type_hdr->length * entries);
+				crat_table->total_entries += entries;
 
-			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-					sub_type_hdr->length * entries);
-		}
+				sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+						sub_type_hdr->length * entries);
+			}
 #else
-		pr_info("IO link not available for non x86 platforms\n");
+			pr_info("IO link not available for non x86 platforms\n");
 #endif
+		}
 
 		crat_table->num_domains++;
 	}
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#1 lore raw

PATCH 2/4 drm/amdkfd: Don't create crat memory sub-table if node has no memory

In some configuration, there is CPU-only (no memory) numa node. Don't create crat memory sub-table for such node.

---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +11 -6
index 56e6dff..dd7772c 100644
@@ -1583,7 +1583,8 @@ static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
  *	@avail_size: Available size in the memory
  *	@sub_type_hdr: Memory into which compute info will be filled in
  *
- *	Return 0 if successful else return -ve value
+ *	Return memory size in bytes if successful else return -ve value
+ *	Returning 0 means this numa node has no memory
  */
 static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
 			int proximity_domain,
@@ -1619,7 +1620,7 @@ static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
 	sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
 	sub_type_hdr->proximity_domain = proximity_domain;
 
-	return 0;
+	return mem_in_bytes;
 }
 
 #ifdef CONFIG_X86_64
@@ -1746,11 +1747,15 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
 			pr_err("fill mem for cpu failed\n");
 			return ret;
 		}
-		crat_table->length += sub_type_hdr->length;
-		crat_table->total_entries++;
 
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-			sub_type_hdr->length);
+		/* ret == 0: this node has no memory */
+		if (ret > 0) {
+			crat_table->length += sub_type_hdr->length;
+			crat_table->total_entries++;
+
+			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+					sub_type_hdr->length);
+		}
 
 		if (kfd_numa_node_to_apic_id(numa_node_id) != -1) {
 			/* Fill in Subtype: IO Link */
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#2 lore raw

PATCH 3/4 drm/amdkfd: Use simd_count to determine whether it is a GPU node

In reply to Oak Zeng

Previously we used cpu_cores_count==0 to determine whether a node is a GPU node. This is not correct any more since we introduced memory only numa node. For memory only numa node, cpu_cores_count is also 0 but it is not a GPU node.

---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -1
index dd7772c..87226d5 100644
@@ -835,7 +835,7 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 	list_for_each_entry(dev, device_list, list) {
 		if (mem->proximity_domain == dev->proximity_domain) {
 			/* We're on GPU node */
-			if (dev->node_props.cpu_cores_count == 0) {
+			if (dev->node_props.simd_count != 0) {
 				/* APU */
 				if (mem->visibility_type == 0)
 					heap_type =
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#3 lore raw

PATCH 4/4 drm/amdkfd: Patch memory parameters for all CPU nodes

In reply to Oak Zeng

There can be more than one CPU nodes. Patch memory width and clock info for all CPU nodes, instead of only the first one.

---
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_topology.c +3 -3
index 9be66ba..e982829 100644
@@ -1073,10 +1073,10 @@ int kfd_topology_init(void)
 	if (cpu_only_node) {
 		/* Add additional information to CPU only node created above */
 		down_write(&topology_lock);
-		kdev = list_first_entry(&topology_device_list,
-				struct kfd_topology_device, list);
+		list_for_each_entry(kdev, &topology_device_list, list) {
+			kfd_add_non_crat_information(kdev);
+		}
 		up_write(&topology_lock);
-		kfd_add_non_crat_information(kdev);
 	}
 
 err:
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
maintainer
#4 lore raw
In reply to Oak Zeng

Am 2021-06-01 um 3:07 p.m. schrieb Oak Zeng:

Quoted 2 lines

Previously kfd driver assumes all CPU numa nodes are associated with system memory

This is not true. Systems where there is no memory installed in one or more CPU sockets have been supported. The only thing that was not supported is NUMA domains with memory but no CPU (or GPU).

Quoted 46 lines

and there is no memory only numa node. This
is not true anymore for ALDEBARAN A+A set up. This patch creates memory only node in kfd sysfs.

---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 73 ++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 31 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +6 -8
index 3251fe2..56e6dff 100644
@@ -827,8 +827,11 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 	uint32_t flags = 0;
 	uint32_t width;
 
-	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
-			mem->proximity_domain);
+	size_in_bytes =
+		((uint64_t)mem->length_high << 32) +
+		mem->length_low;
+	pr_debug("Found memory entry in CRAT table with proximity_domain=%d, size %lld\n",
+			mem->proximity_domain, size_in_bytes);
 	list_for_each_entry(dev, device_list, list) {
 		if (mem->proximity_domain == dev->proximity_domain) {
 			/* We're on GPU node */
@@ -848,9 +851,6 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
 				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
 
-			size_in_bytes =
-				((uint64_t)mem->length_high << 32) +
-							mem->length_low;
 			width = mem->width;
 
 			/* Multiple banks of the same type are aggregated into
@@ -1718,51 +1718,62 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
 
 	for_each_online_node(numa_node_id) {
+		pr_debug("numa node id %d\n", numa_node_id);
 		if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
-			continue;
-
-		/* Fill in Subtype: Compute Unit */

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) · line 1725

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -7
-		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
-			crat_table->num_domains,
-			(struct crat_subtype_computeunit *)sub_type_hdr);
-		if (ret < 0)
-			return ret;
-		crat_table->length += sub_type_hdr->length;
-		crat_table->total_entries++;
+			pr_debug("Numa node %d is a memory only numa node\n", numa_node_id);
+
+		if (kfd_numa_node_to_apic_id(numa_node_id) != -1) {

This should be an else-branch of the previous "if".

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +10 -0
+			/* Fill in Subtype: Compute Unit */
+			ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
+					crat_table->num_domains,
+					(struct crat_subtype_computeunit *)sub_type_hdr);
+			if (ret < 0) {
+				pr_err("fill cu for cpu failed\n");
+				return ret;
+			}
+			crat_table->length += sub_type_hdr->length;
+			crat_table->total_entries++;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) · line 1743

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -2
 
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-			sub_type_hdr->length);
+			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+					sub_type_hdr->length);
+		}
 
 		/* Fill in Subtype: Memory */
 		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
 			crat_table->num_domains,

So you're creating a dangling memory object that's not associated with any node and therefore has no IO links? I'm not sure how that will work. I was expecting that there would be at least a dummy CPU node with 0 cores but with IO links to express the NUMA distances.

Regards,
  Felix

Quoted 40 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +16 -13
 			(struct crat_subtype_memory *)sub_type_hdr);
-		if (ret < 0)
+		if (ret < 0) {
+			pr_err("fill mem for cpu failed\n");
 			return ret;
+		}
 		crat_table->length += sub_type_hdr->length;
 		crat_table->total_entries++;
 
 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
 			sub_type_hdr->length);
 
-		/* Fill in Subtype: IO Link */
+		if (kfd_numa_node_to_apic_id(numa_node_id) != -1) {
+			/* Fill in Subtype: IO Link */
 #ifdef CONFIG_X86_64
-		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
-				&entries,
-				(struct crat_subtype_iolink *)sub_type_hdr);
-		if (ret < 0)
-			return ret;
+			ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
+					&entries,
+					(struct crat_subtype_iolink *)sub_type_hdr);
+			if (ret < 0) {
+				pr_err("fill iolink for cpu failed\n");
+				return ret;
+			}
 
-		if (entries) {
-			crat_table->length += (sub_type_hdr->length * entries);
-			crat_table->total_entries += entries;
+			if (entries) {
+				crat_table->length += (sub_type_hdr->length * entries);
+				crat_table->total_entries += entries;
 
-			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-					sub_type_hdr->length * entries);
-		}
+				sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) · line 1779

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +4 -1
+						sub_type_hdr->length * entries);
+			}
 #else
-		pr_info("IO link not available for non x86 platforms\n");
+			pr_info("IO link not available for non x86 platforms\n");
 #endif
+		}
 
 		crat_table->num_domains++;
 	}

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx