Skip to contentSkip to conversation

kho: skip memoryless NUMA nodes when reserving scratch areas

PATCH Superseded Evangelos Petrongonas 3 participants 3 messages Jan 16 – Jan 18

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 kho: skip memoryless NUMA nodes when reserving scratch areas

kho_reserve_scratch() iterates over all online NUMA nodes to allocate per-node scratch memory. On systems with memoryless NUMA nodes (nodes that have CPUs but no memory), memblock_alloc_range_nid() fails because there is no memory available on that node. This causes KHO initialization to fail and kho_enable to be set to false.

Some ARM64 systems have NUMA topologies where certain nodes contain only CPUs without any associated memory. These configurations are valid and should not prevent KHO from functioning.

Fix this by introducing kho_mem_nodes_count() which counts only nodes that have memory (N_MEMORY state), and skip memoryless nodes in the per-node scratch allocation loop.

---
 kernel/liveupdate/kexec_handover.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)
kernel/liveupdate/kexec_handover.c +22 -1
index 9dc51fab604f..c970ed08b477 100644
@@ -623,6 +623,23 @@ static phys_addr_t __init scratch_size_node(int nid)
 	return round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 }
 
+/*
+ * Count online NUMA nodes that have memory. Memoryless nodes cannot have
+ * scratch memory and should be excluded.
+ */
+static unsigned int __init kho_mem_nodes_count(void)
+{
+	unsigned int cnt = 0;
+	int nid;
+
+	for_each_online_node(nid) {
+		if (node_state(nid, N_MEMORY))
+			cnt++;
+	}
+
+	return cnt;
+}
+
 /**
  * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
  *
@@ -643,7 +660,7 @@ static void __init kho_reserve_scratch(void)
 	scratch_size_update();
 
 	/* FIXME: deal with node hot-plug/remove */
-	kho_scratch_cnt = num_online_nodes() + 2;
+	kho_scratch_cnt = kho_mem_nodes_count() + 2;
 	size = kho_scratch_cnt * sizeof(*kho_scratch);
 	kho_scratch = memblock_alloc(size, PAGE_SIZE);
 	if (!kho_scratch)
@@ -674,6 +691,10 @@ static void __init kho_reserve_scratch(void)
 	i++;
 
 	for_each_online_node(nid) {
+		/* Skip memoryless nodes - we cannot allocate scratch memory there */
+		if (!node_state(nid, N_MEMORY))
+			continue;
+
 		size = scratch_size_node(nid);
 		addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES,
 						0, MEMBLOCK_ALLOC_ACCESSIBLE,
Signature · 12 lines
2.43.0




Amazon Web Services Development Center Germany GmbH
Tamara-Danz-Str. 13
10243 Berlin
Geschaeftsfuehrung: Christof Hellmis, Andreas Stieger
Eingetragen am Amtsgericht Charlottenburg unter HRB 257764 B
Sitz: Berlin
Ust-ID: DE 365 538 597
maintainer
#1 lore raw

Hi Evangelos,

Quoted 34 lines · Evangelos Petrongonas

kho_reserve_scratch() iterates over all online NUMA nodes to allocate per-node scratch memory. On systems with memoryless NUMA nodes (nodes that have CPUs but no memory), memblock_alloc_range_nid() fails because there is no memory available on that node. This causes KHO initialization to fail and kho_enable to be set to false.

Some ARM64 systems have NUMA topologies where certain nodes contain only CPUs without any associated memory. These configurations are valid and should not prevent KHO from functioning.

Fix this by introducing kho_mem_nodes_count() which counts only nodes that have memory (N_MEMORY state), and skip memoryless nodes in the per-node scratch allocation loop.

---
 kernel/liveupdate/kexec_handover.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)
kernel/liveupdate/kexec_handover.c +7 -0
index 9dc51fab604f..c970ed08b477 100644
@@ -623,6 +623,23 @@ static phys_addr_t __init scratch_size_node(int nid)
 	return round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 }
 
+/*
+ * Count online NUMA nodes that have memory. Memoryless nodes cannot have
+ * scratch memory and should be excluded.
+ */
+static unsigned int __init kho_mem_nodes_count(void)
+{
+	unsigned int cnt = 0;

kernel/liveupdate/kexec_handover.c static phys_addr_t __init scratch_size_node(int nid) · line 642

Quoted 10 lines
kernel/liveupdate/kexec_handover.c +10 -0
+	int nid;
+
+	for_each_online_node(nid) {
+		if (node_state(nid, N_MEMORY))
+			cnt++;
+	}
+
+	return cnt;
+}
+

You don't need this. You can use nodes_weight(nodes_state[N_MEMORY]) directly. Other than this, LGTM.

Quoted 23 lines
kernel/liveupdate/kexec_handover.c +5 -1
 /**
  * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
  *
@@ -643,7 +660,7 @@ static void __init kho_reserve_scratch(void)
 	scratch_size_update();
 
 	/* FIXME: deal with node hot-plug/remove */
-	kho_scratch_cnt = num_online_nodes() + 2;
+	kho_scratch_cnt = kho_mem_nodes_count() + 2;
 	size = kho_scratch_cnt * sizeof(*kho_scratch);
 	kho_scratch = memblock_alloc(size, PAGE_SIZE);
 	if (!kho_scratch)
@@ -674,6 +691,10 @@ static void __init kho_reserve_scratch(void)
 	i++;
 
 	for_each_online_node(nid) {
+		/* Skip memoryless nodes - we cannot allocate scratch memory there */
+		if (!node_state(nid, N_MEMORY))
+			continue;
+
 		size = scratch_size_node(nid);
 		addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES,
 						0, MEMBLOCK_ALLOC_ACCESSIBLE,
Signature · 2 lines
Regards,
Pratyush Yadav
maintainer
#2 lore raw
Quoted 67 lines · Pratyush Yadav

Hi Evangelos,

Quoted 34 lines · Evangelos Petrongonas

kho_reserve_scratch() iterates over all online NUMA nodes to allocate per-node scratch memory. On systems with memoryless NUMA nodes (nodes that have CPUs but no memory), memblock_alloc_range_nid() fails because there is no memory available on that node. This causes KHO initialization to fail and kho_enable to be set to false.

Some ARM64 systems have NUMA topologies where certain nodes contain only CPUs without any associated memory. These configurations are valid and should not prevent KHO from functioning.

Fix this by introducing kho_mem_nodes_count() which counts only nodes that have memory (N_MEMORY state), and skip memoryless nodes in the per-node scratch allocation loop.

---
 kernel/liveupdate/kexec_handover.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)
kernel/liveupdate/kexec_handover.c +7 -0
index 9dc51fab604f..c970ed08b477 100644
@@ -623,6 +623,23 @@ static phys_addr_t __init scratch_size_node(int nid)
 	return round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 }
 
+/*
+ * Count online NUMA nodes that have memory. Memoryless nodes cannot have
+ * scratch memory and should be excluded.
+ */
+static unsigned int __init kho_mem_nodes_count(void)
+{
+	unsigned int cnt = 0;

kernel/liveupdate/kexec_handover.c static phys_addr_t __init scratch_size_node(int nid) · line 642

Quoted 10 lines
kernel/liveupdate/kexec_handover.c +10 -0
+	int nid;
+
+	for_each_online_node(nid) {
+		if (node_state(nid, N_MEMORY))
+			cnt++;
+	}
+
+	return cnt;
+}
+

You don't need this. You can use nodes_weight(nodes_state[N_MEMORY]) directly. Other than this, LGTM.

Quoted 15 lines
kernel/liveupdate/kexec_handover.c +1 -1
 /**
  * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
  *
@@ -643,7 +660,7 @@ static void __init kho_reserve_scratch(void)
 	scratch_size_update();
 
 	/* FIXME: deal with node hot-plug/remove */
-	kho_scratch_cnt = num_online_nodes() + 2;
+	kho_scratch_cnt = kho_mem_nodes_count() + 2;
 	size = kho_scratch_cnt * sizeof(*kho_scratch);
 	kho_scratch = memblock_alloc(size, PAGE_SIZE);
 	if (!kho_scratch)
@@ -674,6 +691,10 @@ static void __init kho_reserve_scratch(void)
 	i++;
 
Quoted 5 lines

kernel/liveupdate/kexec_handover.c static void __init kho_reserve_scratch(void) · line 697

Quoted 5 lines
kernel/liveupdate/kexec_handover.c +4 -0
 	for_each_online_node(nid) {
+		/* Skip memoryless nodes - we cannot allocate scratch memory there */
+		if (!node_state(nid, N_MEMORY))
+			continue;
+

And here you can use for_each_node_state(nid, N_MEMORY)

Quoted 7 lines
Quoted 3 lines
kernel/liveupdate/kexec_handover.c +0 -0
 		size = scratch_size_node(nid);
 		addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES,
 						0, MEMBLOCK_ALLOC_ACCESSIBLE,
Signature · 2 lines
Regards,
Pratyush Yadav
Signature · 2 lines
Sincerely yours,
Mike.