Skip to contentSkip to conversation

powerpc/mm: Fix node look up with numa=off boot

PATCH In review Aneesh Kumar K.V 4 participants 11 messages Jun 29, 2019 – Jul 9, 2019 PowerPC

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 powerpc/mm: Fix node look up with numa=off boot

If we boot with numa=off, we need to make sure we return NUMA_NO_NODE when looking up associativity details of resources. Without this, we hit crash like below

BUG: Unable to handle kernel data access at 0x40000000008
Faulting instruction address: 0xc000000008f31704
cpu 0x1b: Vector: 380 (Data SLB Access) at [c00000000b9bb320]
    pc: c000000008f31704: _raw_spin_lock+0x14/0x100
    lr: c0000000083f41fc: ____cache_alloc_node+0x5c/0x290
    sp: c00000000b9bb5b0

msr: 800000010280b033
dar: 40000000008
current = 0xc00000000b9a2700
paca = 0xc00000000a740c00 irqmask: 0x03 irq_happened: 0x01
pid = 1, comm = swapper/27
Linux version 5.2.0-rc4-00925-g74e188c620b1 (root@linux-d8ip) (gcc version 7.4.1 20190424 [gcc-7-branch revision 270538] (SUSE Linux)) #34 SMP Sat Jun 29 00:41:02 EDT 2019 enter ? for help
[link register ] c0000000083f41fc ____cache_alloc_node+0x5c/0x290
[c00000000b9bb5b0] 0000000000000dc0 (unreliable)
[c00000000b9bb5f0] c0000000083f48c8 kmem_cache_alloc_node_trace+0x138/0x360
[c00000000b9bb670] c000000008aa789c devres_alloc_node+0x4c/0xa0
[c00000000b9bb6a0] c000000008337218 devm_memremap+0x58/0x130
[c00000000b9bb6f0] c000000008aed00c devm_nsio_enable+0xdc/0x170
[c00000000b9bb780] c000000008af3b6c nd_pmem_probe+0x4c/0x180
[c00000000b9bb7b0] c000000008ad84cc nvdimm_bus_probe+0xac/0x260
[c00000000b9bb840] c000000008aa0628 really_probe+0x148/0x500
[c00000000b9bb8d0] c000000008aa0d7c driver_probe_device+0x19c/0x1d0
[c00000000b9bb950] c000000008aa11bc device_driver_attach+0xcc/0x100
[c00000000b9bb990] c000000008aa12ec __driver_attach+0xfc/0x1e0
[c00000000b9bba10] c000000008a9d0a4 bus_for_each_dev+0xb4/0x130
[c00000000b9bba70] c000000008a9fc04 driver_attach+0x34/0x50
[c00000000b9bba90] c000000008a9f118 bus_add_driver+0x1d8/0x300
[c00000000b9bbb20] c000000008aa2358 driver_register+0x98/0x1a0
[c00000000b9bbb90] c000000008ad7e6c __nd_driver_register+0x5c/0x100
[c00000000b9bbbf0] c0000000093efbac nd_pmem_driver_init+0x34/0x48
[c00000000b9bbc10] c0000000080106c0 do_one_initcall+0x60/0x2d0
[c00000000b9bbce0] c00000000938463c kernel_init_freeable+0x384/0x48c
[c00000000b9bbdb0] c000000008010a5c kernel_init+0x2c/0x160
[c00000000b9bbe20] c00000000800ba54 ret_from_kernel_thread+0x5c/0x68

---
 arch/powerpc/mm/numa.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
arch/powerpc/mm/numa.c +6 -2
index 917904d2fe97..f6d68baeaa96 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
 {
 	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1)
+	if (min_common_depth == -1 || !numa_enabled)
 		goto out;
 
 	if (of_read_number(associativity, 1) >= min_common_depth)
@@ -416,10 +416,14 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)
 static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
 {
 	struct assoc_arrays aa = { .arrays = NULL };
+	/* is that correct? */
 	int default_nid = 0;
 	int nid = default_nid;
 	int rc, index;
 
+	if (!numa_enabled)
+		return NUMA_NO_NODE;
+
 	rc = of_get_assoc_arrays(&aa);
 	if (rc)
 		return default_nid;
@@ -808,7 +812,7 @@ static void __init find_possible_nodes(void)
 	struct device_node *rtas;
 	u32 numnodes, i;
 
-	if (min_common_depth <= 0)
+	if (min_common_depth <= 0 || !numa_enabled)
 		return;
 
 	rtas = of_find_node_by_path("/rtas");
Signature · 1 line
2.21.0
#1 lore raw

PATCH 2/2 powerpc/mm: Conslidate numa_enable check and min_common_depth check

Update min_common_depth = -1 if numa is disabled. This help us to avoid checking for both in different code paths.

---
 arch/powerpc/mm/numa.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/mm/numa.c +5 -4
index f6d68baeaa96..c84062a390cc 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
 {
 	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
 		goto out;
 
 	if (of_read_number(associativity, 1) >= min_common_depth)
@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)
 
 	if (numa_enabled == 0) {
 		printk(KERN_WARNING "NUMA disabled by user\n");
+		min_common_depth = -1;
 		return -1;
 	}
 
@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)
 	unsigned int node;
 	unsigned int cpu, count;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
 		return;
 
 	for_each_online_node(node) {
@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)
 	struct device_node *rtas;
 	u32 numnodes, i;
 
-	if (min_common_depth <= 0 || !numa_enabled)
+	if (min_common_depth <= 0)
 		return;
 
 	rtas = of_find_node_by_path("/rtas");
@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)
 	struct device_node *memory = NULL;
 	int nid;
 
-	if (!numa_enabled || (min_common_depth < 0))
+	if (min_common_depth < 0)
 		return first_online_node;
 
 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
Signature · 1 line
2.21.0
#2 lore raw

Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check

Quoted 49 lines · Aneesh Kumar K.V

Update min_common_depth = -1 if numa is disabled. This help us to avoid checking for both in different code paths.

---
  arch/powerpc/mm/numa.c | 9 +++++----
  1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/mm/numa.c +4 -3
index f6d68baeaa96..c84062a390cc 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
  {
  	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
  		goto out;
 
  	if (of_read_number(associativity, 1) >= min_common_depth)
@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)
 
  	if (numa_enabled == 0) {
  		printk(KERN_WARNING "NUMA disabled by user\n");
+		min_common_depth = -1;
  		return -1;
  	}
 
@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)
  	unsigned int node;
  	unsigned int cpu, count;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
  		return;
 
  	for_each_online_node(node) {
@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)
  	struct device_node *rtas;
  	u32 numnodes, i;
 
-	if (min_common_depth <= 0 || !numa_enabled)
+	if (min_common_depth <= 0)
  		return;
 
  	rtas = of_find_node_by_path("/rtas");

arch/powerpc/mm/numa.c int hot_add_scn_to_nid(unsigned long scn_addr) · line 1021

Quoted 8 lines
arch/powerpc/mm/numa.c +1 -1
@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)
  	struct device_node *memory = NULL;
  	int nid;
 
-	if (!numa_enabled || (min_common_depth < 0))
+	if (min_common_depth < 0)
  		return first_online_node;
 
  	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");

I was not sure whether a reverse switch if better so that we have

if (!numa_enabled) check every where and we do the below

@@ -625,14 +624,15 @@ static int __init parse_numa_properties(void)

if (numa_enabled == 0) {
printk(KERN_WARNING "NUMA disabled by user\n");
- min_common_depth = -1;
return -1;
}

min_common_depth = find_min_common_depth();

- if (min_common_depth < 0)
+ if (min_common_depth < 0) {
+ numa_enabled = false;
return min_common_depth;
+ }

-aneesh

#3 lore raw

Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check

In reply to Aneesh Kumar K.V
Quoted 22 lines · Aneesh Kumar K.V

Update min_common_depth = -1 if numa is disabled. This help us to avoid checking for both in different code paths.

---
 arch/powerpc/mm/numa.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/mm/numa.c +1 -1
index f6d68baeaa96..c84062a390cc 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
 {
 	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
 		goto out;
 
 	if (of_read_number(associativity, 1) >= min_common_depth)

arch/powerpc/mm/numa.c static int __init parse_numa_properties(void) · line 630

Quoted 6 lines
arch/powerpc/mm/numa.c +1 -0
@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)
 
 	if (numa_enabled == 0) {
 		printk(KERN_WARNING "NUMA disabled by user\n");
+		min_common_depth = -1;
 		return -1;
 	}

I would prefer updating the definition of variable 'min_common_depth' to

static int min_common_depth = -1;

This would handle the case where someone calls 'associativity_to_nid()' and other functions that read 'min_common_depth' and get an invalid result back. And also handle the case where kernel is booted with 'numa = off'.

Also the init value 'min_common_depth == 0' indicates that the first word in "ibm,associativity" array represents the node-id which is wrong. Instead its the length of the "ibm,associativity" array.

Quoted 31 lines
arch/powerpc/mm/numa.c +3 -3
@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)
 	unsigned int node;
 	unsigned int cpu, count;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
 		return;
 
 	for_each_online_node(node) {
@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)
 	struct device_node *rtas;
 	u32 numnodes, i;
 
-	if (min_common_depth <= 0 || !numa_enabled)
+	if (min_common_depth <= 0)
 		return;
 
 	rtas = of_find_node_by_path("/rtas");
@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)
 	struct device_node *memory = NULL;
 	int nid;
 
-	if (!numa_enabled || (min_common_depth < 0))
+	if (min_common_depth < 0)
 		return first_online_node;
 
 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
Signature · 1 line
2.21.0
Signature · 2 lines
Vaibhav Jain <[email protected]>
Linux Technology Center, IBM India Pvt. Ltd.
#4 lore raw

Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check

Quoted 33 lines · Vaibhav Jain
Quoted 22 lines · Aneesh Kumar K.V

Update min_common_depth = -1 if numa is disabled. This help us to avoid checking for both in different code paths.

---
  arch/powerpc/mm/numa.c | 9 +++++----
  1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/mm/numa.c +1 -1
index f6d68baeaa96..c84062a390cc 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
  {
  	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1 || !numa_enabled)
+	if (min_common_depth == -1)
  		goto out;
 
  	if (of_read_number(associativity, 1) >= min_common_depth)

arch/powerpc/mm/numa.c static int __init parse_numa_properties(void) · line 630

Quoted 6 lines
arch/powerpc/mm/numa.c +1 -0
@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)
 
  	if (numa_enabled == 0) {
  		printk(KERN_WARNING "NUMA disabled by user\n");
+		min_common_depth = -1;
  		return -1;
  	}

I would prefer updating the definition of variable 'min_common_depth' to

Quoted 6 lines

static int min_common_depth = -1;

This would handle the case where someone calls 'associativity_to_nid()' and other functions that read 'min_common_depth' and get an invalid result back. And also handle the case where kernel is booted with 'numa = off'.

Sure. As mentioned in another email, I am wondering whether all that min_common_depth check should be if !numa_enabled. That makes it much easy to read. I will respin once i get more clarity on of_drconf_to_nid_single usage.

Quoted 4 lines

Also the init value 'min_common_depth == 0' indicates that the first word in "ibm,associativity" array represents the node-id which is wrong. Instead its the length of the "ibm,associativity" array.

-aneesh

#5 lore raw
In reply to Aneesh Kumar K.V
Quoted 61 lines · Aneesh Kumar K.V

If we boot with numa=off, we need to make sure we return NUMA_NO_NODE when looking up associativity details of resources. Without this, we hit crash like below

BUG: Unable to handle kernel data access at 0x40000000008
Faulting instruction address: 0xc000000008f31704
cpu 0x1b: Vector: 380 (Data SLB Access) at [c00000000b9bb320]
     pc: c000000008f31704: _raw_spin_lock+0x14/0x100
     lr: c0000000083f41fc: ____cache_alloc_node+0x5c/0x290
     sp: c00000000b9bb5b0
    msr: 800000010280b033
    dar: 40000000008

current = 0xc00000000b9a2700
paca = 0xc00000000a740c00 irqmask: 0x03 irq_happened: 0x01
pid = 1, comm = swapper/27
Linux version 5.2.0-rc4-00925-g74e188c620b1 (root@linux-d8ip) (gcc version 7.4.1 20190424 [gcc-7-branch revision 270538] (SUSE Linux)) #34 SMP Sat Jun 29 00:41:02 EDT 2019 enter ? for help
[link register ] c0000000083f41fc ____cache_alloc_node+0x5c/0x290
[c00000000b9bb5b0] 0000000000000dc0 (unreliable)
[c00000000b9bb5f0] c0000000083f48c8 kmem_cache_alloc_node_trace+0x138/0x360
[c00000000b9bb670] c000000008aa789c devres_alloc_node+0x4c/0xa0
[c00000000b9bb6a0] c000000008337218 devm_memremap+0x58/0x130
[c00000000b9bb6f0] c000000008aed00c devm_nsio_enable+0xdc/0x170
[c00000000b9bb780] c000000008af3b6c nd_pmem_probe+0x4c/0x180
[c00000000b9bb7b0] c000000008ad84cc nvdimm_bus_probe+0xac/0x260
[c00000000b9bb840] c000000008aa0628 really_probe+0x148/0x500
[c00000000b9bb8d0] c000000008aa0d7c driver_probe_device+0x19c/0x1d0
[c00000000b9bb950] c000000008aa11bc device_driver_attach+0xcc/0x100
[c00000000b9bb990] c000000008aa12ec __driver_attach+0xfc/0x1e0
[c00000000b9bba10] c000000008a9d0a4 bus_for_each_dev+0xb4/0x130
[c00000000b9bba70] c000000008a9fc04 driver_attach+0x34/0x50
[c00000000b9bba90] c000000008a9f118 bus_add_driver+0x1d8/0x300
[c00000000b9bbb20] c000000008aa2358 driver_register+0x98/0x1a0
[c00000000b9bbb90] c000000008ad7e6c __nd_driver_register+0x5c/0x100
[c00000000b9bbbf0] c0000000093efbac nd_pmem_driver_init+0x34/0x48
[c00000000b9bbc10] c0000000080106c0 do_one_initcall+0x60/0x2d0
[c00000000b9bbce0] c00000000938463c kernel_init_freeable+0x384/0x48c
[c00000000b9bbdb0] c000000008010a5c kernel_init+0x2c/0x160
[c00000000b9bbe20] c00000000800ba54 ret_from_kernel_thread+0x5c/0x68

---
  arch/powerpc/mm/numa.c | 8 ++++++--
  1 file changed, 6 insertions(+), 2 deletions(-)
arch/powerpc/mm/numa.c +1 -1
index 917904d2fe97..f6d68baeaa96 100644
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity)
  {
  	int nid = NUMA_NO_NODE;
 
-	if (min_common_depth == -1)
+	if (min_common_depth == -1 || !numa_enabled)
  		goto out;
 
  	if (of_read_number(associativity, 1) >= min_common_depth)
@@ -416,10 +416,14 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)
  static int of_drconf_to_nid_single(struct drmem_lmb *lmb)

arch/powerpc/mm/numa.c static int of_get_assoc_arrays(struct assoc_arrays *aa) · line 426

Quoted 10 lines
arch/powerpc/mm/numa.c +4 -0
  {
  	struct assoc_arrays aa = { .arrays = NULL };
+	/* is that correct? */
  	int default_nid = 0;
  	int nid = default_nid;
  	int rc, index;
 
+	if (!numa_enabled)
+		return NUMA_NO_NODE;
+

I guess we should have here.

modified arch/powerpc/mm/numa.c
@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)
static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
{
struct assoc_arrays aa = { .arrays = NULL };
- /* is that correct? */
int default_nid = 0;
int nid = default_nid;
int rc, index;

- if (!numa_enabled)
+ if ((min_common_depth < 0) || !numa_enabled)
return NUMA_NO_NODE;

rc = of_get_assoc_arrays(&aa);

Nathan,

Can you check this?

arch/powerpc/mm/numa.c static void __init find_possible_nodes(void) · line 818

Quoted 13 lines
arch/powerpc/mm/numa.c +1 -1
  	rc = of_get_assoc_arrays(&aa);
  	if (rc)
  		return default_nid;
@@ -808,7 +812,7 @@ static void __init find_possible_nodes(void)
  	struct device_node *rtas;
  	u32 numnodes, i;
 
-	if (min_common_depth <= 0)
+	if (min_common_depth <= 0 || !numa_enabled)
  		return;
 
  	rtas = of_find_node_by_path("/rtas");

-aneesh

#6 lore raw
Quoted 20 lines · Aneesh Kumar K.V

I guess we should have here.

modified arch/powerpc/mm/numa.c

arch/powerpc/mm/numa.c +1 -2
@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays 
 *aa)
  static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
  {
  	struct assoc_arrays aa = { .arrays = NULL };
-	/* is that correct? */
  	int default_nid = 0;
  	int nid = default_nid;
  	int rc, index;
 
-	if (!numa_enabled)
+	if ((min_common_depth < 0) || !numa_enabled)
  		return NUMA_NO_NODE;
 
  	rc = of_get_assoc_arrays(&aa);
Quoted 3 lines

Nathan,

Can you check this?

Looks like it would do the right thing.

Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)

#7 lore raw
Quoted 21 lines · Nathan Lynch
Quoted 20 lines · Aneesh Kumar K.V

I guess we should have here.

modified arch/powerpc/mm/numa.c

arch/powerpc/mm/numa.c +1 -2
@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays
 *aa)
   static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
   {
   	struct assoc_arrays aa = { .arrays = NULL };
-	/* is that correct? */
   	int default_nid = 0;
   	int nid = default_nid;
   	int rc, index;
 
-	if (!numa_enabled)
+	if ((min_common_depth < 0) || !numa_enabled)
   		return NUMA_NO_NODE;
 
   	rc = of_get_assoc_arrays(&aa);
Quoted 9 lines
Quoted 3 lines

Nathan,

Can you check this?

Looks like it would do the right thing.

Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)

That is used in kdump kernel.

-aneesh

#8 lore raw
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines

Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)

That is used in kdump kernel.

I see, thanks.

#9 lore raw
Quoted 8 lines · Nathan Lynch
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines

Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)

That is used in kdump kernel.

I see, thanks.

That doesn't mean it's a good idea :)

Does it actually reduce memory usage much? Last time I dug into the kdump kernel's usage of weird command line flags none of them really did anything useful.

cheers

#10 lore raw
Quoted 8 lines · Michael Ellerman
Quoted 7 lines · Nathan Lynch
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines

Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)

That is used in kdump kernel.

Quoted 7 lines
Quoted 1 line

I see, thanks.

That doesn't mean it's a good idea :)

Does it actually reduce memory usage much? Last time I dug into the kdump kernel's usage of weird command line flags none of them really did anything useful.

I think it's intended to work around bugs in numa initialization, e.g.

https://www.suse.com/support/kb/doc/?id=7023399

Hopefully the original bug with numa/kdump interaction has been fixed?