powerpc/mm: Fix node look up with numa=off boot
PATCH 1/2 powerpc/mm: Fix node look up with numa=off boot
If we boot with numa=off, we need to make sure we return NUMA_NO_NODE when looking up associativity details of resources. Without this, we hit crash like below
BUG: Unable to handle kernel data access at 0x40000000008
Faulting instruction address: 0xc000000008f31704
cpu 0x1b: Vector: 380 (Data SLB Access) at [c00000000b9bb320]
pc: c000000008f31704: _raw_spin_lock+0x14/0x100
lr: c0000000083f41fc: ____cache_alloc_node+0x5c/0x290
sp: c00000000b9bb5b0 msr: 800000010280b033
dar: 40000000008
current = 0xc00000000b9a2700
paca = 0xc00000000a740c00 irqmask: 0x03 irq_happened: 0x01
pid = 1, comm = swapper/27
Linux version 5.2.0-rc4-00925-g74e188c620b1 (root@linux-d8ip) (gcc version 7.4.1 20190424 [gcc-7-branch revision 270538] (SUSE Linux)) #34 SMP Sat Jun 29 00:41:02 EDT 2019 enter ? for help
[link register ] c0000000083f41fc ____cache_alloc_node+0x5c/0x290
[c00000000b9bb5b0] 0000000000000dc0 (unreliable)
[c00000000b9bb5f0] c0000000083f48c8 kmem_cache_alloc_node_trace+0x138/0x360
[c00000000b9bb670] c000000008aa789c devres_alloc_node+0x4c/0xa0
[c00000000b9bb6a0] c000000008337218 devm_memremap+0x58/0x130
[c00000000b9bb6f0] c000000008aed00c devm_nsio_enable+0xdc/0x170
[c00000000b9bb780] c000000008af3b6c nd_pmem_probe+0x4c/0x180
[c00000000b9bb7b0] c000000008ad84cc nvdimm_bus_probe+0xac/0x260
[c00000000b9bb840] c000000008aa0628 really_probe+0x148/0x500
[c00000000b9bb8d0] c000000008aa0d7c driver_probe_device+0x19c/0x1d0
[c00000000b9bb950] c000000008aa11bc device_driver_attach+0xcc/0x100
[c00000000b9bb990] c000000008aa12ec __driver_attach+0xfc/0x1e0
[c00000000b9bba10] c000000008a9d0a4 bus_for_each_dev+0xb4/0x130
[c00000000b9bba70] c000000008a9fc04 driver_attach+0x34/0x50
[c00000000b9bba90] c000000008a9f118 bus_add_driver+0x1d8/0x300
[c00000000b9bbb20] c000000008aa2358 driver_register+0x98/0x1a0
[c00000000b9bbb90] c000000008ad7e6c __nd_driver_register+0x5c/0x100
[c00000000b9bbbf0] c0000000093efbac nd_pmem_driver_init+0x34/0x48
[c00000000b9bbc10] c0000000080106c0 do_one_initcall+0x60/0x2d0
[c00000000b9bbce0] c00000000938463c kernel_init_freeable+0x384/0x48c
[c00000000b9bbdb0] c000000008010a5c kernel_init+0x2c/0x160
[c00000000b9bbe20] c00000000800ba54 ret_from_kernel_thread+0x5c/0x68
--- arch/powerpc/mm/numa.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
arch/powerpc/mm/numa.c +6 -2
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 ) +if (min_common_depth ==-1 || !numa_enabled) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)
@@ -416,10 +416,14 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)static int of_drconf_to_nid_single (struct drmem_lmb *lmb) {struct assoc_arrays aa = { .arrays =NULL }; +/* is that correct? */ int default_nid =0 ;int nid = default_nid;int rc, index; +if (!numa_enabled) +return NUMA_NO_NODE ; + rc =of_get_assoc_arrays (&aa);if (rc)return default_nid;
@@ -808,7 +812,7 @@ static void __init find_possible_nodes(void)struct device_node *rtas;u32 numnodes, i; -if (min_common_depth <=0 ) +if (min_common_depth <=0 || !numa_enabled)return ; rtas =of_find_node_by_path ("/rtas" );
Signature · 1 line
2.21.0
PATCH 2/2 powerpc/mm: Conslidate numa_enable check and min_common_depth check
Update min_common_depth = -1 if numa is disabled. This help us to avoid checking for both in different code paths.
--- arch/powerpc/mm/numa.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/mm/numa.c +5 -4
@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 ) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)
@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)if (numa_enabled ==0 ) {printk (KERN_WARNING "NUMA disabled by user\n" ); + min_common_depth =-1 ;return -1 ; }
@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)unsigned int node;unsigned int cpu, count; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 )return ;for_each_online_node (node) {
@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)struct device_node *rtas;u32 numnodes, i; -if (min_common_depth <=0 || !numa_enabled) +if (min_common_depth <=0 )return ; rtas =of_find_node_by_path ("/rtas" );
@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)struct device_node *memory =NULL ;int nid; -if (!numa_enabled || (min_common_depth <0 )) +if (min_common_depth <0 )return first_online_node; memory =of_find_node_by_path ("/ibm,dynamic-reconfiguration-memory" );
Signature · 1 line
2.21.0
Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check
Quoted 49 lines · Aneesh Kumar K.V
Update
min_common_depth= -1 if numa is disabled. This help us to avoid checking for both in different code paths.Signed-off-by Aneesh Kumar K.V <[email protected]>--- arch/powerpc/mm/numa.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)arch/powerpc/mm/numa.c +4
-3@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 ) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)if (numa_enabled ==0 ) {printk (KERN_WARNING "NUMA disabled by user\n" ); + min_common_depth =-1 ;return -1 ; }@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)unsigned int node;unsigned int cpu, count; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 )return ;for_each_online_node (node) {@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)struct device_node *rtas;u32 numnodes, i; -if (min_common_depth <=0 || !numa_enabled) +if (min_common_depth <=0 )return ; rtas =of_find_node_by_path ("/rtas" );
arch/powerpc/mm/numa.c int hot_add_scn_to_nid(unsigned long scn_addr) · line 1021
Quoted 8 lines
arch/powerpc/mm/numa.c +1
-1@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)struct device_node *memory =NULL ;int nid; -if (!numa_enabled || (min_common_depth <0 )) +if (min_common_depth <0 )return first_online_node; memory =of_find_node_by_path ("/ibm,dynamic-reconfiguration-memory" );
I was not sure whether a reverse switch if better so that we have
if (!numa_enabled) check every where and we do the below
@@ -625,14 +624,15 @@ static int __init parse_numa_properties(void)
if (numa_enabled == 0) {
printk(KERN_WARNING "NUMA disabled by user\n");
- min_common_depth = -1;
return -1;
}
min_common_depth = find_min_common_depth();
- if (min_common_depth < 0)
+ if (min_common_depth < 0) {
+ numa_enabled = false;
return min_common_depth;
+ }
-aneesh
Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check
In reply to Aneesh Kumar K.VQuoted 22 lines · Aneesh Kumar K.V
Update
min_common_depth= -1 if numa is disabled. This help us to avoid checking for both in different code paths.Signed-off-by Aneesh Kumar K.V <[email protected]>--- arch/powerpc/mm/numa.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)arch/powerpc/mm/numa.c +1
-1@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 ) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)
arch/powerpc/mm/numa.c static int __init parse_numa_properties(void) · line 630
Quoted 6 lines
arch/powerpc/mm/numa.c +1
-0@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)if (numa_enabled ==0 ) {printk (KERN_WARNING "NUMA disabled by user\n" ); + min_common_depth =-1 ;return -1 ; }
I would prefer updating the definition of variable 'min_common_depth' to
static int min_common_depth = -1;
This would handle the case where someone calls 'associativity_to_nid()' and other functions that read 'min_common_depth' and get an invalid result back. And also handle the case where kernel is booted with 'numa = off'.
Also the init value 'min_common_depth == 0' indicates that the first word in "ibm,associativity" array represents the node-id which is wrong. Instead its the length of the "ibm,associativity" array.
Quoted 31 lines
arch/powerpc/mm/numa.c +3
-3@@ -747,7 +748,7 @@ void __init dump_numa_cpu_topology(void)unsigned int node;unsigned int cpu, count; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 )return ;for_each_online_node (node) {@@ -812,7 +813,7 @@ static void __init find_possible_nodes(void)struct device_node *rtas;u32 numnodes, i; -if (min_common_depth <=0 || !numa_enabled) +if (min_common_depth <=0 )return ; rtas =of_find_node_by_path ("/rtas" );@@ -1014,7 +1015,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)struct device_node *memory =NULL ;int nid; -if (!numa_enabled || (min_common_depth <0 )) +if (min_common_depth <0 )return first_online_node; memory =of_find_node_by_path ("/ibm,dynamic-reconfiguration-memory" );Signature · 1 line
2.21.0
Signature · 2 lines
Vaibhav Jain <[email protected]> Linux Technology Center, IBM India Pvt. Ltd.
Re: ["RFC PATCH" 2/2] powerpc/mm: Conslidate numa_enable check and min_common_depth check
Quoted 33 lines · Vaibhav Jain
Quoted 22 lines · Aneesh Kumar K.V
Update
min_common_depth= -1 if numa is disabled. This help us to avoid checking for both in different code paths.Signed-off-by Aneesh Kumar K.V <[email protected]>--- arch/powerpc/mm/numa.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)arch/powerpc/mm/numa.c +1
-1@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 || !numa_enabled) +if (min_common_depth ==-1 ) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)arch/powerpc/mm/numa.c static int __init parse_numa_properties(void) · line 630
Quoted 6 lines
arch/powerpc/mm/numa.c +1
-0@@ -625,6 +625,7 @@ static int __init parse_numa_properties(void)if (numa_enabled ==0 ) {printk (KERN_WARNING "NUMA disabled by user\n" ); + min_common_depth =-1 ;return -1 ; }I would prefer updating the definition of variable '
min_common_depth' to
Quoted 6 lines
static int
min_common_depth= -1;This would handle the case where someone calls '
associativity_to_nid()' and other functions that read 'min_common_depth' and get an invalid result back. And also handle the case where kernel is booted with 'numa = off'.
Sure. As mentioned in another email, I am wondering whether all that min_common_depth check should be if !numa_enabled. That makes it much easy to read. I will respin once i get more clarity on of_drconf_to_nid_single usage.
Quoted 4 lines
Also the init value '
min_common_depth== 0' indicates that the first word in "ibm,associativity" array represents the node-id which is wrong. Instead its the length of the "ibm,associativity" array.
-aneesh
Quoted 61 lines · Aneesh Kumar K.V
If we boot with numa=off, we need to make sure we return
NUMA_NO_NODEwhen looking up associativity details of resources. Without this, we hit crash like belowBUG: Unable to handle kernel data access at 0x40000000008 Faulting instruction address: 0xc000000008f31704 cpu 0x1b: Vector: 380 (Data SLB Access) at [c00000000b9bb320] pc: c000000008f31704: _raw_spin_lock+0x14/0x100 lr: c0000000083f41fc: ____cache_alloc_node+0x5c/0x290 sp: c00000000b9bb5b0 msr: 800000010280b033 dar: 40000000008current =
0xc00000000b9a2700
paca =0xc00000000a740c00irqmask:0x03irq_happened:0x01
pid = 1, comm = swapper/27
Linux version 5.2.0-rc4-00925-g74e188c620b1 (root@linux-d8ip) (gcc version 7.4.1 20190424 [gcc-7-branch revision 270538] (SUSE Linux)) #34 SMP Sat Jun 29 00:41:02 EDT 2019 enter ? for help
[link register ]c0000000083f41fc____cache_alloc_node+0x5c/0x290
[c00000000b9bb5b0]0000000000000dc0(unreliable)
[c00000000b9bb5f0]c0000000083f48c8kmem_cache_alloc_node_trace+0x138/0x360
[c00000000b9bb670]c000000008aa789cdevres_alloc_node+0x4c/0xa0
[c00000000b9bb6a0]c000000008337218devm_memremap+0x58/0x130
[c00000000b9bb6f0]c000000008aed00cdevm_nsio_enable+0xdc/0x170
[c00000000b9bb780]c000000008af3b6cnd_pmem_probe+0x4c/0x180
[c00000000b9bb7b0]c000000008ad84ccnvdimm_bus_probe+0xac/0x260
[c00000000b9bb840]c000000008aa0628really_probe+0x148/0x500
[c00000000b9bb8d0]c000000008aa0d7cdriver_probe_device+0x19c/0x1d0
[c00000000b9bb950]c000000008aa11bcdevice_driver_attach+0xcc/0x100
[c00000000b9bb990]c000000008aa12ec__driver_attach+0xfc/0x1e0
[c00000000b9bba10]c000000008a9d0a4bus_for_each_dev+0xb4/0x130
[c00000000b9bba70]c000000008a9fc04driver_attach+0x34/0x50
[c00000000b9bba90]c000000008a9f118bus_add_driver+0x1d8/0x300
[c00000000b9bbb20]c000000008aa2358driver_register+0x98/0x1a0
[c00000000b9bbb90]c000000008ad7e6c__nd_driver_register+0x5c/0x100
[c00000000b9bbbf0]c0000000093efbacnd_pmem_driver_init+0x34/0x48
[c00000000b9bbc10]c0000000080106c0do_one_initcall+0x60/0x2d0
[c00000000b9bbce0]c00000000938463ckernel_init_freeable+0x384/0x48c
[c00000000b9bbdb0]c000000008010a5ckernel_init+0x2c/0x160
[c00000000b9bbe20]c00000000800ba54ret_from_kernel_thread+0x5c/0x68Reported-and-debugged-by Vaibhav Jain <[email protected]>Signed-off-by Aneesh Kumar K.V <[email protected]>--- arch/powerpc/mm/numa.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)arch/powerpc/mm/numa.c +1
-1@@ -212,7 +212,7 @@ static int associativity_to_nid(const __be32 *associativity) {int nid =NUMA_NO_NODE ; -if (min_common_depth ==-1 ) +if (min_common_depth ==-1 || !numa_enabled) goto out;if (of_read_number (associativity,1 ) >= min_common_depth)@@ -416,10 +416,14 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)static int of_drconf_to_nid_single (struct drmem_lmb *lmb)
arch/powerpc/mm/numa.c static int of_get_assoc_arrays(struct assoc_arrays *aa) · line 426
Quoted 10 lines
arch/powerpc/mm/numa.c +4
-0{struct assoc_arrays aa = { .arrays =NULL }; +/* is that correct? */ int default_nid =0 ;int nid = default_nid;int rc, index; +if (!numa_enabled) +return NUMA_NO_NODE ; +
I guess we should have here.
modified arch/powerpc/mm/numa.c
@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)
static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
{
struct assoc_arrays aa = { .arrays = NULL };
- /* is that correct? */
int default_nid = 0;
int nid = default_nid;
int rc, index;
- if (!numa_enabled)
+ if ((min_common_depth < 0) || !numa_enabled)
return NUMA_NO_NODE;
rc = of_get_assoc_arrays(&aa);
Nathan,
Can you check this?
arch/powerpc/mm/numa.c static void __init find_possible_nodes(void) · line 818
Quoted 13 lines
arch/powerpc/mm/numa.c +1
-1rc =of_get_assoc_arrays (&aa);if (rc)return default_nid;@@ -808,7 +812,7 @@ static void __init find_possible_nodes(void)struct device_node *rtas;u32 numnodes, i; -if (min_common_depth <=0 ) +if (min_common_depth <=0 || !numa_enabled)return ; rtas =of_find_node_by_path ("/rtas" );
-aneesh
Quoted 20 lines · Aneesh Kumar K.V
I guess we should have here.
modified
arch/powerpc/mm/numa.carch/powerpc/mm/numa.c +1
-2@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)static int of_drconf_to_nid_single (struct drmem_lmb *lmb) {struct assoc_arrays aa = { .arrays =NULL }; -/* is that correct? */ int default_nid =0 ;int nid = default_nid;int rc, index; -if (!numa_enabled) +if ((min_common_depth <0 ) || !numa_enabled)return NUMA_NO_NODE ;rc = of_get_assoc_arrays(&aa);
Quoted 3 lines
Nathan,
Can you check this?
Looks like it would do the right thing.
Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)
Quoted 21 lines · Nathan Lynch
Quoted 20 lines · Aneesh Kumar K.V
I guess we should have here.
modified
arch/powerpc/mm/numa.carch/powerpc/mm/numa.c +1
-2@@ -416,12 +416,11 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa)static int of_drconf_to_nid_single (struct drmem_lmb *lmb) {struct assoc_arrays aa = { .arrays =NULL }; -/* is that correct? */ int default_nid =0 ;int nid = default_nid;int rc, index; -if (!numa_enabled) +if ((min_common_depth <0 ) || !numa_enabled)return NUMA_NO_NODE ;rc = of_get_assoc_arrays(&aa);
Quoted 9 lines
Quoted 3 lines
Nathan,
Can you check this?
Looks like it would do the right thing.
Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)
That is used in kdump kernel.
-aneesh
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines
Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)
That is used in kdump kernel.
I see, thanks.
Michael Ellerman
reviewerQuoted 8 lines · Nathan Lynch
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines
Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)
That is used in kdump kernel.
I see, thanks.
That doesn't mean it's a good idea :)
Does it actually reduce memory usage much? Last time I dug into the kdump kernel's usage of weird command line flags none of them really did anything useful.
cheers
Quoted 8 lines · Michael Ellerman
Quoted 7 lines · Nathan Lynch
Quoted 5 lines · Aneesh Kumar K.V
Quoted 3 lines
Just checking: do people still need numa=off? Seems like it's a maintenance burden :-)
That is used in kdump kernel.
Quoted 7 lines
Quoted 1 line
I see, thanks.
That doesn't mean it's a good idea :)
Does it actually reduce memory usage much? Last time I dug into the kdump kernel's usage of weird command line flags none of them really did anything useful.
I think it's intended to work around bugs in numa initialization, e.g.
https://www.suse.com/support/kb/doc/?id=7023399
Hopefully the original bug with numa/kdump interaction has been fixed?