Skip to contentSkip to conversation

powerpc/drmem: Don't compute the NUMA node for each LMB

PATCH Superseded Laurent Dufour 3 participants 6 messages Aug 5, 2020 – Apr 4, 2022 PowerPC

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 powerpc/drmem: Don't compute the NUMA node for each LMB

All the LMB from the same set of ibm,dynamic-memory-v2 property are sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

---
 arch/powerpc/mm/drmem.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)
arch/powerpc/mm/drmem.c +19 -2
index b2eeea39684c..3819c523c65b 100644
@@ -397,7 +397,7 @@ static void __init init_drmem_v1_lmbs(const __be32 *prop)
 
 static void __init init_drmem_v2_lmbs(const __be32 *prop)
 {
-	struct drmem_lmb *lmb;
+	struct drmem_lmb *lmb, *first;
 	struct of_drconf_cell_v2 dr_cell;
 	const __be32 *p;
 	u32 i, j, lmb_sets;
@@ -422,10 +422,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	/* second pass, read in the LMB information */
 	lmb_index = 0;
 	p = prop;
+	first = NULL;
 
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +446,16 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
 		}
 	}
 }
Signature · 1 line
2.28.0
#1 lore raw
Thank you for the patch! Yet something to improve:

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':

>> arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |        ^~
   arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |                     ^~

vim +457 arch/powerpc/mm/drmem.c

397
398 static void __init init_drmem_v2_lmbs(const __be32 *prop)
399 {
400 struct drmem_lmb *lmb, *first;
401 struct of_drconf_cell_v2 dr_cell;
402 const __be32 *p;
403 u32 i, j, lmb_sets;
404 int lmb_index;
405
406 lmb_sets = of_read_number(prop++, 1);
407 if (lmb_sets == 0)
408 return;
409
410 /* first pass, calculate the number of LMBs */
411 p = prop;
412 for (i = 0; i < lmb_sets; i++) {
413 read_drconf_v2_cell(&dr_cell, &p);
414 drmem_info->n_lmbs += dr_cell.seq_lmbs;
415 }
416
417 drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
418 GFP_KERNEL);
419 if (!drmem_info->lmbs)
420 return;
421
422 /* second pass, read in the LMB information */
423 lmb_index = 0;
424 p = prop;
425 first = NULL;
426
427 for (i = 0; i < lmb_sets; i++) {
428 read_drconf_v2_cell(&dr_cell, &p);
429
430 /*
431 * Fetch the NUMA node id for the fist set or if the
432 * associativity index is different from the previous set.
433 */
434 if (first && dr_cell.aa_index != first->aa_index)
435 first = NULL;
436
437 for (j = 0; j < dr_cell.seq_lmbs; j++) {
438 lmb = &drmem_info->lmbs[lmb_index++];
439
440 lmb->base_addr = dr_cell.base_addr;
441 dr_cell.base_addr += drmem_info->lmb_size;
442
443 lmb->drc_index = dr_cell.drc_index;
444 dr_cell.drc_index++;
445
446 lmb->aa_index = dr_cell.aa_index;
447 lmb->flags = dr_cell.flags;
448
449 /*
450 * All the LMB in the set share the same NUMA
451 * associativity property. So read that node only once.
452 */
453 if (!first) {
454 lmb_set_nid(lmb);
455 first = lmb;
456 } else {

 > 457					lmb->nid = first->nid;

458 }
459 }
460 }
461 }
462

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#2 lore raw

Le 05/08/2020 à 12:43, kernel test robot a écrit :

Quoted 25 lines

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

Quoted 7 lines

arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':

Quoted 1 line
Quoted 1 line

arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'

      457 |     lmb->nid = first->nid;
          |        ^~
    arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
      457 |     lmb->nid = first->nid;
          |                     ^~

My mistake, the nid member is only present when CONFIG_MEMORY_HOTPLUG is set.

I'll send a new version fixing this.

Quoted 73 lines

vim +457 arch/powerpc/mm/drmem.c

    397	
    398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
    399	{
    400		struct drmem_lmb *lmb, *first;
    401		struct of_drconf_cell_v2 dr_cell;
    402		const __be32 *p;
    403		u32 i, j, lmb_sets;
    404		int lmb_index;
    405	
    406		lmb_sets = of_read_number(prop++, 1);
    407		if (lmb_sets == 0)
    408			return;
    409	
    410		/* first pass, calculate the number of LMBs */
    411		p = prop;
    412		for (i = 0; i < lmb_sets; i++) {
    413			read_drconf_v2_cell(&dr_cell, &p);
    414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
    415		}
    416	
    417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
    418					   GFP_KERNEL);
    419		if (!drmem_info->lmbs)
    420			return;
    421	
    422		/* second pass, read in the LMB information */
    423		lmb_index = 0;
    424		p = prop;
    425		first = NULL;
    426	
    427		for (i = 0; i < lmb_sets; i++) {
    428			read_drconf_v2_cell(&dr_cell, &p);
    429	
    430			/*
    431			 * Fetch the NUMA node id for the fist set or if the
    432			 * associativity index is different from the previous set.
    433			 */
    434			if (first && dr_cell.aa_index != first->aa_index)
    435				first = NULL;
    436	
    437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
    438				lmb = &drmem_info->lmbs[lmb_index++];
    439	
    440				lmb->base_addr = dr_cell.base_addr;
    441				dr_cell.base_addr += drmem_info->lmb_size;
    442	
    443				lmb->drc_index = dr_cell.drc_index;
    444				dr_cell.drc_index++;
    445	
    446				lmb->aa_index = dr_cell.aa_index;
    447				lmb->flags = dr_cell.flags;
    448	
    449				/*
    450				 * All the LMB in the set share the same NUMA
    451				 * associativity property. So read that node only once.
    452				 */
    453				if (!first) {
    454					lmb_set_nid(lmb);
    455					first = lmb;
    456				} else {
Quoted 1 line

457 lmb->nid = first->nid;

    458				}
    459			}
    460		}
    461	}
    462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]

#3 lore raw

PATCH v2 1/1 powerpc/drmem: Don't compute the NUMA node for each LMB

In reply to kernel test robot

All the LMB from the same set of ibm,dynamic-memory-v2 property are sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid field is not present in the drmem_lmb structure.

---
 arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)
arch/powerpc/mm/drmem.c +24 -1
index b2eeea39684c..c11b6ec99ea3 100644
@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	const __be32 *p;
 	u32 i, j, lmb_sets;
 	int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+	struct drmem_lmb *first = NULL;
+#endif
 
 	lmb_sets = of_read_number(prop++, 1);
 	if (lmb_sets == 0)
@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+#endif
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
+#endif
 		}
 	}
 }
Signature · 1 line
2.28.0
#4 lore raw

Le 05/08/2020 à 15:35, Laurent Dufour a écrit :

Quoted 6 lines

All the LMB from the same set of ibm,dynamic-memory-v2 property are sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to lmb_set_nid() have been made instead of 1022.

Quoted 6 lines

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid field is not present in the drmem_lmb structure.

It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem: don't cache node id in drmem_lmb struct").

If not, anyway it conflicts with that patch so it has to be rebased.

Thanks
Christophe

Quoted 54 lines
---
  arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
  1 file changed, 24 insertions(+), 1 deletion(-)
arch/powerpc/mm/drmem.c +24 -1
index b2eeea39684c..c11b6ec99ea3 100644
@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
  	const __be32 *p;
  	u32 i, j, lmb_sets;
  	int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+	struct drmem_lmb *first = NULL;
+#endif
 
  	lmb_sets = of_read_number(prop++, 1);
  	if (lmb_sets == 0)
@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
  	for (i = 0; i < lmb_sets; i++) {
  		read_drconf_v2_cell(&dr_cell, &p);
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+#endif
+
  		for (j = 0; j < dr_cell.seq_lmbs; j++) {
  			lmb = &drmem_info->lmbs[lmb_index++];
 
@@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
  			lmb->aa_index = dr_cell.aa_index;
  			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
+#endif
  		}
  	}
  }
#5 lore raw
Quoted 11 lines · Christophe Leroy

Le 05/08/2020 à 15:35, Laurent Dufour a écrit :

Quoted 8 lines

All the LMB from the same set of ibm,dynamic-memory-v2 property are sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Quoted 7 lines
Quoted 4 lines

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid field is not present in the drmem_lmb structure.

It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem:
don't cache node id in drmem_lmb struct").

That patch has been superseded and can be dropped.

Thanks,
Laurent.

Quoted 59 lines

If not, anyway it conflicts with that patch so it has to be rebased.

Thanks
Christophe

Quoted 53 lines

---
  arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-

  1 file changed, 24 insertions(+), 1 deletion(-)
arch/powerpc/mm/drmem.c +3 -0
index b2eeea39684c..c11b6ec99ea3 100644
@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32
 *prop)
       const __be32 *p;
       u32 i, j, lmb_sets;
       int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+    struct drmem_lmb *first = NULL;
+#endif
         lmb_sets = of_read_number(prop++, 1);
       if (lmb_sets == 0)
@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32
 *prop)
       for (i = 0; i < lmb_sets; i++) {
           read_drconf_v2_cell(&dr_cell, &p);

  +#ifdef CONFIG_MEMORY_HOTPLUG
+        /*
+         * Fetch the NUMA node id for the fist set or if the +         * associativity index is different from the previous set. +         */
+        if (first && dr_cell.aa_index != first->aa_index) +            first = NULL;

arch/powerpc/mm/drmem.c +2 -0
+#endif
+

          for (j = 0; j < dr_cell.seq_lmbs; j++) {
              lmb = &drmem_info->lmbs[lmb_index++];
  @@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
              lmb->aa_index = dr_cell.aa_index;
              lmb->flags = dr_cell.flags;
  -            lmb_set_nid(lmb);

arch/powerpc/mm/drmem.c +12 -0
+#ifdef CONFIG_MEMORY_HOTPLUG
+            /*
+             * All the LMB in the set share the same NUMA
+             * associativity property. So read that node only once.
+             */
+            if (!first) {
+                lmb_set_nid(lmb);
+                first = lmb;
+            } else {
+                lmb->nid = first->nid;
+            }
+#endif

          }
      }
  }