Skip to contentSkip to conversation

selftest/vm: Fix ksm selftest to run with different NUMA topologies

PATCH Superseded Aneesh Kumar K.V 4 participants 6 messages Sep 13, 2021

Subsystems (2)

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 selftest/vm: Fix ksm selftest to run with different NUMA topologies

Platforms can have non-contiguous NUMA nodes like below

#numactl -H
available: 2 nodes (0,8)
.....
node distances:
node 0 8

  0:  10  40
  8:  40  10

#numactl -H
available: 1 nodes (1)
....
node distances:
node 1
1: 10

Hence update the test to not assume the presence of Node 0 and 1 and also use numa_num_configured_nodes() instead of numa_max_node for finding whether to skip the test.

Cc Zhansaya Bagdauletkyzy <[email protected]>
Cc Pavel Tatashin <[email protected]>
Cc Tyler Hicks <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Shuah Khan <[email protected]>
---
 tools/testing/selftests/vm/ksm_tests.c | 32 +++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)
tools/testing/selftests/vm/ksm_tests.c +29 -3
index b61dcdb44c5b..5bdfc5fe4d57 100644
@@ -354,12 +354,37 @@ static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int
 	return KSFT_FAIL;
 }
 
+static unsigned long get_next_mem_node(unsigned long node)
+{
+
+	long node_size;
+	unsigned long i;
+	unsigned long max_node = numa_max_node();
+	/*
+	 * start from node and find the next memory node
+	 */
+restart:
+	for (i = node + 1; i <= max_node; i++) {
+		node_size = numa_node_size(i, NULL);
+		if (node_size > 0)
+			return i;
+	}
+	node = -1;
+	goto restart;
+}
+
+static unsigned long get_first_mem_node(void)
+{
+	return get_next_mem_node(-1);
+}
+
 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
 				size_t page_size)
 {
 	void *numa1_map_ptr, *numa2_map_ptr;
 	struct timespec start_time;
 	int page_count = 2;
+	unsigned long first_node;
 
 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 		perror("clock_gettime");
@@ -370,7 +395,7 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		perror("NUMA support not enabled");
 		return KSFT_SKIP;
 	}
-	if (numa_max_node() < 1) {
+	if (numa_num_configured_nodes() <= 1) {
 		printf("At least 2 NUMA nodes must be available\n");
 		return KSFT_SKIP;
 	}
@@ -378,8 +403,9 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		return KSFT_FAIL;
 
 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-	numa1_map_ptr = numa_alloc_onnode(page_size, 0);
-	numa2_map_ptr = numa_alloc_onnode(page_size, 1);
+	first_node = get_first_mem_node();
+	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
+	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
 	if (!numa1_map_ptr || !numa2_map_ptr) {
 		perror("numa_alloc_onnode");
 		return KSFT_FAIL;
Signature · 1 line
2.31.1
Reviewed-by on v1 1/1
#1 lore raw
Quoted 17 lines · Aneesh Kumar K.V

Platforms can have non-contiguous NUMA nodes like below

#numactl -H
available: 2 nodes (0,8)
.....
node distances:
node 0 8

  0:  10  40
  8:  40  10

#numactl -H
available: 1 nodes (1)
....
node distances:
node 1
1: 10

Quoted 4 lines

Hence update the test to not assume the presence of Node 0 and 1 and also use numa_num_configured_nodes() instead of numa_max_node for finding whether to skip the test.

I think this should be added:

Fixes 82e717ad3501 ("selftests: vm: add KSM merging across nodes test")

The rest looks good to me:

Tyler

Quoted 76 lines
Cc Zhansaya Bagdauletkyzy <[email protected]>
Cc Pavel Tatashin <[email protected]>
Cc Tyler Hicks <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Shuah Khan <[email protected]>
---
 tools/testing/selftests/vm/ksm_tests.c | 32 +++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)
tools/testing/selftests/vm/ksm_tests.c +29 -3
index b61dcdb44c5b..5bdfc5fe4d57 100644
@@ -354,12 +354,37 @@ static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int
 	return KSFT_FAIL;
 }
 
+static unsigned long get_next_mem_node(unsigned long node)
+{
+
+	long node_size;
+	unsigned long i;
+	unsigned long max_node = numa_max_node();
+	/*
+	 * start from node and find the next memory node
+	 */
+restart:
+	for (i = node + 1; i <= max_node; i++) {
+		node_size = numa_node_size(i, NULL);
+		if (node_size > 0)
+			return i;
+	}
+	node = -1;
+	goto restart;
+}
+
+static unsigned long get_first_mem_node(void)
+{
+	return get_next_mem_node(-1);
+}
+
 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
 				size_t page_size)
 {
 	void *numa1_map_ptr, *numa2_map_ptr;
 	struct timespec start_time;
 	int page_count = 2;
+	unsigned long first_node;
 
 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 		perror("clock_gettime");
@@ -370,7 +395,7 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		perror("NUMA support not enabled");
 		return KSFT_SKIP;
 	}
-	if (numa_max_node() < 1) {
+	if (numa_num_configured_nodes() <= 1) {
 		printf("At least 2 NUMA nodes must be available\n");
 		return KSFT_SKIP;
 	}
@@ -378,8 +403,9 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		return KSFT_FAIL;
 
 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-	numa1_map_ptr = numa_alloc_onnode(page_size, 0);
-	numa2_map_ptr = numa_alloc_onnode(page_size, 1);
+	first_node = get_first_mem_node();
+	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
+	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
 	if (!numa1_map_ptr || !numa2_map_ptr) {
 		perror("numa_alloc_onnode");
 		return KSFT_FAIL;
Signature · 1 line
2.31.1
#2 lore raw
In reply to Aneesh Kumar K.V
Quoted 1 line · Aneesh Kumar K.V

Hi Aneesh,

Thank you for this fix. My comments below:

Quoted 1 line

Platforms can have non-contiguous NUMA nodes like below

Thanks for catching this.

Quoted 1 line

+static unsigned long get_next_mem_node(unsigned long node)

NUMA should be "int" not "unsigned long", especially as it is confusing with negative numbers used in get_first_mem_node(). Also, numa_max_node() returns int.

Quoted 7 lines
tools/testing/selftests/vm/ksm_tests.c +7 -0
+{
+
+       long node_size;
+       unsigned long i;
+       unsigned long max_node = numa_max_node();
+       /*
+        * start from node and find the next memory node

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 374

Quoted 10 lines
tools/testing/selftests/vm/ksm_tests.c +10 -0
+        */
+restart:
+       for (i = node + 1; i <= max_node; i++) {
+               node_size = numa_node_size(i, NULL);
+               if (node_size > 0)
+                       return i;
+       }
+       node = -1;
+       goto restart;
+}

I would rewrite the above without goto, and possibility of stacking in an infinite loop. Something like this should work:

       for (i = node + 1; i <= max_node + node; i++) {
               node_size = numa_node_size(i % (max_node + 1), NULL);
               if (node_size > 0)
                       break;
       }

        return i % (max_node + 1);

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 378

Quoted 4 lines
tools/testing/selftests/vm/ksm_tests.c +4 -0
+
+static unsigned long get_first_mem_node(void)
+{
+       return get_next_mem_node(-1);

Next after the last node would make more sense:

return get_next_mem_node(numa_max_node());

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 387

Quoted 9 lines
tools/testing/selftests/vm/ksm_tests.c +3 -0
+}
+
 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
                                size_t page_size)
 {
        void *numa1_map_ptr, *numa2_map_ptr;
        struct timespec start_time;
        int page_count = 2;
+       unsigned long first_node;

Please use int.

Quoted 13 lines
        if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
                perror("clock_gettime");
tools/testing/selftests/vm/ksm_tests.c +1 -1
@@ -370,7 +395,7 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
                perror("NUMA support not enabled");
                return KSFT_SKIP;
        }
-       if (numa_max_node() < 1) {
+       if (numa_num_configured_nodes() <= 1) {
                printf("At least 2 NUMA nodes must be available\n");
                return KSFT_SKIP;
        }

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a · line 408

Quoted 8 lines
tools/testing/selftests/vm/ksm_tests.c +3 -2
@@ -378,8 +403,9 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
                return KSFT_FAIL;
 
        /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-       numa1_map_ptr = numa_alloc_onnode(page_size, 0);
-       numa2_map_ptr = numa_alloc_onnode(page_size, 1);
+       first_node = get_first_mem_node();
+       numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
+       numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));

Thanks,
Pasha

#3 lore raw
Quoted 12 lines · Pasha Tatashin
Quoted 1 line · Aneesh Kumar K.V

Hi Aneesh,

Thank you for this fix. My comments below:

Quoted 1 line

Platforms can have non-contiguous NUMA nodes like below

Thanks for catching this.

Quoted 8 lines
Quoted 1 line

+static unsigned long get_next_mem_node(unsigned long node)

NUMA should be "int" not "unsigned long", especially as it is confusing with negative numbers used in get_first_mem_node(). Also, numa_max_node() returns int.

sure will update.

Quoted 21 lines
Quoted 7 lines
tools/testing/selftests/vm/ksm_tests.c +7 -0
+{
+
+       long node_size;
+       unsigned long i;
+       unsigned long max_node = numa_max_node();
+       /*
+        * start from node and find the next memory node

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 374

Quoted 10 lines
tools/testing/selftests/vm/ksm_tests.c +10 -0
+        */
+restart:
+       for (i = node + 1; i <= max_node; i++) {
+               node_size = numa_node_size(i, NULL);
+               if (node_size > 0)
+                       return i;
+       }
+       node = -1;
+       goto restart;
+}

I would rewrite the above without goto, and possibility of stacking in an infinite loop. Something like this should work:

Quoted 8 lines
        for (i = node + 1; i <= max_node + node; i++) {
                node_size = numa_node_size(i % (max_node + 1), NULL);
                if (node_size > 0)
                        break;
        }

         return i % (max_node + 1);

I didn't quiet follow this. not all nodes can have memory and node numbers are discontiguous.

Quoted 10 lines

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 378

Quoted 4 lines
tools/testing/selftests/vm/ksm_tests.c +4 -0
+
+static unsigned long get_first_mem_node(void)
+{
+       return get_next_mem_node(-1);

Next after the last node would make more sense:

return get_next_mem_node(numa_max_node());

Yes, that would work, but is that really useful? We would essentially skip the for loop in first iteration set node = -1 internally and do the for loop again.

Quoted 27 lines

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 387

Quoted 9 lines
tools/testing/selftests/vm/ksm_tests.c +3 -0
+}
+
  static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
                                 size_t page_size)
  {
         void *numa1_map_ptr, *numa2_map_ptr;
         struct timespec start_time;
         int page_count = 2;
+       unsigned long first_node;

Please use int.

Quoted 15 lines
         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
                 perror("clock_gettime");
tools/testing/selftests/vm/ksm_tests.c +1 -1
@@ -370,7 +395,7 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
                 perror("NUMA support not enabled");
                 return KSFT_SKIP;
         }
-       if (numa_max_node() < 1) {
+       if (numa_num_configured_nodes() <= 1) {
                 printf("At least 2 NUMA nodes must be available\n");
                 return KSFT_SKIP;
         }
@@ -378,8 +403,9 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
                 return KSFT_FAIL;
 
Quoted 10 lines

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a · line 408

Quoted 6 lines
tools/testing/selftests/vm/ksm_tests.c +3 -2
         /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-       numa1_map_ptr = numa_alloc_onnode(page_size, 0);
-       numa2_map_ptr = numa_alloc_onnode(page_size, 1);
+       first_node = get_first_mem_node();
+       numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
+       numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));

Thanks,
Pasha

-aneesh

#4 lore raw
Quoted 25 lines
Quoted 25 lines
Quoted 5 lines
tools/testing/selftests/vm/ksm_tests.c +5 -0
+       long node_size;
+       unsigned long i;
+       unsigned long max_node = numa_max_node();
+       /*
+        * start from node and find the next memory node

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 374

Quoted 10 lines
tools/testing/selftests/vm/ksm_tests.c +10 -0
+        */
+restart:
+       for (i = node + 1; i <= max_node; i++) {
+               node_size = numa_node_size(i, NULL);
+               if (node_size > 0)
+                       return i;
+       }
+       node = -1;
+       goto restart;
+}

I would rewrite the above without goto, and possibility of stacking in an infinite loop. Something like this should work:

        for (i = node + 1; i <= max_node + node; i++) {
                node_size = numa_node_size(i % (max_node + 1), NULL);
                if (node_size > 0)
                        break;
        }
Quoted 6 lines
Quoted 2 lines
         return i % (max_node + 1);

I didn't quiet follow this. not all nodes can have memory and node numbers are discontiguous.

I understand, the above loop searches for the first node with memory, and returns it. It cannot end up in the infinite loop in case if there are bugs where numa_node_size() are zero for all nodes for example. Also, it does not have gotos.

Quoted 8 lines
Quoted 8 lines

tools/testing/selftests/vm/ksm_tests.c static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int · line 378

Quoted 5 lines
tools/testing/selftests/vm/ksm_tests.c +4 -0
+
+static unsigned long get_first_mem_node(void)
+{
+       return get_next_mem_node(-1);

Next after the last node would make more sense:

Quoted 7 lines
Quoted 3 lines

return get_next_mem_node(numa_max_node());

Yes, that would work, but is that really useful? We would essentially skip the for loop in first iteration set node = -1 internally and do the for loop again.

It is useful in terms that get_next_mem_node() is supplied with real nodes. Also, it works better with the version of loop that I am proposing above: "max_node + node" when node == -1 does not yield the desired result.

Thanks,
Pasha

Reviewed-by on v1 1/1
#5 lore raw
In reply to Aneesh Kumar K.V
Quoted 2 lines · Aneesh Kumar K.V

Platforms can have non-contiguous NUMA nodes like below

Hi Aneesh,

Thank you for pointing this out.

Everything looks good to me:

Zhansaya

Quoted 95 lines

#numactl -H
available: 2 nodes (0,8)
.....
node distances:
node 0 8

  0:  10  40
  8:  40  10

#numactl -H
available: 1 nodes (1)
....
node distances:
node 1
1: 10

Hence update the test to not assume the presence of Node 0 and 1 and also use numa_num_configured_nodes() instead of numa_max_node for finding whether to skip the test.

Cc Zhansaya Bagdauletkyzy <[email protected]>
Cc Pavel Tatashin <[email protected]>
Cc Tyler Hicks <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Shuah Khan <[email protected]>
---
 tools/testing/selftests/vm/ksm_tests.c | 32 +++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)
tools/testing/selftests/vm/ksm_tests.c +29 -3
index b61dcdb44c5b..5bdfc5fe4d57 100644
@@ -354,12 +354,37 @@ static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int
 	return KSFT_FAIL;
 }
 
+static unsigned long get_next_mem_node(unsigned long node)
+{
+
+	long node_size;
+	unsigned long i;
+	unsigned long max_node = numa_max_node();
+	/*
+	 * start from node and find the next memory node
+	 */
+restart:
+	for (i = node + 1; i <= max_node; i++) {
+		node_size = numa_node_size(i, NULL);
+		if (node_size > 0)
+			return i;
+	}
+	node = -1;
+	goto restart;
+}
+
+static unsigned long get_first_mem_node(void)
+{
+	return get_next_mem_node(-1);
+}
+
 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
 				size_t page_size)
 {
 	void *numa1_map_ptr, *numa2_map_ptr;
 	struct timespec start_time;
 	int page_count = 2;
+	unsigned long first_node;
 
 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 		perror("clock_gettime");
@@ -370,7 +395,7 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		perror("NUMA support not enabled");
 		return KSFT_SKIP;
 	}
-	if (numa_max_node() < 1) {
+	if (numa_num_configured_nodes() <= 1) {
 		printf("At least 2 NUMA nodes must be available\n");
 		return KSFT_SKIP;
 	}
@@ -378,8 +403,9 @@ static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_a
 		return KSFT_FAIL;
 
 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-	numa1_map_ptr = numa_alloc_onnode(page_size, 0);
-	numa2_map_ptr = numa_alloc_onnode(page_size, 1);
+	first_node = get_first_mem_node();
+	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
+	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
 	if (!numa1_map_ptr || !numa2_map_ptr) {
 		perror("numa_alloc_onnode");
 		return KSFT_FAIL;
Signature · 1 line
2.31.1