mm/demotion: Memory tiers and demotion

In the current kernel, memory tiers are defined implicitly via a demotion path relationship between NUMA nodes, which is created during the kernel initialization and updated when a NUMA node is hot-added or hot-removed. The current implementation puts all nodes with CPU into the top tier,

and builds the tier hierarchy
tier-by-tier by establishing the per-node demotion targets based on the distances between nodes.

This current memory tier kernel interface needs to be improved for

several important use cases,

The current tier initialization code always initializes each memory-only NUMA node into a lower tier. But a memory-only NUMA node may have a high performance memory device (e.g. a DRAM device attached via CXL.mem

or a DRAM-backed memory-only node on
a virtual machine) and should be put into a higher tier.

The current tier hierarchy always puts CPU nodes into the top tier. But on a system with HBM or GPU devices, the memory-only NUMA nodes mapping these devices should be in the top tier, and DRAM nodes with CPUs are better to be placed into the next lower tier.

With current kernel higher tier node can only be demoted to selected nodes on the

next lower tier as defined by the demotion path, not any other node from any lower tier. This strict, hard-coded demotion order

does not work in all use cases (e.g. some use cases may want to allow cross-socket demotion to another node in the same demotion tier as a fallback when the preferred demotion node is out of space), This demotion order is also inconsistent with the page allocation fallback order when all the nodes in a higher tier are out of space: The page allocation can fall back to any node from any lower tier, whereas the demotion order doesn't allow that.

The current kernel also don't provide any interfaces for the userspace to learn about the memory tier hierarchy in order to optimize its memory allocations.

This patch series address the above by defining memory tiers explicitly.

Linux kernel presents memory devices as NUMA nodes and each memory device is of a specific type. The memory type of a device is represented by its performance level. A memory tier corresponds to a range of performance levels. This allows for classifying memory devices with a specific performance range into a memory tier.

This patch configures the range/chunk size to be 128. The default DRAM performance level is 512. We can have 4 memory tiers below the default DRAM performance level which cover the range 0 - 127, 127 - 255, 256- 383, 384 - 511. Slower memory devices like persistent memory will have performance levels below the default DRAM level and hence will be placed in these 4 lower tiers.

While reclaim we migrate pages from fast(higher) tiers to slow(lower) tiers when the fast(higher) tier is under memory pressure.

A kernel parameter is provided to override the default memory tier.

Link https://lore.kernel.org/linux-mm/CAAPL-u9Wv+nH1VOZTj=9p9S70Y3Qz3+63EkqncRDdHfubsrjfw@mail.gmail.com open here

Link https://lore.kernel.org/linux-mm/[email protected] open here

Signed-off-by Jagdish Gediya <[email protected]>

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  18 +++++++
 include/linux/node.h         |   6 +++
 mm/Makefile                  |   1 +
 mm/memory-tiers.c            | 101 +++++++++++++++++++++++++++++++++++
 4 files changed, 126 insertions(+)
 create mode 100644 include/linux/memory-tiers.h
 create mode 100644 mm/memory-tiers.c

include/linux/memory-tiers.h new file +0 -0

new file mode 100644
index 000000000000..f28f9910a4e7

include/linux/memory-tiers.h new file +9 -0

+/*
+ * For now let's have 4 memory tier below default DRAM tier.
+ */
+#define MEMTIER_PERF_LEVEL_DRAM	(1 << (MEMTIER_CHUNK_BITS + 2))
+/* leave one tier below this slow pmem */
+#define MEMTIER_PERF_LEVEL_PMEM	(1 << MEMTIER_CHUNK_BITS)
+
+#endif	/* CONFIG_NUMA */
+#endif  /* _LINUX_MEMORY_TIERS_H */

include/linux/node.h +0 -0

index 40d641a8bfb0..a2a16d4104fd 100644

include/linux/node.h +0 -0

 };
 
 struct memory_block;

mm/Makefile +1 -0

index 6f9ffa968a1a..d30acebc2164 100644
@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/
 obj-$(CONFIG_FAILSLAB) += failslab.o
 obj-$(CONFIG_MEMTEST)		+= memtest.o
 obj-$(CONFIG_MIGRATION) += migrate.o
+obj-$(CONFIG_NUMA) += memory-tiers.o
 obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
 obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
 obj-$(CONFIG_PAGE_COUNTER) += page_counter.o

mm/memory-tiers.c new file +2 -0

new file mode 100644
index 000000000000..61bb84c54091
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/types.h>

mm/memory-tiers.c new file +12 -0

+/*
+ * performance levels are grouped into memtiers each of chunk size
+ * memtier_perf_chunk
+ */
+static struct memory_tier *find_create_memory_tier(unsigned int perf_level)
+{
+	bool found_slot = false;
+	struct list_head *ent;
+	struct memory_tier *memtier, *new_memtier;
+	unsigned int memtier_perf_chunk_size = 1 << MEMTIER_CHUNK_BITS;
+	/*
+	 * zero is special in that it indicates uninitialized

mm/memory-tiers.c new file +22 -0

+		memtier = list_entry(ent, struct memory_tier, list);
+		if (perf_level == memtier->perf_level) {
+			return memtier;
+		} else if (perf_level < memtier->perf_level) {
+			found_slot = true;
+			break;
+		}
+	}
+
+	new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL);
+	if (!new_memtier)
+		return ERR_PTR(-ENOMEM);
+
+	new_memtier->perf_level = perf_level;
+	if (found_slot)
+		list_add_tail(&new_memtier->list, ent);
+	else
+		list_add_tail(&new_memtier->list, &memory_tiers);
+	return new_memtier;
+}
+
+static int __init memory_tier_init(void)

mm/memory-tiers.c new file +11 -0

+	mutex_lock(&memory_tier_lock);
+	memtier = find_create_memory_tier(default_memtier_perf_level);
+	if (IS_ERR(memtier))
+		panic("%s() failed to register memory tier: %ld\n",
+		      __func__, PTR_ERR(memtier));
+
+	/* CPU only nodes are not part of memory tiers. */
+	memtier->nodelist = node_states[N_MEMORY];
+
+	/*
+	 * nodes that are already online and that doesn't

Quoted 6 lines

include/linux/node.h +0 -0

index 40d641a8bfb0..a2a16d4104fd 100644

include/linux/node.h struct node { · line 100

Quoted 9 lines

include/linux/node.h +6 -0

@@ -92,6 +92,12 @@ struct node {
 	struct list_head cache_attrs;
 	struct device *cache_dev;
 #endif
+	/*
+	 * For memory devices, perf_level describes
+	 * the device performance and how it should be used
+	 * while building a memory hierarchy.
+	 */
+	int perf_level;

Think again, I found that "perf_level" may be not the best abstraction of the performance of memory devices. In concept, it's an abstraction of the memory bandwidth. But it will not reflect the memory latency.

Instead, the previous proposed "abstract_distance" is an abstraction of the memory latency. Per my understanding, the memory latency has more direct influence on system performance. And because the latency of the memory device will increase if the memory accessing throughput nears its max bandwidth, so the memory bandwidth can be reflected in the "abstract distance" too. That is, the "abstract distance" is an abstraction of the memory latency under the expected memory accessing throughput. The "offset" to the default "abstract distance" reflects the different expected memory accessing throughput.

So, I think we need some kind of abstraction of the memory latency instead of memory bandwidth, e.g., "abstract distance".

Quoted 29 lines

Quoted 5 lines

include/linux/node.h +0 -0

index 40d641a8bfb0..a2a16d4104fd 100644

include/linux/node.h struct node { · line 100

Quoted 9 lines

include/linux/node.h +6 -0

@@ -92,6 +92,12 @@ struct node {
 	struct list_head cache_attrs;
 	struct device *cache_dev;
 #endif
+	/*
+	 * For memory devices, perf_level describes
+	 * the device performance and how it should be used
+	 * while building a memory hierarchy.
+	 */
+	int perf_level;

Think again, I found that "perf_level" may be not the best abstraction of the performance of memory devices. In concept, it's an abstraction of the memory bandwidth. But it will not reflect the memory latency.

Instead, the previous proposed "abstract_distance" is an abstraction of the memory latency. Per my understanding, the memory latency has more direct influence on system performance. And because the latency of the memory device will increase if the memory accessing throughput nears its max bandwidth, so the memory bandwidth can be reflected in the "abstract distance" too. That is, the "abstract distance" is an abstraction of the memory latency under the expected memory accessing throughput. The "offset" to the default "abstract distance" reflects the different expected memory accessing throughput.

Quoted 3 lines

So, I think we need some kind of abstraction of the memory latency instead of memory bandwidth, e.g., "abstract distance".

I am reworking other parts of the patch set based on your feedback.

This part I guess we need to reach some consensus.

IMHO perf_level (performance level) can indicate a combination of both latency and bandwidth.

It is an abstract concept that indicates the performance of the device. As we learn more about which device attribute makes more impact in defining hierarchy, performance level will give more weightage to that specific attribute. It could be write latency or bandwidth. For me, distance has a direct linkage to latency because that is how we define numa distance now. Adding abstract to the name is not making it more abstract than perf_level.

I am open to suggestions from others. Wei Xu has also suggested perf_level name. I can rename this to abstract_distance if that indicates the goal better.

Quoted 9 lines · Aneesh Kumar K V

It is an abstract concept that indicates the performance of the device. As we learn more about which device attribute makes more impact in defining hierarchy, performance level will give more weightage to that specific attribute. It could be write latency or bandwidth. For me, distance has a direct linkage to latency because that is how we define numa distance now. Adding abstract to the name is not making it more abstract than perf_level.

I am open to suggestions from others. Wei Xu has also suggested perf_level name. I can rename this to abstract_distance if that indicates the goal better.

I'm open to naming. But I think that it's good to define it at some degree instead of completely opaque stuff. If it's latency based, then low value corresponds to high performance. If it's bandwidth based, then low value corresponds to low performance.

Hi, Wei and Johannes,

What do you think about this?

If the new NUMA node onlined doesn't have a performance level assigned, the kernel adds the NUMA node to default memory tier.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  1 +
 mm/memory-tiers.c            | 75 ++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

include/linux/memory-tiers.h +1 -0

index ef380a39db3a..3d5f14d57ae6 100644
@@ -14,6 +14,7 @@
 #define MEMTIER_PERF_LEVEL_DRAM	(1 << (MEMTIER_CHUNK_BITS + 2))
 /* leave one tier below this slow pmem */
 #define MEMTIER_PERF_LEVEL_PMEM	(1 << MEMTIER_CHUNK_BITS)
+#define MEMTIER_HOTPLUG_PRIO	100
 
 extern bool numa_demotion_enabled;
 

mm/memory-tiers.c +3 -0

index 41a21cc5ae55..cc3a47ec18e4 100644
@@ -5,6 +5,7 @@
 #include <linux/lockdep.h>
 #include <linux/moduleparam.h>
 #include <linux/node.h>
+#include <linux/memory.h>
 #include <linux/memory-tiers.h>
 
 struct memory_tier {

@@ -64,6 +65,78 @@ static struct memory_tier *find_create_memory_tier(unsigned int perf_level)
 	return new_memtier;
 }
 
+static struct memory_tier *__node_get_memory_tier(int node)
+{

mm/memory-tiers.c +13 -0

+{
+	int perf_level;
+	struct memory_tier *memtier;
+
+	mutex_lock(&memory_tier_lock);
+
+	memtier = __node_get_memory_tier(node);
+	if (!memtier) {
+		perf_level = node_devices[node]->perf_level;
+		memtier = find_create_memory_tier(perf_level);
+		node_set(node, memtier->nodelist);
+	}
+	mutex_unlock(&memory_tier_lock);

mm/memory-tiers.c +16 -0

+{
+	struct memory_notify *arg = _arg;
+
+	/*
+	 * Only update the node migration order when a node is
+	 * changing status, like online->offline.
+	 */
+	if (arg->status_change_nid < 0)
+		return notifier_from_errno(0);
+
+	switch (action) {
+	case MEM_OFFLINE:
+		clear_node_memory_tier(arg->status_change_nid);
+		break;
+	case MEM_ONLINE:
+		init_node_memory_tier(arg->status_change_nid);

Quoted 37 lines · Aneesh Kumar K.V

If the new NUMA node onlined doesn't have a performance level assigned, the kernel adds the NUMA node to default memory tier.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  1 +
 mm/memory-tiers.c            | 75 ++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

include/linux/memory-tiers.h +1 -0

index ef380a39db3a..3d5f14d57ae6 100644
@@ -14,6 +14,7 @@
 #define MEMTIER_PERF_LEVEL_DRAM	(1 << (MEMTIER_CHUNK_BITS + 2))
 /* leave one tier below this slow pmem */
 #define MEMTIER_PERF_LEVEL_PMEM	(1 << MEMTIER_CHUNK_BITS)
+#define MEMTIER_HOTPLUG_PRIO	100
 
 extern bool numa_demotion_enabled;
 

mm/memory-tiers.c +1 -0

index 41a21cc5ae55..cc3a47ec18e4 100644
@@ -5,6 +5,7 @@
 #include <linux/lockdep.h>
 #include <linux/moduleparam.h>
 #include <linux/node.h>
+#include <linux/memory.h>
 #include <linux/memory-tiers.h>
 
 struct memory_tier {

@@ -64,6 +65,78 @@ static struct memory_tier *find_create_memory_tier(unsigned int perf_level)
 	return new_memtier;
 }
 

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 79

Quoted 12 lines

mm/memory-tiers.c +12 -0

+static struct memory_tier *__node_get_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	list_for_each_entry(memtier, &memory_tiers, list) {
+		if (node_isset(node, memtier->nodelist))
+			return memtier;
+	}
+	return NULL;
+}
+
+static void init_node_memory_tier(int node)

set_node_memory_tier()?

Quoted 14 lines

mm/memory-tiers.c +13 -0

+{
+	int perf_level;
+	struct memory_tier *memtier;
+
+	mutex_lock(&memory_tier_lock);
+
+	memtier = __node_get_memory_tier(node);
+	if (!memtier) {
+		perf_level = node_devices[node]->perf_level;
+		memtier = find_create_memory_tier(perf_level);
+		node_set(node, memtier->nodelist);
+	}
+	mutex_unlock(&memory_tier_lock);

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 102

Quoted 10 lines

mm/memory-tiers.c +10 -0

+}
+
+static void clear_node_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	mutex_lock(&memory_tier_lock);
+	memtier = __node_get_memory_tier(node);
+	if (memtier)
+		node_clear(node, memtier->nodelist);

When memtier->nodelist becomes empty, we need to free memtier?

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 110

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_unlock(&memory_tier_lock);
+}
+
+/*
+ * This runs whether reclaim-based migration is enabled or not,
+ * which ensures that the user can turn reclaim-based migration
+ * at any time without needing to recalculate migration targets.
+ */

The comments doesn't apply here.

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 112

Quoted 2 lines

mm/memory-tiers.c +2 -0

+static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
+						 unsigned long action, void *_arg)

Now we are building memory tiers instead of working on demotion. So I think we should rename the function to memtier_hotplug_callback().

Quoted 16 lines

mm/memory-tiers.c +16 -0

+{
+	struct memory_notify *arg = _arg;
+
+	/*
+	 * Only update the node migration order when a node is
+	 * changing status, like online->offline.
+	 */
+	if (arg->status_change_nid < 0)
+		return notifier_from_errno(0);
+
+	switch (action) {
+	case MEM_OFFLINE:
+		clear_node_memory_tier(arg->status_change_nid);
+		break;
+	case MEM_ONLINE:
+		init_node_memory_tier(arg->status_change_nid);

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 138

Quoted 10 lines

mm/memory-tiers.c +10 -0

+		break;
+	}
+
+	return notifier_from_errno(0);
+}
+
+static void __init migrate_on_reclaim_init(void)
+{
+	hotplug_memory_notifier(migrate_on_reclaim_callback, MEMTIER_HOTPLUG_PRIO);
+}

I suggest to call hotplug_memory_notifier() in memory_tier_init() directly. We are not working on demotion here.

mm/memory-tiers.c static int __init memory_tier_init(void) · line 176

Quoted 13 lines

mm/memory-tiers.c +3 -0

+
 static int __init memory_tier_init(void)
 {
 	int node;

@@ -96,6 +169,8 @@ static int __init memory_tier_init(void)
 			node_property->perf_level = default_memtier_perf_level;
 	}
 	mutex_unlock(&memory_tier_lock);
+
+	migrate_on_reclaim_init();
 	return 0;
 }
 subsys_initcall(memory_tier_init);

Best Regards,
Huang, Ying

Quoted 39 lines · Ying

Quoted 37 lines · Aneesh Kumar K.V

If the new NUMA node onlined doesn't have a performance level assigned, the kernel adds the NUMA node to default memory tier.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  1 +
 mm/memory-tiers.c            | 75 ++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

include/linux/memory-tiers.h +1 -0

index ef380a39db3a..3d5f14d57ae6 100644
@@ -14,6 +14,7 @@
 #define MEMTIER_PERF_LEVEL_DRAM	(1 << (MEMTIER_CHUNK_BITS + 2))
 /* leave one tier below this slow pmem */
 #define MEMTIER_PERF_LEVEL_PMEM	(1 << MEMTIER_CHUNK_BITS)
+#define MEMTIER_HOTPLUG_PRIO	100
 
 extern bool numa_demotion_enabled;
 

mm/memory-tiers.c +1 -0

index 41a21cc5ae55..cc3a47ec18e4 100644
@@ -5,6 +5,7 @@
 #include <linux/lockdep.h>
 #include <linux/moduleparam.h>
 #include <linux/node.h>
+#include <linux/memory.h>
 #include <linux/memory-tiers.h>
 
 struct memory_tier {

@@ -64,6 +65,78 @@ static struct memory_tier *find_create_memory_tier(unsigned int perf_level)
 	return new_memtier;
 }
 

Quoted 14 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 79

Quoted 12 lines

mm/memory-tiers.c +12 -0

+static struct memory_tier *__node_get_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	list_for_each_entry(memtier, &memory_tiers, list) {
+		if (node_isset(node, memtier->nodelist))
+			return memtier;
+	}
+	return NULL;
+}
+
+static void init_node_memory_tier(int node)

set_node_memory_tier()?

That was done based on feedback from Alistair

https://lore.kernel.org/linux-mm/[email protected]

Quoted 1 line

Quoted 4 lines

Quoted 4 lines

mm/memory-tiers.c +4 -0

+{
+	int perf_level;
+	struct memory_tier *memtier;
+

Quoted 8 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 91

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_lock(&memory_tier_lock);
+
+	memtier = __node_get_memory_tier(node);
+	if (!memtier) {
+		perf_level = node_devices[node]->perf_level;
+		memtier = find_create_memory_tier(perf_level);
+		node_set(node, memtier->nodelist);
+	}

Quoted 61 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 102

Quoted 11 lines

mm/memory-tiers.c +11 -0

+	mutex_unlock(&memory_tier_lock);
+}
+
+static void clear_node_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	mutex_lock(&memory_tier_lock);
+	memtier = __node_get_memory_tier(node);
+	if (memtier)
+		node_clear(node, memtier->nodelist);

When memtier->nodelist becomes empty, we need to free memtier?

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 110

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_unlock(&memory_tier_lock);
+}
+
+/*
+ * This runs whether reclaim-based migration is enabled or not,
+ * which ensures that the user can turn reclaim-based migration
+ * at any time without needing to recalculate migration targets.
+ */

The comments doesn't apply here.

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 112

Quoted 2 lines

mm/memory-tiers.c +2 -0

+static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
+						 unsigned long action, void *_arg)

Now we are building memory tiers instead of working on demotion. So I think we should rename the function to memtier_hotplug_callback().

Quoted 16 lines

mm/memory-tiers.c +16 -0

+{
+	struct memory_notify *arg = _arg;
+
+	/*
+	 * Only update the node migration order when a node is
+	 * changing status, like online->offline.
+	 */
+	if (arg->status_change_nid < 0)
+		return notifier_from_errno(0);
+
+	switch (action) {
+	case MEM_OFFLINE:
+		clear_node_memory_tier(arg->status_change_nid);
+		break;
+	case MEM_ONLINE:
+		init_node_memory_tier(arg->status_change_nid);

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 138

Quoted 10 lines

mm/memory-tiers.c +10 -0

+		break;
+	}
+
+	return notifier_from_errno(0);
+}
+
+static void __init migrate_on_reclaim_init(void)
+{
+	hotplug_memory_notifier(migrate_on_reclaim_callback, MEMTIER_HOTPLUG_PRIO);
+}

I suggest to call hotplug_memory_notifier() in memory_tier_init() directly. We are not working on demotion here.

Quoted 16 lines

mm/memory-tiers.c static int __init memory_tier_init(void) · line 176

Quoted 13 lines

mm/memory-tiers.c +3 -0

+
 static int __init memory_tier_init(void)
 {
 	int node;

@@ -96,6 +169,8 @@ static int __init memory_tier_init(void)
 			node_property->perf_level = default_memtier_perf_level;
 	}
 	mutex_unlock(&memory_tier_lock);
+
+	migrate_on_reclaim_init();
 	return 0;
 }
 subsys_initcall(memory_tier_init);

Best Regards,
Huang, Ying

Will update the patch in next iteration to take care of other feedback.

Quoted 60 lines · Aneesh Kumar K V

Quoted 39 lines · Ying

Quoted 37 lines · Aneesh Kumar K.V

If the new NUMA node onlined doesn't have a performance level assigned, the kernel adds the NUMA node to default memory tier.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  1 +
 mm/memory-tiers.c            | 75 ++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

include/linux/memory-tiers.h +1 -0

index ef380a39db3a..3d5f14d57ae6 100644
@@ -14,6 +14,7 @@
 #define MEMTIER_PERF_LEVEL_DRAM	(1 << (MEMTIER_CHUNK_BITS + 2))
 /* leave one tier below this slow pmem */
 #define MEMTIER_PERF_LEVEL_PMEM	(1 << MEMTIER_CHUNK_BITS)
+#define MEMTIER_HOTPLUG_PRIO	100
 
 extern bool numa_demotion_enabled;
 

mm/memory-tiers.c +1 -0

index 41a21cc5ae55..cc3a47ec18e4 100644
@@ -5,6 +5,7 @@
 #include <linux/lockdep.h>
 #include <linux/moduleparam.h>
 #include <linux/node.h>
+#include <linux/memory.h>
 #include <linux/memory-tiers.h>
 
 struct memory_tier {

@@ -64,6 +65,78 @@ static struct memory_tier *find_create_memory_tier(unsigned int perf_level)
 	return new_memtier;
 }
 

Quoted 14 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 79

Quoted 12 lines

mm/memory-tiers.c +12 -0

+static struct memory_tier *__node_get_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	list_for_each_entry(memtier, &memory_tiers, list) {
+		if (node_isset(node, memtier->nodelist))
+			return memtier;
+	}
+	return NULL;
+}
+
+static void init_node_memory_tier(int node)

set_node_memory_tier()?

That was done based on feedback from Alistair

https://lore.kernel.org/linux-mm/[email protected]

Quoted 1 line

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

mm/memory-tiers.c +4 -0

+{
+	int perf_level;
+	struct memory_tier *memtier;
+

Quoted 8 lines

Quoted 8 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 91

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_lock(&memory_tier_lock);
+
+	memtier = __node_get_memory_tier(node);
+	if (!memtier) {
+		perf_level = node_devices[node]->perf_level;
+		memtier = find_create_memory_tier(perf_level);
+		node_set(node, memtier->nodelist);
+	}

It's related to Alistair's comments too. When will memtier != NULL here? We may need just VM_WARN_ON() here?

Quoted 80 lines

Quoted 62 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 102

Quoted 12 lines

mm/memory-tiers.c +11 -0

+	mutex_unlock(&memory_tier_lock);
+}
+
+static void clear_node_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	mutex_lock(&memory_tier_lock);
+	memtier = __node_get_memory_tier(node);
+	if (memtier)
+		node_clear(node, memtier->nodelist);

When memtier->nodelist becomes empty, we need to free memtier?

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 110

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_unlock(&memory_tier_lock);
+}
+
+/*
+ * This runs whether reclaim-based migration is enabled or not,
+ * which ensures that the user can turn reclaim-based migration
+ * at any time without needing to recalculate migration targets.
+ */

The comments doesn't apply here.

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 112

Quoted 2 lines

mm/memory-tiers.c +2 -0

+static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
+						 unsigned long action, void *_arg)

Now we are building memory tiers instead of working on demotion. So I think we should rename the function to memtier_hotplug_callback().

Quoted 16 lines

mm/memory-tiers.c +16 -0

+{
+	struct memory_notify *arg = _arg;
+
+	/*
+	 * Only update the node migration order when a node is
+	 * changing status, like online->offline.
+	 */
+	if (arg->status_change_nid < 0)
+		return notifier_from_errno(0);
+
+	switch (action) {
+	case MEM_OFFLINE:
+		clear_node_memory_tier(arg->status_change_nid);
+		break;
+	case MEM_ONLINE:
+		init_node_memory_tier(arg->status_change_nid);

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 138

Quoted 10 lines

mm/memory-tiers.c +10 -0

+		break;
+	}
+
+	return notifier_from_errno(0);
+}
+
+static void __init migrate_on_reclaim_init(void)
+{
+	hotplug_memory_notifier(migrate_on_reclaim_callback, MEMTIER_HOTPLUG_PRIO);
+}

I suggest to call hotplug_memory_notifier() in memory_tier_init() directly. We are not working on demotion here.

Quoted 16 lines

mm/memory-tiers.c static int __init memory_tier_init(void) · line 176

Quoted 13 lines

mm/memory-tiers.c +3 -0

+
 static int __init memory_tier_init(void)
 {
 	int node;

@@ -96,6 +169,8 @@ static int __init memory_tier_init(void)
 			node_property->perf_level = default_memtier_perf_level;
 	}
 	mutex_unlock(&memory_tier_lock);
+
+	migrate_on_reclaim_init();
 	return 0;
 }
 subsys_initcall(memory_tier_init);

Best Regards,
Huang, Ying

Quoted 1 line

Will update the patch in next iteration to take care of other feedback.

Thanks!

Best Regards,
Huang, Ying

Quoted 4 lines · Ying

Quoted 2 lines · Aneesh Kumar K V

Quoted 1 line · Ying

"Aneesh Kumar K.V" <[email protected]> writes:

Quoted 21 lines

Quoted 21 lines

Quoted 15 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 79

Quoted 13 lines

mm/memory-tiers.c +12 -0

+static struct memory_tier *__node_get_memory_tier(int node)
+{
+	struct memory_tier *memtier;
+
+	list_for_each_entry(memtier, &memory_tiers, list) {
+		if (node_isset(node, memtier->nodelist))
+			return memtier;
+	}
+	return NULL;
+}
+
+static void init_node_memory_tier(int node)

set_node_memory_tier()?

That was done based on feedback from Alistair

https://lore.kernel.org/linux-mm/[email protected]

Quoted 1 line

Quoted 15 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

mm/memory-tiers.c +4 -0

+{
+	int perf_level;
+	struct memory_tier *memtier;
+

Quoted 8 lines

Quoted 8 lines

mm/memory-tiers.c static struct memory_tier *find_create_memory_tier(unsigned int perf_level) · line 91

Quoted 8 lines

mm/memory-tiers.c +8 -0

+	mutex_lock(&memory_tier_lock);
+
+	memtier = __node_get_memory_tier(node);
+	if (!memtier) {
+		perf_level = node_devices[node]->perf_level;
+		memtier = find_create_memory_tier(perf_level);
+		node_set(node, memtier->nodelist);
+	}

It's related to Alistair's comments too. When will memtier != NULL here? We may need just VM_WARN_ON() here?

When the platform driver sets memory tier directly. With the old code it can happen when dax/kmem register a node to a memory tier. With memory_type proposal this can happen if the node is part of memory type that is already added to a memory tier.

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

mm/memory-tiers.c +3 -0

+	mutex_unlock(&memory_tier_lock);
+}
+

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

As the simplest first version, this can be just hard coded.

Best Regards,
Huang, Ying

Quoted 171 lines · Ying

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

Quoted 6 lines

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

I was trying to make it architecture specific so that we have a placeholder to fine-tune this better. For example, ppc64 will look at device tree details to find the performance level and x86 will look at ACPI data structure. Adding that hotplug callback in dax/kmem will prevent that architecture-specific customization?

I would expect that callback to move to the generic ACPI layer so that even firmware managed CXL devices can be added to a lower tier? I don't understand ACPI enough to find the right abstraction for that hotplug callback.

Quoted 2 lines

As the simplest first version, this can be just hard coded.

If you are suggesting to not use hotplug callback, one of the challenge was node_devices[nid] get allocated pretty late when we try to online the node.

Quoted 185 lines · Aneesh Kumar K V

Quoted 171 lines · Ying

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

Quoted 6 lines

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

I was trying to make it architecture specific so that we have a placeholder to fine-tune this better. For example, ppc64 will look at device tree details to find the performance level and x86 will look at ACPI data structure. Adding that hotplug callback in dax/kmem will prevent that architecture-specific customization?

Quoted 3 lines

I would expect that callback to move to the generic ACPI layer so that even firmware managed CXL devices can be added to a lower tier? I don't understand ACPI enough to find the right abstraction for that hotplug callback.

I'm OK for this to be architecture specific.

But ACPI NFIT isn't enough for x86. For example, PMEM can be added to a virtual machine as normal memory nodes without NFIT. Instead, PMEM is marked via "memmap=<nn>G!<ss>G" or "efi_fake_mem=<nn>G@<ss>G:0x40000", and dax/kmem.c is used to hot-add the memory.

So, before a more sophisticated version is implemented for x86. The simplest version as I suggested below works even better.

Quoted 5 lines

Quoted 2 lines

As the simplest first version, this can be just hard coded.

If you are suggesting to not use hotplug callback, one of the challenge was node_devices[nid] get allocated pretty late when we try to online the node.

As the simplest first version, this can be as simple as,

/* dax/kmem.c */
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)

{
	node_devices[dev_dax->target_node]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
	/* add_memory_driver_managed() */
}

To be compatible with ppc64 version, how about make dev_dax_kmem_probe() set perf_level only if it's uninitialized?

Quoted 216 lines · Ying

Quoted 185 lines · Aneesh Kumar K V

Quoted 171 lines · Ying

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

Quoted 6 lines

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

I was trying to make it architecture specific so that we have a placeholder to fine-tune this better. For example, ppc64 will look at device tree details to find the performance level and x86 will look at ACPI data structure. Adding that hotplug callback in dax/kmem will prevent that architecture-specific customization?

Quoted 3 lines

I would expect that callback to move to the generic ACPI layer so that even firmware managed CXL devices can be added to a lower tier? I don't understand ACPI enough to find the right abstraction for that hotplug callback.

I'm OK for this to be architecture specific.

But ACPI NFIT isn't enough for x86. For example, PMEM can be added to a virtual machine as normal memory nodes without NFIT. Instead, PMEM is marked via "memmap=<nn>G!<ss>G" or "efi_fake_mem=<nn>G@<ss>G:0x40000", and dax/kmem.c is used to hot-add the memory.

So, before a more sophisticated version is implemented for x86. The simplest version as I suggested below works even better.

Quoted 5 lines

Quoted 2 lines

As the simplest first version, this can be just hard coded.

If you are suggesting to not use hotplug callback, one of the challenge was node_devices[nid] get allocated pretty late when we try to online the node.

As the simplest first version, this can be as simple as,

/* dax/kmem.c */
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)

{
	node_devices[dev_dax->target_node]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
	/* add_memory_driver_managed() */
}

Quoted 2 lines

To be compatible with ppc64 version, how about make dev_dax_kmem_probe() set perf_level only if it's uninitialized?

That will result in kernel crash because node_devices[dev_dax->target_node] is not initialized there.

it get allocated in add_memory_resource -> __try_online_node -> register_one_node -> __register_one_node -> node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);

Quoted 222 lines · Aneesh Kumar K V

Quoted 211 lines · Ying

Quoted 185 lines · Aneesh Kumar K V

Quoted 171 lines · Ying

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

Quoted 6 lines

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

I was trying to make it architecture specific so that we have a placeholder to fine-tune this better. For example, ppc64 will look at device tree details to find the performance level and x86 will look at ACPI data structure. Adding that hotplug callback in dax/kmem will prevent that architecture-specific customization?

Quoted 3 lines

I would expect that callback to move to the generic ACPI layer so that even firmware managed CXL devices can be added to a lower tier? I don't understand ACPI enough to find the right abstraction for that hotplug callback.

I'm OK for this to be architecture specific.

But ACPI NFIT isn't enough for x86. For example, PMEM can be added to a virtual machine as normal memory nodes without NFIT. Instead, PMEM is marked via "memmap=<nn>G!<ss>G" or "efi_fake_mem=<nn>G@<ss>G:0x40000", and dax/kmem.c is used to hot-add the memory.

So, before a more sophisticated version is implemented for x86. The simplest version as I suggested below works even better.

Quoted 5 lines

Quoted 2 lines

As the simplest first version, this can be just hard coded.

If you are suggesting to not use hotplug callback, one of the challenge was node_devices[nid] get allocated pretty late when we try to online the node.

As the simplest first version, this can be as simple as,

/* dax/kmem.c */
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)

Quoted 7 lines

{
	node_devices[dev_dax->target_node]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
	/* add_memory_driver_managed() */
}

To be compatible with ppc64 version, how about make dev_dax_kmem_probe() set perf_level only if it's uninitialized?

That will result in kernel crash because node_devices[dev_dax->target_node] is not initialized there.

Quoted 3 lines

it get allocated in add_memory_resource -> __try_online_node -> register_one_node -> __register_one_node -> node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);

Ah, right! So we need some other way to do that, for example, a global array as follows,

  int node_perf_levels[MAX_NUMNODES];

And, I think that we need to consider the memory type here too. As suggested by Johannes, memory type describes a set of memory devices (NUMA nodes) with same performance character (that is, abstract distance or perf level). The data structure can be something as follows,

  struct memory_type {
        int perf_level;
        struct list_head tier_sibling;
        nodemask_t nodes;
  };

The memory_type should be created and managed by the device drivers (e.g., dax/kmem) which manages the memory devices. In the future, we will represent it in sysfs, and a per-memory_type knob will be provided to offset the perf_level of all memory devices managed by the memory_type.

With memory_type, the memory_tier becomes,

  struct memory_tier {
        int perf_level_start;
        struct list_head sibling;
        struct list_head memory_types;
  };

And we need an array to map from nid to memory_type, e.g., as follows,

  struct memory_type *node_memory_types[MAX_NUMNODES];

We need to manage the memory_type in device drivers, instead of ACPI or device tree callbacks.

Because memory_type is an important part of the explicit memory tier implementation and may influence the design, I suggest to include it in the implementation now. It appears not too complex anyway.

Quoted 231 lines · Ying

Quoted 217 lines · Aneesh Kumar K V

Quoted 216 lines · Ying

Quoted 185 lines · Aneesh Kumar K V

Quoted 171 lines · Ying

Quoted 154 lines · Aneesh Kumar K.V

By default, all nodes are assigned to the default memory tier which is the memory tier designated for nodes with DRAM

Set dax kmem device node's tier to slower memory tier by assigning performance level to MEMTIER_PERF_LEVEL_PMEM. PMEM tier appears below the default memory tier in demotion order.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 arch/powerpc/platforms/pseries/papr_scm.c | 41 ++++++++++++++++++++---
 drivers/acpi/nfit/core.c                  | 41 ++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 6 deletions(-)

arch/powerpc/platforms/pseries/papr_scm.c +36 -5

index 82cae08976bc..3b6164418d6f 100644
@@ -14,6 +14,8 @@
 #include <linux/delay.h>
 #include <linux/seq_buf.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 
 #include <asm/plpar_wrappers.h>
 #include <asm/papr_pdsm.h>

@@ -98,6 +100,7 @@ struct papr_scm_priv {
 	bool hcall_flush_required;
 
 	uint64_t bound_addr;
+	int target_node;
 
 	struct nvdimm_bus_descriptor bus_desc;
 	struct nvdimm_bus *bus;

@@ -1278,6 +1281,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	p->bus_desc.module = THIS_MODULE;
 	p->bus_desc.of_node = p->pdev->dev.of_node;
 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
+	p->target_node = dev_to_node(&p->pdev->dev);
 
 	/* Set the dimm command family mask to accept PDSMs */
 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);

@@ -1322,7 +1326,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?
 
 	memset(&ndr_desc, 0, sizeof(ndr_desc));
-	target_nid = dev_to_node(&p->pdev->dev);
+	target_nid = p->target_node;
 	online_nid = numa_map_to_online_node(target_nid);
 	ndr_desc.numa_node = online_nid;
 	ndr_desc.target_node = target_nid;

@@ -1582,15 +1586,42 @@ static struct platform_driver papr_scm_driver = {
 	},
 };
 
+static int papr_scm_callback(struct notifier_block *self,
+			     unsigned long action, void *arg)
+{
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct papr_scm_priv *p;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&papr_ndr_lock);
+	list_for_each_entry(p, &papr_nd_regions, region_list) {
+		if (p->target_node == nid) {
+			node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+			break;
+		}
+	}
+
+	mutex_unlock(&papr_ndr_lock);
+	return NOTIFY_OK;
+}
+
 static int __init papr_scm_init(void)
 {
 	int ret;
 
 	ret = platform_driver_register(&papr_scm_driver);
-	if (!ret)
-		mce_register_notifier(&mce_ue_nb);
-
-	return ret;
+	if (ret)
+		return ret;
+	mce_register_notifier(&mce_ue_nb);
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(papr_scm_callback, MEMTIER_HOTPLUG_PRIO + 1);
+	return 0;
 }
 module_init(papr_scm_init);
 

drivers/acpi/nfit/core.c +35 -0

index ae5f4acf2675..7ea1017ef790 100644
@@ -15,6 +15,8 @@
 #include <linux/sort.h>
 #include <linux/io.h>
 #include <linux/nd.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
 #include <asm/cacheflush.h>
 #include <acpi/nfit.h>
 #include "intel.h"

@@ -3470,6 +3472,39 @@ static struct acpi_driver acpi_nfit_driver = {
 	},
 };
 
+static int nfit_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	bool found = false;
+	struct memory_notify *mnb = arg;
+	int nid = mnb->status_change_nid;
+	struct nfit_spa *nfit_spa;
+	struct acpi_nfit_desc *acpi_desc;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	mutex_lock(&acpi_desc_lock);
+	list_for_each_entry(acpi_desc, &acpi_descs, list) {
+		mutex_lock(&acpi_desc->init_mutex);
+		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
+			struct acpi_nfit_system_address *spa = nfit_spa->spa;
+			int target_node = pxm_to_node(spa->proximity_domain);
+
+			if (target_node == nid) {
+				node_devices[nid]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&acpi_desc->init_mutex);
+		if (found)
+			break;
+	}
+	mutex_unlock(&acpi_desc_lock);
+	return NOTIFY_OK;
+}
+
 static __init int nfit_init(void)
 {
 	int ret;

@@ -3509,7 +3544,11 @@ static __init int nfit_init(void)
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);

drivers/acpi/nfit/core.c static __init int nfit_init(void) · line 3554

Quoted 10 lines

drivers/acpi/nfit/core.c +5 -1

 	}
-
+	/*
+	 * register a memory hotplug notifier at prio 2 so that we
+	 * can update the perf level for the node.
+	 */
+	hotplug_memory_notifier(nfit_callback, MEMTIER_HOTPLUG_PRIO + 1);
 	return ret;
 
 }

I don't think that it's a good idea to set perf_level of a memory device (node) via NFIT only.

Quoted 6 lines

For example, we may prefer HMAT over NFIT when it's available. So the perf_level should be set in dax/kmem.c based on information provided by ACPI or other information sources. ACPI can provide some functions/data structures to let drivers (like dax/kmem.c) to query the properties of the memory device (node).

I was trying to make it architecture specific so that we have a placeholder to fine-tune this better. For example, ppc64 will look at device tree details to find the performance level and x86 will look at ACPI data structure. Adding that hotplug callback in dax/kmem will prevent that architecture-specific customization?

Quoted 3 lines

I would expect that callback to move to the generic ACPI layer so that even firmware managed CXL devices can be added to a lower tier? I don't understand ACPI enough to find the right abstraction for that hotplug callback.

I'm OK for this to be architecture specific.

But ACPI NFIT isn't enough for x86. For example, PMEM can be added to a virtual machine as normal memory nodes without NFIT. Instead, PMEM is marked via "memmap=<nn>G!<ss>G" or "efi_fake_mem=<nn>G@<ss>G:0x40000", and dax/kmem.c is used to hot-add the memory.

So, before a more sophisticated version is implemented for x86. The simplest version as I suggested below works even better.

Quoted 5 lines

Quoted 2 lines

As the simplest first version, this can be just hard coded.

If you are suggesting to not use hotplug callback, one of the challenge was node_devices[nid] get allocated pretty late when we try to online the node.

As the simplest first version, this can be as simple as,

/* dax/kmem.c */
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)

{
	node_devices[dev_dax->target_node]->perf_level = MEMTIER_PERF_LEVEL_PMEM;
	/* add_memory_driver_managed() */
}

Quoted 8 lines

Quoted 2 lines

To be compatible with ppc64 version, how about make dev_dax_kmem_probe() set perf_level only if it's uninitialized?

That will result in kernel crash because node_devices[dev_dax->target_node] is not initialized there.

it get allocated in add_memory_resource -> __try_online_node -> register_one_node -> __register_one_node -> node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);

Ah, right! So we need some other way to do that, for example, a global array as follows,

Quoted 1 line

  int node_perf_levels[MAX_NUMNODES];

This would be much simpler than adding memory_type, but then it is a memory device property and hence it will be a good idea to group it together with other properties in node_devices[]. We could look at allocating nodes_devices[] for dax/kmem nodes from within the driver?

I did implement memory_type and it do bring some additional complexity though it simplfy the interface.

I was looking at the platform drivers calling
struct memory_dev_type *register_memory_type(int perf_level, int node) to register a new memory_type. if dax/kmem don't find a memory_dev_type registered for the NUMA node it will assign default pmem type.

	if (!node_memory_types[numa_node]) {
		/*
		 * low level drivers didn't initialize the memory type.
		 * assign a default level.
		 */
		node_memory_types[numa_node] = &default_pmem_type;
		node_set(numa_node, default_pmem_type.nodes);
	}

This should allow ACPI or papr_scm to fine tune the memory type of the deivce they are initializing

Quoted 11 lines

And, I think that we need to consider the memory type here too. As suggested by Johannes, memory type describes a set of memory devices (NUMA nodes) with same performance character (that is, abstract distance or perf level). The data structure can be something as follows,

  struct memory_type {
        int perf_level;
        struct list_head tier_sibling;
        nodemask_t nodes;
  };

memory type is already used in include/linux/memremap.h

enum memory_type

How about struct memory_dev_type?

How about we also add memtier here that is only accessed with memory_tier_lock held? That will allow easy access to the memory tier this type belongs

Quoted 15 lines

The memory_type should be created and managed by the device drivers (e.g., dax/kmem) which manages the memory devices. In the future, we will represent it in sysfs, and a per-memory_type knob will be provided to offset the perf_level of all memory devices managed by the memory_type.

With memory_type, the memory_tier becomes,

  struct memory_tier {
        int perf_level_start;
        struct list_head sibling;
        struct list_head memory_types;
  };

Quoted 3 lines

And we need an array to map from nid to memory_type, e.g., as follows,

  struct memory_type *node_memory_types[MAX_NUMNODES];

Changing the perf level of a memory devices will move it from one memory type to the other and such a move should will also results in updating node's memory tier. ie, it will be something like below

static void update_node_perf_level(int node, struct memory_dev_type *memtype)

{
	pg_data_t *pgdat;
	struct memory_dev_type *current_memtype;
	struct memory_tier *memtier;

	pgdat = NODE_DATA(node);
	if (!pgdat)
		return;

	mutex_lock(&memory_tier_lock);
	/*
	 * Make sure we mark the memtier NULL before we assign the new memory tier
	 * to the NUMA node. This make sure that anybody looking at NODE_DATA
	 * finds a NULL memtier or the one which is still valid.
	 */
	rcu_assign_pointer(pgdat->memtier, NULL);
	synchronize_rcu();

	if (!memtype->memtier) {
		memtier = find_create_memory_tier(memtype);
		if (IS_ERR(memtier))
			goto err_out;
	}
	current_memtype = node_memory_types[node];
	node_clear(node, current_memtype->nodes);
	/*
	 * If current memtype becomes empty, we should kill the memory tiers
	 */
	node_set(node, memtype->nodes);
        node_memory_types[node] = memtype;
	rcu_assign_pointer(pgdat->memtier, memtype->memtier);
	establish_migration_targets();
err_out:
	mutex_unlock(&memory_tier_lock);
}

Quoted 8 lines

We need to manage the memory_type in device drivers, instead of ACPI or device tree callbacks.

Because memory_type is an important part of the explicit memory tier implementation and may influence the design, I suggest to include it in the implementation now. It appears not too complex anyway.

One thing I observed while implementing this is the additional complexity while walking the memory tiers. Any tier related operation impacting memory numa nodes now becomes a two linked list walk as below.

ex:

list_for_each_entry(memtier, &memory_tiers, list) {
	list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling)
		nodes_or(nodes, nodes, memtype->nodes);

As we offline N_MEMORY nodes, we now have to do

	memtype = node_memory_types[node];
        if (nodes_empty(memtype->nodes)) {
        	list_del(&memtype->tier_sibiling);
                        if (list_empty(&current_memtier->memory_types))
                        	destroy_memory_tier(current_memtier);

This patch switch the demotion target building logic to use memory tiers instead of NUMA distance. All N_MEMORY NUMA nodes will be placed in the default memory tier and additional memory tiers will be added by drivers like dax kmem.

This patch builds the demotion target for a NUMA node by looking at all memory tiers below the tier to which the NUMA node belongs. The closest node in the immediately following memory tier is used as a demotion target.

Since we are now only building demotion target for N_MEMORY NUMA nodes the CPU hotplug calls are removed in this patch.

A new memory tier can be inserted into the tier hierarchy for a new set of nodes without affecting the node assignment of any existing memtier, provided that there is enough gap in the performance level values for the new memtier.

The absolute value of performance level of a memtier doesn't necessarily carry any meaning. Its value relative to other memtiers decides the level of this memtier in the tier hierarchy.

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 include/linux/memory-tiers.h |  12 ++
 include/linux/migrate.h      |  13 --
 mm/memory-tiers.c            | 218 ++++++++++++++++++-
 mm/migrate.c                 | 394 -----------------------------------
 mm/vmstat.c                  |   4 -
 5 files changed, 229 insertions(+), 412 deletions(-)

include/linux/memory-tiers.h +12 -0

index 3d5f14d57ae6..852e86bd0a23 100644
@@ -17,9 +17,21 @@
 #define MEMTIER_HOTPLUG_PRIO	100
 
 extern bool numa_demotion_enabled;
+#ifdef CONFIG_MIGRATION
+int next_demotion_node(int node);
+#else
+static inline int next_demotion_node(int node)
+{
+	return NUMA_NO_NODE;
+}
+#endif
 
 #else
 
 #define numa_demotion_enabled	false
+static inline int next_demotion_node(int node)
+{
+	return NUMA_NO_NODE;
+}
 #endif	/* CONFIG_NUMA */
 #endif  /* _LINUX_MEMORY_TIERS_H */

include/linux/migrate.h +0 -13

index 43e737215f33..93fab62e6548 100644
@@ -75,19 +75,6 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
 
 #endif /* CONFIG_MIGRATION */
 
-#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA)
-extern void set_migration_target_nodes(void);
-extern void migrate_on_reclaim_init(void);
-extern int next_demotion_node(int node);
-#else
-static inline void set_migration_target_nodes(void) {}
-static inline void migrate_on_reclaim_init(void) {}
-static inline int next_demotion_node(int node)
-{
-        return NUMA_NO_NODE;
-}
-#endif
-
 #ifdef CONFIG_COMPACTION
 extern int PageMovable(struct page *page);
 extern void __SetPageMovable(struct page *page, struct address_space *mapping);

mm/memory-tiers.c +15 -0

index cc3a47ec18e4..a8cfe2ca3903 100644
@@ -6,17 +6,88 @@
 #include <linux/moduleparam.h>
 #include <linux/node.h>
 #include <linux/memory.h>
+#include <linux/random.h>
 #include <linux/memory-tiers.h>
 
+#include "internal.h"
+
 struct memory_tier {
 	struct list_head list;
 	int perf_level;
 	nodemask_t nodelist;
 };
 
+struct demotion_nodes {
+	nodemask_t preferred;
+};
+
 static LIST_HEAD(memory_tiers);
 static DEFINE_MUTEX(memory_tier_lock);
 
+#ifdef CONFIG_MIGRATION
+/*
+ * node_demotion[] examples:
+ *
+ * Example 1:
+ *
+ * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes.
+ *

mm/memory-tiers.c +77 -0

+ *
+ * node_demotion[0].preferred = 2
+ * node_demotion[1].preferred = 3
+ * node_demotion[2].preferred = <empty>
+ * node_demotion[3].preferred = <empty>
+ *
+ * Example 2:
+ *
+ * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node.
+ *
+ * node distances:
+ * node   0    1    2
+ *    0  10   20   30
+ *    1  20   10   30
+ *    2  30   30   10
+ *
+ * memory_tiers[0] = <empty>
+ * memory_tiers[1] = 0-2
+ * memory_tiers[2] = <empty>
+ *
+ * node_demotion[0].preferred = <empty>
+ * node_demotion[1].preferred = <empty>
+ * node_demotion[2].preferred = <empty>
+ *
+ * Example 3:
+ *
+ * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node.
+ *
+ * node distances:
+ * node   0    1    2
+ *    0  10   20   30
+ *    1  20   10   40
+ *    2  30   40   10
+ *
+ * memory_tiers[0] = 1
+ * memory_tiers[1] = 0
+ * memory_tiers[2] = 2
+ *
+ * node_demotion[0].preferred = 2
+ * node_demotion[1].preferred = 0
+ * node_demotion[2].preferred = <empty>
+ *
+ */
+static struct demotion_nodes *node_demotion __read_mostly;
+#endif /* CONFIG_MIGRATION */
+
 /*
  * For now let's have 4 memory tier below default DRAM tier.
  */

@@ -76,6 +147,136 @@ static struct memory_tier *__node_get_memory_tier(int node)
 	return NULL;
 }
 
+#ifdef CONFIG_MIGRATION
+/**
+ * next_demotion_node() - Get the next node in the demotion path
+ * @node: The starting node to lookup the next node
+ *
+ * Return: node id for next memory node in the demotion path hierarchy
+ * from @node; NUMA_NO_NODE if @node is terminal.  This does not keep
+ * @node online or guarantee that it *continues* to be the next demotion
+ * target.
+ */
+int next_demotion_node(int node)
+{
+	struct demotion_nodes *nd;
+	int target;
+
+	if (!node_demotion)
+		return NUMA_NO_NODE;
+
+	nd = &node_demotion[node];
+
+	/*
+	 * node_demotion[] is updated without excluding this
+	 * function from running.
+	 *
+	 * Make sure to use RCU over entire code blocks if
+	 * node_demotion[] reads need to be consistent.
+	 */
+	rcu_read_lock();
+	/*
+	 * If there are multiple target nodes, just select one
+	 * target node randomly.

mm/memory-tiers.c +85 -0

+{
+	int node;
+
+	for_each_node_state(node, N_MEMORY)
+		node_demotion[node].preferred = NODE_MASK_NONE;
+}
+
+static void disable_all_migrate_targets(void)
+{
+	__disable_all_migrate_targets();
+
+	/*
+	 * Ensure that the "disable" is visible across the system.
+	 * Readers will see either a combination of before+disable
+	 * state or disable+after.  They will never see before and
+	 * after state together.
+	 */
+	synchronize_rcu();
+}
+/*
+ * Find an automatic demotion target for all memory
+ * nodes. Failing here is OK.  It might just indicate
+ * being at the end of a chain.
+ */
+static void establish_migration_targets(void)
+{
+	struct memory_tier *memtier;
+	struct demotion_nodes *nd;
+	int target = NUMA_NO_NODE, node;
+	int distance, best_distance;
+	nodemask_t used;
+
+	if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION))
+		return;
+
+	disable_all_migrate_targets();
+
+	for_each_node_state(node, N_MEMORY) {
+		best_distance = -1;
+		nd = &node_demotion[node];
+
+		memtier = __node_get_memory_tier(node);
+		if (!memtier || list_is_last(&memtier->list, &memory_tiers))
+			continue;
+		/*
+		 * Get the next memtier to find the  demotion node list.
+		 */
+		memtier = list_next_entry(memtier, list);
+
+		/*
+		 * find_next_best_node, use 'used' nodemask as a skip list.
+		 * Add all memory nodes except the selected memory tier
+		 * nodelist to skip list so that we find the best node from the
+		 * memtier nodelist.
+		 */
+		nodes_andnot(used, node_states[N_MEMORY], memtier->nodelist);
+
+		/*
+		 * Find all the nodes in the memory tier node list of same best distance.
+		 * add them to the preferred mask. We randomly select between nodes
+		 * in the preferred mask when allocating pages during demotion.
+		 */
+		do {
+			target = find_next_best_node(node, &used);
+			if (target == NUMA_NO_NODE)
+				break;
+
+			distance = node_distance(node, target);
+			if (distance == best_distance || best_distance == -1) {
+				best_distance = distance;
+				node_set(target, nd->preferred);
+			} else {
+				break;
+			}
+		} while (1);
+	}
+}
+#else
+static inline void disable_all_migrate_targets(void) {}
+static inline void establish_migration_targets(void) {}
+#endif /* CONFIG_MIGRATION */
+
 static void init_node_memory_tier(int node)
 {
 	int perf_level;

@@ -84,11 +285,19 @@ static void init_node_memory_tier(int node)
 	mutex_lock(&memory_tier_lock);
 
 	memtier = __node_get_memory_tier(node);
+	/*
+	 * if node is already part of the tier proceed with the
+	 * current tier value, because we might want to establish

mm/memory-tiers.c +3 -1

 	mutex_unlock(&memory_tier_lock);
 }
 

@@ -98,8 +307,10 @@ static void clear_node_memory_tier(int node)
 
 	mutex_lock(&memory_tier_lock);
 	memtier = __node_get_memory_tier(node);
-	if (memtier)
+	if (memtier) {
 		node_clear(node, memtier->nodelist);
+		establish_migration_targets();
+	}
 	mutex_unlock(&memory_tier_lock);
 }
 

mm/memory-tiers.c +3 -0

+					GFP_KERNEL);
+		WARN_ON(!node_demotion);
+	}
 	hotplug_memory_notifier(migrate_on_reclaim_callback, MEMTIER_HOTPLUG_PRIO);
 }
 

mm/migrate.c +0 -394

index fce7d4a9e940..c758c9c21d7d 100644
@@ -2117,398 +2117,4 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
 	return 0;
 }
 #endif /* CONFIG_NUMA_BALANCING */
-
-/*
- * node_demotion[] example:
- *
- * Consider a system with two sockets.  Each socket has
- * three classes of memory attached: fast, medium and slow.
- * Each memory class is placed in its own NUMA node.  The
- * CPUs are placed in the node with the "fast" memory.  The
- * 6 NUMA nodes (0-5) might be split among the sockets like
- * this:
- *
- *	Socket A: 0, 1, 2
- *	Socket B: 3, 4, 5
- *
- * When Node 0 fills up, its memory should be migrated to
- * Node 1.  When Node 1 fills up, it should be migrated to
- * Node 2.  The migration path start on the nodes with the
- * processors (since allocations default to this node) and
- * fast memory, progress through medium and end with the
- * slow memory:
- *
- *	0 -> 1 -> 2 -> stop
- *	3 -> 4 -> 5 -> stop
- *
- * This is represented in the node_demotion[] like this:
- *
- *	{  nr=1, nodes[0]=1 }, // Node 0 migrates to 1
- *	{  nr=1, nodes[0]=2 }, // Node 1 migrates to 2
- *	{  nr=0, nodes[0]=-1 }, // Node 2 does not migrate
- *	{  nr=1, nodes[0]=4 }, // Node 3 migrates to 4
- *	{  nr=1, nodes[0]=5 }, // Node 4 migrates to 5
- *	{  nr=0, nodes[0]=-1 }, // Node 5 does not migrate
- *
- * Moreover some systems may have multiple slow memory nodes.
- * Suppose a system has one socket with 3 memory nodes, node 0
- * is fast memory type, and node 1/2 both are slow memory
- * type, and the distance between fast memory node and slow
- * memory node is same. So the migration path should be:
- *
- *	0 -> 1/2 -> stop
- *
- * This is represented in the node_demotion[] like this:
- *	{ nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2
- *	{ nr=0, nodes[0]=-1, }, // Node 1 dose not migrate
- *	{ nr=0, nodes[0]=-1, }, // Node 2 does not migrate
- */
-
-/*
- * Writes to this array occur without locking.  Cycles are
- * not allowed: Node X demotes to Y which demotes to X...
- *
- * If multiple reads are performed, a single rcu_read_lock()
- * must be held over all reads to ensure that no cycles are
- * observed.
- */
-#define DEFAULT_DEMOTION_TARGET_NODES 15
-
-#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES
-#define DEMOTION_TARGET_NODES	(MAX_NUMNODES - 1)
-#else
-#define DEMOTION_TARGET_NODES	DEFAULT_DEMOTION_TARGET_NODES
-#endif
-
-struct demotion_nodes {
-	unsigned short nr;
-	short nodes[DEMOTION_TARGET_NODES];
-};
-
-static struct demotion_nodes *node_demotion __read_mostly;
-
-/**
- * next_demotion_node() - Get the next node in the demotion path
- * @node: The starting node to lookup the next node
- *
- * Return: node id for next memory node in the demotion path hierarchy
- * from @node; NUMA_NO_NODE if @node is terminal.  This does not keep
- * @node online or guarantee that it *continues* to be the next demotion
- * target.
- */
-int next_demotion_node(int node)
-{
-	struct demotion_nodes *nd;
-	unsigned short target_nr, index;
-	int target;
-
-	if (!node_demotion)
-		return NUMA_NO_NODE;
-
-	nd = &node_demotion[node];
-
-	/*
-	 * node_demotion[] is updated without excluding this
-	 * function from running.  RCU doesn't provide any
-	 * compiler barriers, so the READ_ONCE() is required
-	 * to avoid compiler reordering or read merging.
-	 *
-	 * Make sure to use RCU over entire code blocks if
-	 * node_demotion[] reads need to be consistent.
-	 */
-	rcu_read_lock();
-	target_nr = READ_ONCE(nd->nr);
-
-	switch (target_nr) {
-	case 0:
-		target = NUMA_NO_NODE;
-		goto out;
-	case 1:
-		index = 0;
-		break;
-	default:
-		/*
-		 * If there are multiple target nodes, just select one
-		 * target node randomly.
-		 *
-		 * In addition, we can also use round-robin to select
-		 * target node, but we should introduce another variable
-		 * for node_demotion[] to record last selected target node,
-		 * that may cause cache ping-pong due to the changing of
-		 * last target node. Or introducing per-cpu data to avoid
-		 * caching issue, which seems more complicated. So selecting
-		 * target node randomly seems better until now.
-		 */
-		index = get_random_int() % target_nr;
-		break;
-	}
-
-	target = READ_ONCE(nd->nodes[index]);
-
-out:
-	rcu_read_unlock();
-	return target;
-}
-
-/* Disable reclaim-based migration. */
-static void __disable_all_migrate_targets(void)
-{
-	int node, i;
-
-	if (!node_demotion)
-		return;
-
-	for_each_online_node(node) {
-		node_demotion[node].nr = 0;
-		for (i = 0; i < DEMOTION_TARGET_NODES; i++)
-			node_demotion[node].nodes[i] = NUMA_NO_NODE;
-	}
-}
-
-static void disable_all_migrate_targets(void)
-{
-	__disable_all_migrate_targets();
-
-	/*
-	 * Ensure that the "disable" is visible across the system.
-	 * Readers will see either a combination of before+disable
-	 * state or disable+after.  They will never see before and
-	 * after state together.
-	 *
-	 * The before+after state together might have cycles and
-	 * could cause readers to do things like loop until this
-	 * function finishes.  This ensures they can only see a
-	 * single "bad" read and would, for instance, only loop
-	 * once.
-	 */
-	synchronize_rcu();
-}
-
-/*
- * Find an automatic demotion target for 'node'.
- * Failing here is OK.  It might just indicate
- * being at the end of a chain.
- */
-static int establish_migrate_target(int node, nodemask_t *used,
-				    int best_distance)
-{
-	int migration_target, index, val;
-	struct demotion_nodes *nd;
-
-	if (!node_demotion)
-		return NUMA_NO_NODE;
-
-	nd = &node_demotion[node];
-
-	migration_target = find_next_best_node(node, used);
-	if (migration_target == NUMA_NO_NODE)
-		return NUMA_NO_NODE;
-
-	/*
-	 * If the node has been set a migration target node before,
-	 * which means it's the best distance between them. Still
-	 * check if this node can be demoted to other target nodes
-	 * if they have a same best distance.
-	 */
-	if (best_distance != -1) {
-		val = node_distance(node, migration_target);
-		if (val > best_distance)
-			goto out_clear;
-	}
-
-	index = nd->nr;
-	if (WARN_ONCE(index >= DEMOTION_TARGET_NODES,
-		      "Exceeds maximum demotion target nodes\n"))
-		goto out_clear;
-
-	nd->nodes[index] = migration_target;
-	nd->nr++;
-
-	return migration_target;
-out_clear:
-	node_clear(migration_target, *used);
-	return NUMA_NO_NODE;
-}
-
-/*
- * When memory fills up on a node, memory contents can be
- * automatically migrated to another node instead of
- * discarded at reclaim.
- *
- * Establish a "migration path" which will start at nodes
- * with CPUs and will follow the priorities used to build the
- * page allocator zonelists.
- *
- * The difference here is that cycles must be avoided.  If
- * node0 migrates to node1, then neither node1, nor anything
- * node1 migrates to can migrate to node0. Also one node can
- * be migrated to multiple nodes if the target nodes all have
- * a same best-distance against the source node.
- *
- * This function can run simultaneously with readers of
- * node_demotion[].  However, it can not run simultaneously
- * with itself.  Exclusion is provided by memory hotplug events
- * being single-threaded.
- */
-static void __set_migration_target_nodes(void)
-{
-	nodemask_t next_pass;
-	nodemask_t this_pass;
-	nodemask_t used_targets = NODE_MASK_NONE;
-	int node, best_distance;
-
-	/*
-	 * Avoid any oddities like cycles that could occur
-	 * from changes in the topology.  This will leave
-	 * a momentary gap when migration is disabled.
-	 */
-	disable_all_migrate_targets();
-
-	/*
-	 * Allocations go close to CPUs, first.  Assume that
-	 * the migration path starts at the nodes with CPUs.
-	 */
-	next_pass = node_states[N_CPU];
-again:
-	this_pass = next_pass;
-	next_pass = NODE_MASK_NONE;
-	/*
-	 * To avoid cycles in the migration "graph", ensure
-	 * that migration sources are not future targets by
-	 * setting them in 'used_targets'.  Do this only
-	 * once per pass so that multiple source nodes can
-	 * share a target node.
-	 *
-	 * 'used_targets' will become unavailable in future
-	 * passes.  This limits some opportunities for
-	 * multiple source nodes to share a destination.
-	 */
-	nodes_or(used_targets, used_targets, this_pass);
-
-	for_each_node_mask(node, this_pass) {
-		best_distance = -1;
-
-		/*
-		 * Try to set up the migration path for the node, and the target
-		 * migration nodes can be multiple, so doing a loop to find all
-		 * the target nodes if they all have a best node distance.
-		 */
-		do {
-			int target_node =
-				establish_migrate_target(node, &used_targets,
-							 best_distance);
-
-			if (target_node == NUMA_NO_NODE)
-				break;
-
-			if (best_distance == -1)
-				best_distance = node_distance(node, target_node);
-
-			/*
-			 * Visit targets from this pass in the next pass.
-			 * Eventually, every node will have been part of
-			 * a pass, and will become set in 'used_targets'.
-			 */
-			node_set(target_node, next_pass);
-		} while (1);
-	}
-	/*
-	 * 'next_pass' contains nodes which became migration
-	 * targets in this pass.  Make additional passes until
-	 * no more migrations targets are available.
-	 */
-	if (!nodes_empty(next_pass))
-		goto again;
-}
-
-/*
- * For callers that do not hold get_online_mems() already.
- */
-void set_migration_target_nodes(void)
-{
-	get_online_mems();
-	__set_migration_target_nodes();
-	put_online_mems();
-}
-
-/*
- * This leaves migrate-on-reclaim transiently disabled between
- * the MEM_GOING_OFFLINE and MEM_OFFLINE events.  This runs
- * whether reclaim-based migration is enabled or not, which
- * ensures that the user can turn reclaim-based migration at
- * any time without needing to recalculate migration targets.
- *
- * These callbacks already hold get_online_mems().  That is why
- * __set_migration_target_nodes() can be used as opposed to
- * set_migration_target_nodes().
- */
-#ifdef CONFIG_MEMORY_HOTPLUG
-static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
-						 unsigned long action, void *_arg)
-{
-	struct memory_notify *arg = _arg;
-
-	/*
-	 * Only update the node migration order when a node is
-	 * changing status, like online->offline.  This avoids
-	 * the overhead of synchronize_rcu() in most cases.
-	 */
-	if (arg->status_change_nid < 0)
-		return notifier_from_errno(0);
-
-	switch (action) {
-	case MEM_GOING_OFFLINE:
-		/*
-		 * Make sure there are not transient states where
-		 * an offline node is a migration target.  This
-		 * will leave migration disabled until the offline
-		 * completes and the MEM_OFFLINE case below runs.
-		 */
-		disable_all_migrate_targets();
-		break;
-	case MEM_OFFLINE:
-	case MEM_ONLINE:
-		/*
-		 * Recalculate the target nodes once the node
-		 * reaches its final state (online or offline).
-		 */
-		__set_migration_target_nodes();
-		break;
-	case MEM_CANCEL_OFFLINE:
-		/*
-		 * MEM_GOING_OFFLINE disabled all the migration
-		 * targets.  Reenable them.
-		 */
-		__set_migration_target_nodes();
-		break;
-	case MEM_GOING_ONLINE:
-	case MEM_CANCEL_ONLINE:
-		break;
-	}
-
-	return notifier_from_errno(0);
-}
-#endif
-
-void __init migrate_on_reclaim_init(void)
-{
-	node_demotion = kcalloc(nr_node_ids,
-				sizeof(struct demotion_nodes),
-				GFP_KERNEL);
-	WARN_ON(!node_demotion);
-#ifdef CONFIG_MEMORY_HOTPLUG
-	hotplug_memory_notifier(migrate_on_reclaim_callback, 100);
-#endif
-	/*
-	 * At this point, all numa nodes with memory/CPus have their state
-	 * properly set, so we can build the demotion order now.
-	 * Let us hold the cpu_hotplug lock just, as we could possibily have
-	 * CPU hotplug events during boot.
-	 */
-	cpus_read_lock();
-	set_migration_target_nodes();
-	cpus_read_unlock();
-}
 #endif /* CONFIG_NUMA */
-
-

mm/vmstat.c +0 -1

index 373d2730fcf2..35c6ff97cf29 100644
@@ -28,7 +28,6 @@
 #include <linux/mm_inline.h>
 #include <linux/page_ext.h>
 #include <linux/page_owner.h>
-#include <linux/migrate.h>
 
 #include "internal.h"