mm/demotion: Memory tiers and demotion

From: Jagdish Gediya <[email protected]>

mm/vmscan.c +10 -12

-static struct page *alloc_demote_page(struct page *page, unsigned long node)
+static struct page *alloc_demote_page(struct page *page, unsigned long private)
 {
-	struct migration_target_control mtc = {
-		/*
-		 * Allocate from 'node', or fail quickly and quietly.
-		 * When this happens, 'page' will likely just be discarded
-		 * instead of migrated.
-		 */
-		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
-			    __GFP_THISNODE  | __GFP_NOWARN |
-			    __GFP_NOMEMALLOC | GFP_NOWAIT,
-		.nid = node
-	};
+	struct page *target_page;
+	nodemask_t *allowed_mask;
+	struct migration_target_control *mtc;
+
+	mtc = (struct migration_target_control *)private;
+
+	allowed_mask = mtc->nmask;
+	/*
+	 * make sure we allocate from the target node first also trying to

}

mm/vmscan.c +17 -2

@@ -1487,6 +1500,19 @@ static unsigned int demote_page_list(struct list_head *demote_pages,
 {
 	int target_nid = next_demotion_node(pgdat->node_id);
 	unsigned int nr_succeeded;
+	nodemask_t allowed_mask;
+
+	struct migration_target_control mtc = {
+		/*
+		 * Allocate from 'node', or fail quickly and quietly.
+		 * When this happens, 'page' will likely just be discarded
+		 * instead of migrated.
+		 */
+		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN |
+			__GFP_NOMEMALLOC | GFP_NOWAIT,
+		.nid = target_nid,
+		.nmask = &allowed_mask
+	};
 
 	if (list_empty(demote_pages))
 		return 0;

@@ -1494,10 +1520,12 @@ static unsigned int demote_page_list(struct list_head *demote_pages,
 	if (target_nid == NUMA_NO_NODE)
 		return 0;
 
+	node_get_allowed_targets(pgdat, &allowed_mask);
+
 	/* Demotion ignores all cpuset and mempolicy settings */
 	migrate_pages(demote_pages, alloc_demote_page, NULL,
-			    target_nid, MIGRATE_ASYNC, MR_DEMOTION,
-			    &nr_succeeded);
+		      (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
+		      &nr_succeeded);
 
 	if (current_is_kswapd())
 		__count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded);

Quoted 1 line · Aneesh Kumar K.V

From: Jagdish Gediya <[email protected]>

[...]

Quoted 29 lines

mm/vmscan.c +16 -12

-static struct page *alloc_demote_page(struct page *page, unsigned long node)
+static struct page *alloc_demote_page(struct page *page, unsigned long private)
 {
-	struct migration_target_control mtc = {
-		/*
-		 * Allocate from 'node', or fail quickly and quietly.
-		 * When this happens, 'page' will likely just be discarded
-		 * instead of migrated.
-		 */
-		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
-			    __GFP_THISNODE  | __GFP_NOWARN |
-			    __GFP_NOMEMALLOC | GFP_NOWAIT,
-		.nid = node
-	};
+	struct page *target_page;
+	nodemask_t *allowed_mask;
+	struct migration_target_control *mtc;
+
+	mtc = (struct migration_target_control *)private;
+
+	allowed_mask = mtc->nmask;
+	/*
+	 * make sure we allocate from the target node first also trying to
+	 * reclaim pages from the target node via kswapd if we are low on
+	 * free memory on target node. If we don't do this and if we have low
+	 * free memory on the target memtier, we would start allocating pages
+	 * from higher memory tiers without even forcing a demotion of cold
+	 * pages from the target memtier. This can result in the kernel placing
+	 * hotpages in higher memory tiers.

mm/vmscan.c static void folio_check_dirty_writeback(struct folio *folio, · line 1483

Quoted 4 lines

mm/vmscan.c +4 -0

+	 */
+	mtc->nmask = NULL;
+	mtc->gfp_mask |= __GFP_THISNODE;
+	target_page = alloc_migration_target(page, (unsigned long)&mtc);

I finally managed to get a system setup to start testing some of this out. However it quickly crashed due to the bad pointer in the above call
- you need mtc not &mtc here.

mm/vmscan.c static void folio_check_dirty_writeback(struct folio *folio, · line 1490

Quoted 8 lines

mm/vmscan.c +5 -0

+	if (target_page)
+		return target_page;
+
+	mtc->gfp_mask &= ~__GFP_THISNODE;
+	mtc->nmask = allowed_mask;
 
 	return alloc_migration_target(page, (unsigned long)&mtc);

And here.

Quoted 35 lines · Alistair Popple

Quoted 1 line · Aneesh Kumar K.V

From: Jagdish Gediya <[email protected]>

[...]

Quoted 29 lines

mm/vmscan.c +16 -12

-static struct page *alloc_demote_page(struct page *page, unsigned long node)
+static struct page *alloc_demote_page(struct page *page, unsigned long private)
 {
-	struct migration_target_control mtc = {
-		/*
-		 * Allocate from 'node', or fail quickly and quietly.
-		 * When this happens, 'page' will likely just be discarded
-		 * instead of migrated.
-		 */
-		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
-			    __GFP_THISNODE  | __GFP_NOWARN |
-			    __GFP_NOMEMALLOC | GFP_NOWAIT,
-		.nid = node
-	};
+	struct page *target_page;
+	nodemask_t *allowed_mask;
+	struct migration_target_control *mtc;
+
+	mtc = (struct migration_target_control *)private;
+
+	allowed_mask = mtc->nmask;
+	/*
+	 * make sure we allocate from the target node first also trying to
+	 * reclaim pages from the target node via kswapd if we are low on
+	 * free memory on target node. If we don't do this and if we have low
+	 * free memory on the target memtier, we would start allocating pages
+	 * from higher memory tiers without even forcing a demotion of cold
+	 * pages from the target memtier. This can result in the kernel placing
+	 * hotpages in higher memory tiers.

Quoted 8 lines

mm/vmscan.c static void folio_check_dirty_writeback(struct folio *folio, · line 1483

Quoted 4 lines

mm/vmscan.c +4 -0

+	 */
+	mtc->nmask = NULL;
+	mtc->gfp_mask |= __GFP_THISNODE;
+	target_page = alloc_migration_target(page, (unsigned long)&mtc);

I finally managed to get a system setup to start testing some of this out. However it quickly crashed due to the bad pointer in the above call
- you need mtc not &mtc here.

I remember fixing that during earlier testing. I guess I missed to copy the change from test to my development. Thanks for testing this. I have now also tested the complete series with the above-suggested changes and did make sure we are indeed doing demotion by looking at /proc/vmstat:pgdemote_kswapd/pgdemote_direct

Quoted 11 lines

mm/vmscan.c static void folio_check_dirty_writeback(struct folio *folio, · line 1490

Quoted 8 lines

mm/vmscan.c +5 -0

+	if (target_page)
+		return target_page;
+
+	mtc->gfp_mask &= ~__GFP_THISNODE;
+	mtc->nmask = allowed_mask;
 
 	return alloc_migration_target(page, (unsigned long)&mtc);

And here.

I will fold this changes in and send a v8 after waiting for review feedback from others.

-aneesh

If you fix the issue, kindly add following tag where applicable

Reported-by kernel test robot <[email protected]>

All errors (new ones prefixed by >>):

Quoted 1 line

Quoted 1 line

Documentation/admin-guide/mm/memory-tiering.rst:5: (SEVERE/4) Title overline & underline mismatch.

vim +5 Documentation/admin-guide/mm/memory-tiering.rst

     4	

Quoted 1 line

5 ===========

     6	Memory tiers
     7	============
     8	

From ee8b97451b6ad1869f4d426e2d3825ac20a6e15d Mon Sep 17 00:00:00 2001

From: Jagdish Gediya <[email protected]>

All N_MEMORY nodes are divided into 3 memoty tiers with tier ID value MEMORY_TIER_HBM_GPU, MEMORY_TIER_DRAM and MEMORY_TIER_PMEM. By default, all nodes are assigned to default memory tier.

Demotion path for all N_MEMORY nodes is prepared based on the tier ID value of memory tiers.

This patch adds documention for memory tiering introduction, its sysfs interfaces and how demotion is performed based on memory tiers.

(quoted diff) +21 -0

+Default memory tier can be read from,
+/sys/devices/system/memtier/default_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/default_tier
+memtier200
+
+Max memory tier ID supported can be read from,
+/sys/devices/system/memtier/max_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/max_tier
+400
+
+Individual node's memory tier can be read of set using,
+/sys/devices/system/node/nodeN/memtier	(Read-Write)
+
+where N = node id
+
+When this interface is written, Node is moved from the old memory tier
+to new memory tier and demotion targets for all N_MEMORY nodes are

(quoted diff) +10 -0

+preferred and allowed demotion nodes
+------------------------------------
+
+Preferred nodes for a specific N_MEMORY node are the best nodes
+from the next possible lower memory tier. Allowed nodes for any
+node are all the nodes available in all possible lower memory
+tiers.
+
+Example:
+

Quoted 2 lines · Aneesh Kumar K.V

From: Jagdish Gediya <[email protected]>

Hi Aneesh and Jagdish,

The documentation can be improved, see below.

Quoted 10 lines

All N_MEMORY nodes are divided into 3 memoty tiers with tier ID value MEMORY_TIER_HBM_GPU, MEMORY_TIER_DRAM and MEMORY_TIER_PMEM. By default, all nodes are assigned to default memory tier.

Demotion path for all N_MEMORY nodes is prepared based on the tier ID value of memory tiers.

This patch adds documention for memory tiering introduction, its sysfs interfaces and how demotion is performed based on memory tiers.

I think the patch message should just be:
"Add documentation for memory tiering. It also covers its sysfs interfaces and how demotion is performed based on memory tiers."

Review

Quoted 7 lines

+===========

(quoted diff) +6 -0

+Memory tiers
+============
+
+This document describes explicit memory tiering support along with
+demotion based on memory tiers.
+

This causes htmldocs error, for which I have applied the fixup at [1].

Review

Quoted 8 lines

(quoted diff) +8 -0

+Memory nodes are divided into 3 types of memory tiers with tier ID
+value as shown based on their hardware characteristics.
+
+
+MEMORY_TIER_HBM_GPU
+MEMORY_TIER_DRAM
+MEMORY_TIER_PMEM
+

Use bullet list.

Review

Quoted 7 lines

(quoted diff) +7 -0

+Sysfs interfaces
+================
+
+Nodes belonging to specific tier can be read from,
+/sys/devices/system/memtier/memtierN/nodelist (Read-Only)
+
+Where N is 0 - 2.

The "where" sentence can be compounded into the previous sentence above.

Documentation/admin-guide/mm/memory-tiering.rst hotplug, the memory tier with default tier ID is assigned to the memory node. · line 93

Quoted 12 lines

Documentation/admin-guide/mm/memory-tiering.rst +12 -0

+
+Example 1:
+For a system where Node 0 is CPU + DRAM nodes, Node 1 is HBM node,
+node 2 is a PMEM node an ideal tier layout will be
+
+$ cat /sys/devices/system/memtier/memtier0/nodelist
+1
+$ cat /sys/devices/system/memtier/memtier1/nodelist
+0
+$ cat /sys/devices/system/memtier/memtier2/nodelist
+2
+

The code snippets should have been inside literal code blocks.

Documentation/admin-guide/mm/memory-tiering.rst hotplug, the memory tier with default tier ID is assigned to the memory node. · line 93

Quoted 12 lines

Documentation/admin-guide/mm/memory-tiering.rst +12 -0

+Example 2:
+For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
+nodes.
+
+$ cat /sys/devices/system/memtier/memtier0/nodelist
+cat: /sys/devices/system/memtier/memtier0/nodelist: No such file or
+directory
+$ cat /sys/devices/system/memtier/memtier1/nodelist
+0-1
+$ cat /sys/devices/system/memtier/memtier2/nodelist
+2-3
+

Use literal code block.

Quoted 21 lines

(quoted diff) +21 -0

+Default memory tier can be read from,
+/sys/devices/system/memtier/default_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/default_tier
+memtier200
+
+Max memory tier ID supported can be read from,
+/sys/devices/system/memtier/max_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/max_tier
+400
+
+Individual node's memory tier can be read of set using,
+/sys/devices/system/node/nodeN/memtier	(Read-Write)
+
+where N = node id
+
+When this interface is written, Node is moved from the old memory tier
+to new memory tier and demotion targets for all N_MEMORY nodes are

Review

Quoted 10 lines

(quoted diff) +10 -0

+built again.
+
+For example 1 mentioned above,
+$ cat /sys/devices/system/node/node0/memtier
+1
+$ cat /sys/devices/system/node/node1/memtier
+0
+$ cat /sys/devices/system/node/node2/memtier
+2
+

The same suggestions above apply here, too.

Review

Quoted 8 lines

(quoted diff) +8 -0

+Enable/Disable demotion
+-----------------------
+
+By default demotion is disabled, it can be enabled/disabled using
+below sysfs interface,
+
+$ echo 0/1 or false/true > /sys/kernel/mm/numa/demotion_enabled
+

Use literal code block.

Quoted 10 lines

(quoted diff) +10 -0

+preferred and allowed demotion nodes
+------------------------------------
+
+Preferred nodes for a specific N_MEMORY node are the best nodes
+from the next possible lower memory tier. Allowed nodes for any
+node are all the nodes available in all possible lower memory
+tiers.
+
+Example:
+

Review

Quoted 10 lines

(quoted diff) +10 -0

+For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
+nodes,
+
+node distances:
+node   0    1    2    3
+   0  10   20   30   40
+   1  20   10   40   30
+   2  30   40   10   40
+   3  40   30   40   10
+

Use reST table.

Documentation/admin-guide/mm/memory-tiering.rst from the next possible lower memory tier. Allowed nodes for any

Quoted 13 lines

Documentation/admin-guide/mm/memory-tiering.rst +13 -0

+memory_tiers[0] = <empty>
+memory_tiers[1] = 0-1
+memory_tiers[2] = 2-3
+
+node_demotion[0].preferred = 2
+node_demotion[0].allowed   = 2, 3
+node_demotion[1].preferred = 3
+node_demotion[1].allowed   = 3, 2
+node_demotion[2].preferred = <empty>
+node_demotion[2].allowed   = <empty>
+node_demotion[3].preferred = <empty>
+node_demotion[3].allowed   = <empty>
+

What are these above? Node properties? BTW, use literal code block.

If you don't understand these suggestions above, here is the diff:

---- >8 ----

Documentation/admin-guide/mm/memory-tiering.rst +91 -81

index 0a75e0dab1fd8e..10ec5aab6ddd53 100644
@@ -14,13 +14,13 @@ Introduction
 
 Many systems have multiple types of memory devices e.g. GPU, DRAM and
 PMEM. The memory subsystem of these systems can be called a memory
-tiering system because the performance of the different types of
+tiering system because the performance of each type of
 memory is different. Memory tiers are defined based on the hardware
 capabilities of memory nodes. Each memory tier is assigned a tier ID
 value that determines the memory tier position in demotion order.
 
 The memory tier assignment of each node is independent of each
-other. Moving a node from one tier to another tier doesn't affect
+other. Moving a node from one tier to another doesn't affect
 the tier assignment of any other node.
 
 Memory tiers are used to build the demotion targets for nodes. A node

@@ -32,10 +32,9 @@ Memory tier rank
 Memory nodes are divided into 3 types of memory tiers with tier ID
 value as shown based on their hardware characteristics.
 
-
-MEMORY_TIER_HBM_GPU
-MEMORY_TIER_DRAM
-MEMORY_TIER_PMEM
+  * MEMORY_TIER_HBM_GPU
+  * MEMORY_TIER_DRAM
+  * MEMORY_TIER_PMEM
 
 Memory tiers initialization and (re)assignments
 ===============================================

@@ -49,68 +48,73 @@ hotplug, the memory tier with default tier ID is assigned to the memory node.
 Sysfs interfaces
 ================
 
-Nodes belonging to specific tier can be read from,
-/sys/devices/system/memtier/memtierN/nodelist (Read-Only)
+Nodes belonging to specific tier can be read from
+/sys/devices/system/memtier/memtierN/nodelist, where N is 0 - 2 (read-only)
 
-Where N is 0 - 2.
+Examples:
 
-Example 1:
-For a system where Node 0 is CPU + DRAM nodes, Node 1 is HBM node,
-node 2 is a PMEM node an ideal tier layout will be
+1. On a system where Node 0 is CPU + DRAM nodes, Node 1 is HBM node,
+   node 2 is a PMEM node an ideal tier layout will be:
 
-$ cat /sys/devices/system/memtier/memtier0/nodelist
-1
-$ cat /sys/devices/system/memtier/memtier1/nodelist
-0
-$ cat /sys/devices/system/memtier/memtier2/nodelist
-2
+   .. code-block::
 
-Example 2:
-For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
-nodes.
+      $ cat /sys/devices/system/memtier/memtier0/nodelist
+      1
+      $ cat /sys/devices/system/memtier/memtier1/nodelist
+      0
+      $ cat /sys/devices/system/memtier/memtier2/nodelist
+      2
 
-$ cat /sys/devices/system/memtier/memtier0/nodelist
-cat: /sys/devices/system/memtier/memtier0/nodelist: No such file or
-directory
-$ cat /sys/devices/system/memtier/memtier1/nodelist
-0-1
-$ cat /sys/devices/system/memtier/memtier2/nodelist
-2-3
+2. On a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
+   nodes:
 
-Default memory tier can be read from,
-/sys/devices/system/memtier/default_tier (Read-Only)
+   .. code-block::
 
-e.g.
-$ cat /sys/devices/system/memtier/default_tier
-memtier200
+      $ cat /sys/devices/system/memtier/memtier0/nodelist
+      cat: /sys/devices/system/memtier/memtier0/nodelist: No such file or
+      directory
+      $ cat /sys/devices/system/memtier/memtier1/nodelist
+      0-1
+      $ cat /sys/devices/system/memtier/memtier2/nodelist
+      2-3
 
-Max memory tier ID supported can be read from,
-/sys/devices/system/memtier/max_tier (Read-Only)
+Default memory tier can be read from
+/sys/devices/system/memtier/default_tier (read-only), e.g.:
 
-e.g.
-$ cat /sys/devices/system/memtier/max_tier
-400
+.. code-block::
 
-Individual node's memory tier can be read of set using,
-/sys/devices/system/node/nodeN/memtier	(Read-Write)
+   $ cat /sys/devices/system/memtier/default_tier
+   memtier200
 
-where N = node id
+Max memory tier ID supported can be read from
+/sys/devices/system/memtier/max_tier (read-only), e.g.:
 
-When this interface is written, Node is moved from the old memory tier
+.. code-block::
+
+   $ cat /sys/devices/system/memtier/max_tier
+   400
+
+Individual node's memory tier can be read or set using
+/sys/devices/system/node/nodeN/memtier (read-write), where N = node id.
+
+When this interface is written, node is moved from the old memory tier
 to new memory tier and demotion targets for all N_MEMORY nodes are
 built again.
 
-For example 1 mentioned above,
-$ cat /sys/devices/system/node/node0/memtier
-1
-$ cat /sys/devices/system/node/node1/memtier
-0
-$ cat /sys/devices/system/node/node2/memtier
-2
+For example 1 mentioned above:
+
+.. code-block::
+
+   $ cat /sys/devices/system/node/node0/memtier
+   1
+   $ cat /sys/devices/system/node/node1/memtier
+   0
+   $ cat /sys/devices/system/node/node2/memtier
+   2
 
 Additional memory tiers can be created by writing a tier ID value to this file.
-This results in a new memory tier creation and moving the specific NUMA node to
-that memory tier.
+This results into creating a new tier and moving the specific NUMA node to
+that tier.
 
 Demotion
 ========

@@ -128,19 +132,20 @@ be used.
 
 Instead of a page being discarded during reclaim, it can be moved to
 persistent memory. Allowing page migration during reclaim enables
-these systems to migrate pages from fast(higher) tiers to slow(lower)
-tiers when the fast(higher) tier is under pressure.
+these systems to migrate pages from fast (higher) tiers to slow (lower)
+tiers when the fast (higher) tier is under pressure.
 
 
 Enable/Disable demotion
 -----------------------
 
-By default demotion is disabled, it can be enabled/disabled using
-below sysfs interface,
+By default demotion is disabled. It can be toggled by:
 
-$ echo 0/1 or false/true > /sys/kernel/mm/numa/demotion_enabled
+.. code-block::
 
-preferred and allowed demotion nodes
+   $ echo 0/1 or false/true > /sys/kernel/mm/numa/demotion_enabled
+
+Preferred and allowed demotion nodes
 ------------------------------------
 
 Preferred nodes for a specific N_MEMORY node are the best nodes

@@ -148,35 +153,40 @@ from the next possible lower memory tier. Allowed nodes for any
 node are all the nodes available in all possible lower memory
 tiers.
 
-Example:
+For example, on a system where Node 0 & 1 are CPU + DRAM nodes,
+node 2 & 3 are PMEM nodes:
 
-For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
-nodes,
+  * node distances
 
-node distances:
-node   0    1    2    3
-   0  10   20   30   40
-   1  20   10   40   30
-   2  30   40   10   40
-   3  40   30   40   10
+    ====  ==   ==   ==   ==
+    node   0    1    2    3
+    ====  ==   ==   ==   ==
+       0  10   20   30   40
+       1  20   10   40   30
+       2  30   40   10   40
+       3  40   30   40   10
+    ====  ==   ==   ==   ==
 
-memory_tiers[0] = <empty>
-memory_tiers[1] = 0-1
-memory_tiers[2] = 2-3
+  * node properties
 
-node_demotion[0].preferred = 2
-node_demotion[0].allowed   = 2, 3
-node_demotion[1].preferred = 3
-node_demotion[1].allowed   = 3, 2
-node_demotion[2].preferred = <empty>
-node_demotion[2].allowed   = <empty>
-node_demotion[3].preferred = <empty>
-node_demotion[3].allowed   = <empty>
+    .. code-block::
+
+       memory_tiers[0] = <empty>
+       memory_tiers[1] = 0-1
+       memory_tiers[2] = 2-3
+
+       node_demotion[0].preferred = 2
+       node_demotion[0].allowed   = 2, 3
+       node_demotion[1].preferred = 3
+       node_demotion[1].allowed   = 3, 2
+       node_demotion[2].preferred = <empty>
+       node_demotion[2].allowed   = <empty>
+       node_demotion[3].preferred = <empty>
+       node_demotion[3].allowed   = <empty>
 
 Memory allocation for demotion
 ------------------------------
 
-If a page needs to be demoted from any node, the kernel 1st tries
-to allocate a new page from the node's preferred node and fallbacks to
-node's allowed targets in allocation fallback order.
-
+If a page needs to be demoted from any node, the kernel first tries
+to allocate a new page from the node's preferred target node and fallbacks
+to node's allowed targets in allocation fallback order.

Thanks.

[1]: https://lore.kernel.org/linux-doc/[email protected]/

Signature · 1 line

An old man doll... just what I always wanted! - Clara

From: Jagdish Gediya <[email protected]>

All N_MEMORY nodes are divided into 3 memoty tiers with tier ID value

MEMORY_TIER_HBM_GPU, MEMORY_TIER_DRAM and MEMORY_TIER_PMEM. By default, all nodes are assigned to default memory tier.

Demotion path for all N_MEMORY nodes is prepared based on the tier ID value of memory tiers.

This patch adds documention for memory tiering introduction, its sysfs interfaces and how demotion is performed based on memory tiers.

Suggested-by Wei Xu <[email protected]>

Signed-off-by Jagdish Gediya <[email protected]>

Signed-off-by Aneesh Kumar K.V <[email protected]>

---
 Documentation/admin-guide/mm/index.rst        |   1 +
 .../admin-guide/mm/memory-tiering.rst         | 182 ++++++++++++++++++
 2 files changed, 183 insertions(+)
 create mode 100644 Documentation/admin-guide/mm/memory-tiering.rst

Documentation/admin-guide/mm/index.rst +1 -0

index c21b5823f126..3f211cbca8c3 100644
@@ -32,6 +32,7 @@ the Linux memory management.
    idle_page_tracking
    ksm
    memory-hotplug
+   memory-tiering
    nommu-mmap
    numa_memory_policy
    numaperf

Documentation/admin-guide/mm/memory-tiering.rst new file +182 -1

new file mode 100644
index 000000000000..142c36651f5d
@@ -0,0 +1,182 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _admin_guide_memory_tiering:
+
+===========
+Memory tiers
+============
+
+This document describes explicit memory tiering support along with
+demotion based on memory tiers.
+
+Introduction
+============
+
+Many systems have multiple types of memory devices e.g. GPU, DRAM and
+PMEM. The memory subsystem of these systems can be called a memory
+tiering system because the performance of the different types of
+memory is different. Memory tiers are defined based on the hardware
+capabilities of memory nodes. Each memory tier is assigned a tier ID
+value that determines the memory tier position in demotion order.
+
+The memory tier assignment of each node is independent of each
+other. Moving a node from one tier to another tier doesn't affect
+the tier assignment of any other node.
+
+Memory tiers are used to build the demotion targets for nodes. A node
+can demote its pages to any node of any lower tiers.
+
+Memory tier rank
+=================
+
+Memory nodes are divided into 3 types of memory tiers with tier ID
+value as shown based on their hardware characteristics.
+
+
+MEMORY_TIER_HBM_GPU
+MEMORY_TIER_DRAM
+MEMORY_TIER_PMEM
+
+Memory tiers initialization and (re)assignments
+===============================================
+
+By default, all nodes are assigned to the memory tier with the default tier ID
+DEFAULT_MEMORY_TIER which is 200 (MEMORY_TIER_DRAM). The memory tier of
+the memory node can be either modified through sysfs or from the driver. On
+hotplug, the memory tier with default tier ID is assigned to the memory node.
+
+
+Sysfs interfaces
+================
+
+Nodes belonging to specific tier can be read from,
+/sys/devices/system/memtier/memtierN/nodelist (Read-Only)
+
+Where N is 0 - 2.
+
+Example 1:
+For a system where Node 0 is CPU + DRAM nodes, Node 1 is HBM node,
+node 2 is a PMEM node an ideal tier layout will be
+
+$ cat /sys/devices/system/memtier/memtier0/nodelist
+1
+$ cat /sys/devices/system/memtier/memtier1/nodelist
+0
+$ cat /sys/devices/system/memtier/memtier2/nodelist
+2
+
+Example 2:
+For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
+nodes.
+
+$ cat /sys/devices/system/memtier/memtier0/nodelist
+cat: /sys/devices/system/memtier/memtier0/nodelist: No such file or
+directory
+$ cat /sys/devices/system/memtier/memtier1/nodelist
+0-1
+$ cat /sys/devices/system/memtier/memtier2/nodelist
+2-3
+
+Default memory tier can be read from,
+/sys/devices/system/memtier/default_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/default_tier
+memtier200
+
+Max memory tier ID supported can be read from,
+/sys/devices/system/memtier/max_tier (Read-Only)
+
+e.g.
+$ cat /sys/devices/system/memtier/max_tier
+400
+
+Individual node's memory tier can be read of set using,
+/sys/devices/system/node/nodeN/memtier (Read-Write)
+
+where N = node id
+
+When this interface is written, Node is moved from the old memory tier
+to new memory tier and demotion targets for all N_MEMORY nodes are
+built again.
+
+For example 1 mentioned above,
+$ cat /sys/devices/system/node/node0/memtier
+1
+$ cat /sys/devices/system/node/node1/memtier
+0
+$ cat /sys/devices/system/node/node2/memtier
+2
+
+Additional memory tiers can be created by writing a tier ID value to this file.
+This results in a new memory tier creation and moving the specific NUMA node to
+that memory tier.
+
+Demotion
+========
+
+In a system with DRAM and persistent memory, once DRAM
+fills up, reclaim will start and some of the DRAM contents will be
+thrown out even if there is a space in persistent memory.
+Consequently, allocations will, at some point, start falling over to the slower
+persistent memory.
+
+That has two nasty properties. First, the newer allocations can end up in
+the slower persistent memory. Second, reclaimed data in DRAM are just
+discarded even if there are gobs of space in persistent memory that could
+be used.
+
+Instead of a page being discarded during reclaim, it can be moved to
+persistent memory. Allowing page migration during reclaim enables
+these systems to migrate pages from fast(higher) tiers to slow(lower)
+tiers when the fast(higher) tier is under pressure.
+
+
+Enable/Disable demotion
+-----------------------
+
+By default demotion is disabled, it can be enabled/disabled using
+below sysfs interface,
+
+$ echo 0/1 or false/true > /sys/kernel/mm/numa/demotion_enabled
+
+preferred and allowed demotion nodes
+------------------------------------
+
+Preferred nodes for a specific N_MEMORY node are the best nodes
+from the next possible lower memory tier. Allowed nodes for any
+node are all the nodes available in all possible lower memory
+tiers.
+
+Example:
+
+For a system where Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM
+nodes,
+
+node distances:
+node   0    1    2    3
+   0  10   20   30   40
+   1  20   10   40   30
+   2  30   40   10   40
+   3  40   30   40   10
+
+memory_tiers[0] = <empty>
+memory_tiers[1] = 0-1
+memory_tiers[2] = 2-3
+
+node_demotion[0].preferred = 2
+node_demotion[0].allowed   = 2, 3
+node_demotion[1].preferred = 3
+node_demotion[1].allowed   = 3, 2
+node_demotion[2].preferred = <empty>
+node_demotion[2].allowed   = <empty>
+node_demotion[3].preferred = <empty>
+node_demotion[3].allowed   = <empty>
+
+Memory allocation for demotion
+------------------------------
+
+If a page needs to be demoted from any node, the kernel 1st tries
+to allocate a new page from the node's preferred node and fallbacks to
+node's allowed targets in allocation fallback order.
+
--

2.36.1

The current kernel has the basic memory tiering support: Inactive pages on a higher tier NUMA node can be migrated (demoted) to a lower tier NUMA node to make room for new allocations on the higher tier NUMA node. Frequently accessed pages on a lower tier NUMA node can be migrated (promoted) to a higher tier NUMA node to improve the performance.

In the current kernel, memory tiers are defined implicitly via a demotion path relationship between NUMA nodes, which is created during the kernel initialization and updated when a NUMA node is hot-added or hot-removed. The current implementation puts all nodes with CPU into the top tier, and builds the tier hierarchy tier-by-tier by establishing the per-node demotion targets based on the distances between nodes.

This current memory tier kernel interface needs to be improved for several important use cases:

* The current tier initialization code always initializes each memory-only NUMA node into a lower tier. But a memory-only NUMA node may have a high performance memory device (e.g. a DRAM device attached via CXL.mem or a DRAM-backed memory-only node on a virtual machine) and should be put into a higher tier.

* The current tier hierarchy always puts CPU nodes into the top tier. But on a system with HBM (e.g. GPU memory) devices, these memory-only HBM NUMA nodes should be in the top tier, and DRAM nodes with CPUs are better to be placed into the next lower tier.

* Also because the current tier hierarchy always puts CPU nodes into the top tier, when a CPU is hot-added (or hot-removed) and triggers a memory node from CPU-less into a CPU node (or vice versa), the memory tier hierarchy gets changed, even though no memory node is added or removed. This can make the tier hierarchy unstable and make it difficult to support tier-based memory accounting.

* A higher tier node can only be demoted to selected nodes on the next lower tier as defined by the demotion path, not any other node from any lower tier. This strict, hard-coded demotion order does not work in all use cases (e.g. some use cases may want to allow cross-socket demotion to another node in the same demotion tier as a fallback when the preferred demotion node is out of space), and has resulted in the feature request for an interface to override the system-wide, per-node demotion order from the userspace. This demotion order is also inconsistent with the page allocation fallback order when all the nodes in a higher tier are out of space: The page allocation can fall back to any node from any lower tier, whereas the demotion order doesn't allow that.

* There are no interfaces for the userspace to learn about the memory tier hierarchy in order to optimize its memory allocations.

This patch series make the creation of memory tiers explicit under the control of userspace or device driver.

Memory Tier Initialization
==========================

By default, all memory nodes are assigned to the default tier with tier ID value 200.

A device driver can move up or down its memory nodes from the default tier. For example, PMEM can move down its memory nodes below the default tier, whereas GPU can move up its memory nodes above the default tier.

The kernel initialization code makes the decision on which exact tier a memory node should be assigned to based on the requests from the device drivers as well as the memory device hardware information provided by the firmware.

Hot-adding/removing CPUs doesn't affect memory tier hierarchy.

Memory Allocation for Demotion
==============================
This patch series keep the demotion target page allocation logic same. The demotion page allocation pick the closest NUMA node in the next lower tier to the current NUMA node allocating pages from.

This will be later improved to use the same page allocation strategy using fallback list.

Sysfs Interface:
-------------
Listing current list of memory tiers details:

:/sys/devices/system/memtier$ ls
default_tier max_tier  memtier1  power  uevent
:/sys/devices/system/memtier$ cat default_tier
memtier200
:/sys/devices/system/memtier$ cat max_tier 
400
:/sys/devices/system/memtier$ 

Per node memory tier details:

For a cpu only NUMA node:

:/sys/devices/system/node# cat node0/memtier 
:/sys/devices/system/node# echo 1 > node0/memtier 
:/sys/devices/system/node# cat node0/memtier 
:/sys/devices/system/node# 

For a NUMA node with memory:

:/sys/devices/system/node# cat node1/memtier 
1
:/sys/devices/system/node# ls ../memtier/
default_tier  max_tier  memtier1  power  uevent
:/sys/devices/system/node# echo 2 > node1/memtier 
:/sys/devices/system/node# 
:/sys/devices/system/node# ls ../memtier/
default_tier  max_tier  memtier1  memtier2  power  uevent
:/sys/devices/system/node# cat node1/memtier 
2
:/sys/devices/system/node# 

Removing a memory tier

:/sys/devices/system/node# cat node1/memtier 
2
:/sys/devices/system/node# echo 1 > node1/memtier 
:/sys/devices/system/node# 
:/sys/devices/system/node# cat node1/memtier 
1
:/sys/devices/system/node# 
:/sys/devices/system/node# ls ../memtier/
default_tier  max_tier  memtier1  power  uevent
:/sys/devices/system/node# 

The above resulted in removal of memtier2 which was created in the earlier step.

Changes from v6:
* Drop the usage of rank.
* Address other review feedback.

Changes from v5:
* Remove patch supporting N_MEMORY node removal from memory tiers. memory tiers are going to be used for features other than demotion. Hence keep all N_MEMORY nodes in memory tiers irrespective of whether they want to participate in promotion or demotion.
* Add NODE_DATA->memtier
* Rearrage patches to add sysfs files later.
* Add support to create memory tiers from userspace.
* Address other review feedback.

Changes from v4:
* Address review feedback.
* Reverse the meaning of "rank": higher rank value means higher tier.
* Add "/sys/devices/system/memtier/default_tier".
* Add node_is_toptier

v4:
Add support for explicit memory tiers and ranks.

v3:
- Modify patch 1 subject to make it more specific
- Remove /sys/kernel/mm/numa/demotion_targets interface, use /sys/devices/system/node/demotion_targets instead and make it writable to override node_states[N_DEMOTION_TARGETS].
- Add support to view per node demotion targets via sysfs

v2:
In v1, only 1st patch of this patch series was sent, which was implemented to avoid some of the limitations on the demotion target sharing, however for certain numa topology, the demotion targets found by that patch was not most optimal, so 1st patch in this series is modified according to suggestions from Huang and Baolin. Different examples of demotion list comparasion between existing implementation and changed implementation can be found in the commit message of 1st patch.

Aneesh Kumar K.V (10):
mm/demotion: Add support for explicit memory tiers mm/demotion: Move memory demotion related code mm/demotion/dax/kmem: Set node's memory tier to MEMORY_TIER_PMEM mm/demotion: Add hotplug callbacks to handle new numa node onlined mm/demotion: Build demotion targets based on explicit memory tiers mm/demotion: Expose memory tier details via sysfs mm/demotion: Add per node memory tier attribute to sysfs mm/demotion: Add pg_data_t member to track node memory tier details mm/demotion: Update node_is_toptier to work with memory tiers mm/demotion: Add sysfs ABI documentation

Jagdish Gediya (2):
mm/demotion: Demote pages according to allocation fallback order mm/demotion: Add documentation for memory tiering

 .../ABI/testing/sysfs-kernel-mm-memory-tiers  |  61 ++
 Documentation/admin-guide/mm/index.rst        |   1 +
 .../admin-guide/mm/memory-tiering.rst         | 182 ++++
 drivers/base/node.c                           |  42 +
 drivers/dax/kmem.c                            |   6 +-
 include/linux/memory-tiers.h                  |  72 ++
 include/linux/migrate.h                       |  15 -
 include/linux/mmzone.h                        |   3 +
 include/linux/node.h                          |   5 -
 mm/Makefile                                   |   1 +
 mm/huge_memory.c                              |   1 +
 mm/memory-tiers.c                             | 791 ++++++++++++++++++
 mm/migrate.c                                  | 453 +---------
 mm/mprotect.c                                 |   1 +
 mm/vmscan.c                                   |  57 +-
 mm/vmstat.c                                   |   4 -
 16 files changed, 1204 insertions(+), 491 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers
 create mode 100644 Documentation/admin-guide/mm/memory-tiering.rst
 create mode 100644 include/linux/memory-tiers.h
 create mode 100644 mm/memory-tiers.c

Signature · 1 line

2.36.1