Page demotion for memory reclaim

If a memory node has a preferred migration path to demote cold pages, attempt to move those inactive pages to that migration node before reclaiming. This will better utilize available memory, provide a faster tier than swapping or discarding, and allow such pages to be reused immediately without IO to retrieve the data.

Some places we would like to see this used:

1. Persistent memory being as a slower, cheaper DRAM replacement
2. Remote memory-only "expansion" NUMA nodes
3. Resolving memory imbalances where one NUMA node is seeing more allocation activity than another. This helps keep more recent allocations closer to the CPUs on the node doing the allocating.

Signed-off-by Keith Busch <[email protected]>

---
 include/linux/migrate.h        |  6 ++++++
 include/trace/events/migrate.h |  3 ++-
 mm/debug.c                     |  1 +
 mm/migrate.c                   | 45 ++++++++++++++++++++++++++++++++++++++++++
 mm/vmscan.c                    | 15 ++++++++++++++
 5 files changed, 69 insertions(+), 1 deletion(-)

include/linux/migrate.h +6 -0

index e13d9bf2f9a5..a004cb1b2dbb 100644
@@ -25,6 +25,7 @@ enum migrate_reason {
        MR_MEMPOLICY_MBIND,
        MR_NUMA_MISPLACED,
        MR_CONTIG_RANGE,
+       MR_DEMOTION,
        MR_TYPES
 };
 

@@ -79,6 +80,7 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,
 extern int migrate_page_move_mapping(struct address_space *mapping,
                struct page *newpage, struct page *page, enum migrate_mode mode,
                int extra_count);
+extern bool migrate_demote_mapping(struct page *page);
 #else
 
 static inline void putback_movable_pages(struct list_head *l) {}

@@ -105,6 +107,10 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
        return -ENOSYS;
 }
 
+static inline bool migrate_demote_mapping(struct page *page)
+{
+       return false;
+}
 #endif /* CONFIG_MIGRATION */
 
 #ifdef CONFIG_COMPACTION

include/trace/events/migrate.h +2 -1

index 705b33d1e395..d25de0cc8714 100644
@@ -20,7 +20,8 @@
        EM( MR_SYSCALL,         "syscall_or_cpuset")            \
        EM( MR_MEMPOLICY_MBIND, "mempolicy_mbind")              \
        EM( MR_NUMA_MISPLACED,  "numa_misplaced")               \
-       EMe(MR_CONTIG_RANGE,    "contig_range")
+       EM(MR_CONTIG_RANGE,     "contig_range")                 \
+       EMe(MR_DEMOTION,        "demotion")
 
 /*
  * First define the enums in the above macros to be exported to userspace

mm/debug.c +1 -0

index c0b31b6c3877..53d499f65199 100644
@@ -25,6 +25,7 @@ const char *migrate_reason_names[MR_TYPES] = {
        "mempolicy_mbind",
        "numa_misplaced",
        "cma",
+       "demotion",
 };
 
 const struct trace_print_flags pageflag_names[] = {

mm/migrate.c +45 -0

index 705b320d4b35..83fad87361bf 100644
@@ -1152,6 +1152,51 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
        return rc;
 }
 
+/**
+ * migrate_demote_mapping() - Migrate this page and its mappings to its
+ *                           demotion node.
+ * @page: An isolated, non-compound page that should move to
+ *       its current node's migration path.
+ *
+ * @returns: True if migrate demotion was successful, false otherwise
+ */
+bool migrate_demote_mapping(struct page *page)
+{
+       int rc, next_nid = next_migration_node(page_to_nid(page));
+       struct page *newpage;
+
+       /*
+        * The flags are set to allocate only on the desired node in the
+        * migration path, and to fail fast if not immediately available. We
+        * are already in the memory reclaim path, we don't want heroic
+        * efforts to get a page.
+        */
+       gfp_t mask = GFP_NOWAIT | __GFP_NOWARN | __GFP_NORETRY |
+                    __GFP_NOMEMALLOC | __GFP_THISNODE;
+
+       VM_BUG_ON_PAGE(PageCompound(page), page);
+       VM_BUG_ON_PAGE(PageLRU(page), page);
+
+       if (next_nid < 0)
+               return false;
+
+       newpage = alloc_pages_node(next_nid, mask, 0);
+       if (!newpage)
+               return false;
+
+       /*
+        * MIGRATE_ASYNC is the most light weight and never blocks.
+        */
+       rc = __unmap_and_move_locked(page, newpage, MIGRATE_ASYNC);
+       if (rc != MIGRATEPAGE_SUCCESS) {
+               __free_pages(newpage, 0);
+               return false;
+       }
+
+       set_page_owner_migrate_reason(newpage, MR_DEMOTION);
+       return true;
+}
+
 /*
  * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move().  Work
  * around it.

mm/vmscan.c +4 -0

index a5ad0b35ab8e..0a95804e946a 100644
@@ -1261,6 +1261,21 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                        ; /* try to reclaim the page below */
                }
 
+               if (!PageCompound(page)) {
+                       if (migrate_demote_mapping(page)) {
+                                unlock_page(page);
+                                if (likely(put_page_testzero(page)))

mm/vmscan.c +1 -1

+
                /*
                 * Anonymous process memory has backing store?
                 * Try to allocate it some swap space here.
--

2.14.4

Quoted 4 lines · Keith Busch

mm/vmscan.c +4 -0

+               if (!PageCompound(page)) {
+                       if (migrate_demote_mapping(page)) {
+                                unlock_page(page);
+                                if (likely(put_page_testzero(page)))

mm/vmscan.c static unsigned long shrink_page_list(struct list_head *page_list, · line 1277

Quoted 10 lines

mm/vmscan.c +10 -0

+                                        goto free_it;
+
+                                /*
+                                * Speculative reference will free this page,
+                                * so leave it off the LRU.
+                                */
+                                nr_reclaimed++;
+                                continue;
+                        }
+               }

It looks the reclaim path would fall through if the migration is failed. But, it looks, with patch #4, you may end up trying reclaim an anon page on swapless system if migration is failed?

And, actually I have the same question with Yan Zi. Why not just put the demote candidate into a separate list, then migrate all the candidates in bulk with migrate_pages()?

The kernel has recently added support for using persistent memory as normal RAM:

https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=c221c0b0308fd01d9fb33a16f64d2fd95f8830a4

The persistent memory is hot added to nodes separate from other memory types, which makes it convenient to make node based memory policies.

When persistent memory provides a larger and cheaper address space, but with slower access characteristics than system RAM, we'd like the kernel to make use of these memory-only nodes as a migration tier for pages that would normally be discared during memory reclaim. This is faster than doing IO for swap or page cache, and makes better utilization of available physical address space.

The feature is not enabled by default. The user must opt-in to kernel managed page migration by defining the demotion path. In the future, we may want to have the kernel automatically create this based on heterogeneous memory attributes and CPU locality.

1. The name of “page demotion” seems confusing to me, since I thought it was about large pages demote to small pages as opposite to promoting small pages to THPs. Am I the only one here?

2. For the demotion path, a common case would be from high-performance memory, like HBM or Multi-Channel DRAM, to DRAM, then to PMEM, and finally to disks, right? More general case for demotion path would be derived from the memory performance description from HMAT[1], right? Do you have any algorithm to form such a path from HMAT?

3. Do you have a plan for promoting pages from lower-level memory to higher-level memory, like from PMEM to DRAM? Will this one-way demotion make all pages sink to PMEM and disk?

4. In your patch 3, you created a new method migrate_demote_mapping() to migrate pages to other memory node, is there any problem of reusing existing migrate_pages() interface?

5. In addition, you only migrate base pages, is there any performance concern on migrating THPs? Is it too costly to migrate THPs?

Quoted 3 lines · Zi Yan

1. The name of “page demotion” seems confusing to me, since I thought it was about large pages demote to small pages as opposite to promoting small pages to THPs. Am I the only one here?

If you have a THP, we'll skip the page migration and fall through to split_huge_page_to_list(), then the smaller pages can be considered, migrated and reclaimed individually. Not that we couldn't try to migrate a THP directly. It was just simpler implementation for this first attempt.

Quoted 4 lines

2. For the demotion path, a common case would be from high-performance memory, like HBM or Multi-Channel DRAM, to DRAM, then to PMEM, and finally to disks, right? More general case for demotion path would be derived from the memory performance description from HMAT[1], right? Do you have any algorithm to form such a path from HMAT?

Yes, I have a PoC for the kernel setting up a demotion path based on HMAT properties here:

https://git.kernel.org/pub/scm/linux/kernel/git/kbusch/linux.git/commit/?h=mm-migrate&id=4d007659e1dd1b0dad49514348be4441fbe7cadb

The above is just from an experimental branch.

Quoted 2 lines

3. Do you have a plan for promoting pages from lower-level memory to higher-level memory, like from PMEM to DRAM? Will this one-way demotion make all pages sink to PMEM and disk?

Promoting previously demoted pages would require the application do something to make that happen if you turn demotion on with this series. Kernel auto-promotion is still being investigated, and it's a little trickier than reclaim.

If it sinks to disk, though, the next access behavior is the same as before, without this series.

Quoted 2 lines

4. In your patch 3, you created a new method migrate_demote_mapping() to migrate pages to other memory node, is there any problem of reusing existing migrate_pages() interface?

Yes, we may not want to migrate everything in the shrink_page_list() pages. We might want to keep a page, so we have to do those checks first. At the point we know we want to attempt migration, the page is already locked and not in a list, so it is just easier to directly invoke the new __unmap_and_move_locked() that migrate_pages() eventually also calls.

Quoted 2 lines

5. In addition, you only migrate base pages, is there any performance concern on migrating THPs? Is it too costly to migrate THPs?

It was just easier to consider single pages first, so we let a THP split if possible. I'm not sure of the cost in migrating THPs directly.

Quoted 23 lines · Keith Busch

Quoted 3 lines · Zi Yan

1. The name of “page demotion” seems confusing to me, since I thought it was about large pages demote to small pages as opposite to promoting small pages to THPs. Am I the only one here?

If you have a THP, we'll skip the page migration and fall through to split_huge_page_to_list(), then the smaller pages can be considered, migrated and reclaimed individually. Not that we couldn't try to migrate a THP directly. It was just simpler implementation for this first attempt.

Quoted 4 lines

2. For the demotion path, a common case would be from high-performance memory, like HBM or Multi-Channel DRAM, to DRAM, then to PMEM, and finally to disks, right? More general case for demotion path would be derived from the memory performance description from HMAT[1], right? Do you have any algorithm to form such a path from HMAT?

Yes, I have a PoC for the kernel setting up a demotion path based on HMAT properties here:

https://git.kernel.org/pub/scm/linux/kernel/git/kbusch/linux.git/commit/?h=mm-migrate&id=4d007659e1dd1b0dad49514348be4441fbe7cadb

The above is just from an experimental branch.

Quoted 7 lines

Quoted 2 lines

3. Do you have a plan for promoting pages from lower-level memory to higher-level memory, like from PMEM to DRAM? Will this one-way demotion make all pages sink to PMEM and disk?

Promoting previously demoted pages would require the application do something to make that happen if you turn demotion on with this series. Kernel auto-promotion is still being investigated, and it's a little trickier than reclaim.

Just FYI. I'm currently working on a patchset which tries to promotes page from second tier memory (i.e. PMEM) to DRAM via NUMA balancing. But, NUMA balancing can't deal with unmapped page cache, they have to be promoted from different path, i.e. mark_page_accessed().

And, I do agree with Keith, promotion is definitely trickier than reclaim since kernel can't recognize "hot" pages accurately. NUMA balancing is still corse-grained and inaccurate, but it is simple. If we would like to implement more sophisticated algorithm, in-kernel implementation might be not a good idea.

Quoted 4 lines

2. For the demotion path, a common case would be from high-performance memory, like HBM or Multi-Channel DRAM, to DRAM, then to PMEM, and finally to disks, right? More general case for demotion path would be derived from the memory performance description from HMAT[1], right? Do you have any algorithm to form such a path from HMAT?

Yes, I have a PoC for the kernel setting up a demotion path based on HMAT properties here:

https://git.kernel.org/pub/scm/linux/kernel/git/kbusch/linux.git/commit/?h=mm-migrate&id=4d007659e1dd1b0dad49514348be4441fbe7cadb

The above is just from an experimental branch.

Quoted 3 lines

3. Do you have a plan for promoting pages from lower-level memory to higher-level memory, like from PMEM to DRAM? Will this one-way demotion make all pages sink to PMEM and disk?

Promoting previously demoted pages would require the application do something to make that happen if you turn demotion on with this series. Kernel auto-promotion is still being investigated, and it's a little trickier than reclaim.

If it sinks to disk, though, the next access behavior is the same as before, without this series.

Quoted 2 lines

4. In your patch 3, you created a new method migrate_demote_mapping() to migrate pages to other memory node, is there any problem of reusing existing migrate_pages() interface?

Yes, we may not want to migrate everything in the shrink_page_list() pages. We might want to keep a page, so we have to do those checks first. At the point we know we want to attempt migration, the page is already locked and not in a list, so it is just easier to directly invoke the new __unmap_and_move_locked() that migrate_pages() eventually also calls.

Quoted 3 lines

5. In addition, you only migrate base pages, is there any performance concern on migrating THPs? Is it too costly to migrate THPs?

It was just easier to consider single pages first, so we let a THP split if possible. I'm not sure of the cost in migrating THPs directly.

Quoted 5 lines

Yes, we may not want to migrate everything in the shrink_page_list() pages. We might want to keep a page, so we have to do those checks first. At the point we know we want to attempt migration, the page is already locked and not in a list, so it is just easier to directly invoke the new __unmap_and_move_locked() that migrate_pages() eventually also calls.

Right, I understand that you want to only migrate small pages to begin with. My question is why not using the existing migrate_pages() in your patch 3. Like:

mm/vmscan.c +14 -0

index a5ad0b35ab8e..0a0753af357f 100644
@@ -1261,6 +1261,20 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                        ; /* try to reclaim the page below */
                }
 
+               if (!PageCompound(page)) {
+                       int next_nid = next_migration_node(page);
+                       int err;
+
+                       if (next_nid != TERMINAL_NODE) {
+                               LIST_HEAD(migrate_list);
+                               list_add(&migrate_list, &page->lru);
+                               err = migrate_pages(&migrate_list, alloc_new_node_page, NULL,
+                                       next_nid, MIGRATE_ASYNC, MR_DEMOTION);
+                               if (err)
+                                       putback_movable_pages(&migrate_list);
+                       }
+               }
+
                /*
                 * Anonymous process memory has backing store?
                 * Try to allocate it some swap space here.

Because your new migrate_demote_mapping() basically does the same thing as the code above. If you are not OK with the gfp flags in alloc_new_node_page(), you can just write your own alloc_new_node_page(). :)