mm: multi-gen LRU: Have secondary MMUs participate in MM_WALK
Subsystems (6)
cover mm: multi-gen LRU: Have secondary MMUs participate in MM_WALK
Today, the MM_WALK capability causes MGLRU to clear the young bit from PMDs and PTEs during the page table walk before eviction, but MGLRU does not call the clear_young() MMU notifier in this case. By not calling this notifier, the MM walk takes less time/CPU, but it causes pages that are accessed mostly through KVM / secondary MMUs to appear younger than they should be.
We do call the clear_young() notifier today, but only when attempting to evict the page, so we end up clearing young/accessed information less frequently for secondary MMUs than for mm PTEs, and therefore they appear younger and are less likely to be evicted. Therefore, memory that is *not* being accessed mostly by KVM will be evicted *more* frequently, worsening performance.
ChromeOS observed a tab-open latency regression when enabling MGLRU with a setup that involved running a VM:
Tab-open latency histogram (ms)
Version p50 mean p95 p99 max
base 1315 1198 2347 3454 10319
mglru 2559 1311 7399 12060 43758
fix 1119 926 2470 4211 6947
This series replaces the final non-selftest patchs from this series[1], which introduced a similar change (and a new MMU notifier) with KVM optimizations. I'll send a separate series (to Sean and Paolo) for the KVM optimizations.
This series also makes proactive reclaim with MGLRU possible for KVM memory. I have verified that this functions correctly with the selftest from [1], but given that that test is a KVM selftest, I'll send it with the rest of the KVM optimizations later. Andrew, let me know if you'd like to take the test now anyway.
[1]: https://lore.kernel.org/linux-mm/[email protected]/
Yu Zhao (2):
mm: multi-gen LRU: remove MM_LEAF_OLD and MM_NONLEAF_TOTAL stats
mm: multi-gen LRU: use {ptep,pmdp}_clear_young_notify()
include/linux/mmzone.h | 7 ++- mm/rmap.c | 9 ++-- mm/vmscan.c | 105 +++++++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 61 deletions(-)
base-commit: b5d43fad926a3f542cd06f3c9d286f6f489f7129
Signature · 1 line
2.47.0.105.g07ac214952-goog
PATCH 1/2 mm: multi-gen LRU: remove MM_LEAF_OLD and MM_NONLEAF_TOTAL stats
From: Yu Zhao <[email protected]>
The removed stats, MM_LEAF_OLD and MM_NONLEAF_TOTAL, are not very helpful and become more complicated to properly compute when adding test/clear_young() notifiers in MGLRU's mm walk.
--- include/linux/mmzone.h | 2 -- mm/vmscan.c | 14 +++++--------- 2 files changed, 5 insertions(+), 11 deletions(-)
include/linux/mmzone.h +0 -2
@@ -460,9 +460,7 @@ struct lru_gen_folio {enum {MM_LEAF_TOTAL ,/* total leaf entries */ -MM_LEAF_OLD ,/* old leaf entries */ MM_LEAF_YOUNG ,/* young leaf entries */ -MM_NONLEAF_TOTAL ,/* total non-leaf entries */ MM_NONLEAF_FOUND ,/* non-leaf entries found in Bloom filters */ MM_NONLEAF_ADDED ,/* non-leaf entries added to Bloom filters */ NR_MM_STATS
mm/vmscan.c +5 -9
@@ -3405,7 +3405,6 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,continue ;if (!pte_young (ptent)) { - walk->mm_stats[MM_LEAF_OLD ]++;continue ; }
@@ -3558,7 +3557,6 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk->mm_stats[MM_LEAF_TOTAL ]++;if (!pmd_young (val)) { - walk->mm_stats[MM_LEAF_OLD ]++;continue ; }
@@ -3570,8 +3568,6 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,continue ; } - walk->mm_stats[MM_NONLEAF_TOTAL ]++; -if (!walk->force_scan &&should_clear_pmd_young ()) {if (!pmd_young (val))continue ;
@@ -5262,11 +5258,11 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,for (tier =0 ; tier <MAX_NR_TIERS ; tier++) {seq_printf (m," %10d" , tier);for (type =0 ; type <ANON_AND_FILE ; type++) { -const char *s =" " ; +const char *s ="xxx" ;unsigned long n[3 ] = {};if (seq == max_seq) { - s ="RT " ; + s ="RTx" ; n[0 ] =READ_ONCE (lrugen->avg_refaulted[type][tier]); n[1 ] =READ_ONCE (lrugen->avg_total[type][tier]); }else if (seq == min_seq[type] ||NR_HIST_GENS >1 ) {
@@ -5288,14 +5284,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,seq_puts (m," " );for (i =0 ; i <NR_MM_STATS ; i++) { -const char *s =" " ; +const char *s ="xxxx" ;unsigned long n =0 ;if (seq == max_seq &&NR_HIST_GENS ==1 ) { - s ="LOYNFA" ; + s ="TYFA" ; n =READ_ONCE (mm_state->stats[hist][i]); }else if (seq != max_seq &&NR_HIST_GENS >1 ) { - s ="loynfa" ; + s ="tyfa" ; n =READ_ONCE (mm_state->stats[hist][i]); }
Signature · 1 line
2.47.0.105.g07ac214952-goog
Andrew Morton
maintainerRe: [PATCH 1/2] mm: multi-gen LRU: remove MM_LEAF_OLD and MM_NONLEAF_TOTAL stats
Quoted 6 lines · On Sat, 19 Oct 2024 01:29:38 +0000 James Houghton <[email protected]> wrote:
From: Yu Zhao <[email protected]>
The removed stats,
MM_LEAF_OLDandMM_NONLEAF_TOTAL, are not very helpful and become more complicated to properly compute when adding test/clear_young() notifiers in MGLRU's mm walk.
Patch 2/2 has Fixes: and cc:stable. Patch 2/2 requires patch 1/2. So I added the same Fixes: and cc:stable to patch 1/2.
Re: [PATCH 1/2] mm: multi-gen LRU: remove MM_LEAF_OLD and MM_NONLEAF_TOTAL stats
Quoted 5 lines · Andrew Morton
Quoted 2 lines · On Sat, 19 Oct 2024 01:29:38 +0000 James Houghton <[email protected]> wrote:
From: Yu Zhao <[email protected]>
Quoted 7 lines
Quoted 4 lines
The removed stats,
MM_LEAF_OLDandMM_NONLEAF_TOTAL, are not very helpful and become more complicated to properly compute when adding test/clear_young() notifiers in MGLRU's mm walk.Patch 2/2 has Fixes: and cc:stable. Patch 2/2 requires patch 1/2. So I added the same Fixes: and cc:stable to patch 1/2.
SGTM. Thanks!
PATCH 2/2 mm: multi-gen LRU: use {ptep,pmdp}_clear_young_notify()
In reply to James HoughtonFrom: Yu Zhao <[email protected]>
When the MM_WALK capability is enabled, memory that is mostly accessed by a VM appears younger than it really is, therefore this memory will be less likely to be evicted. Therefore, the presence of a running VM can significantly increase swap-outs for non-VM memory, regressing the performance for the rest of the system.
Fix this regression by always calling {ptep,pmdp}_clear_young_notify() whenever we clear the young bits on PMDs/PTEs.
--- include/linux/mmzone.h | 5 ++- mm/rmap.c | 9 ++--- mm/vmscan.c | 91 +++++++++++++++++++++++------------------- 3 files changed, 55 insertions(+), 50 deletions(-)
include/linux/mmzone.h +3 -2
@@ -557,7 +557,7 @@ struct lru_gen_memcg {void lru_gen_init_pgdat (struct pglist_data *pgdat);void lru_gen_init_lruvec (struct lruvec *lruvec); -void lru_gen_look_around (struct page_vma_mapped_walk *pvmw); +bool lru_gen_look_around (struct page_vma_mapped_walk *pvmw);void lru_gen_init_memcg (struct mem_cgroup *memcg);void lru_gen_exit_memcg (struct mem_cgroup *memcg);
@@ -576,8 +576,9 @@ static inline void lru_gen_init_lruvec(struct lruvec *lruvec) { } -static inline void lru_gen_look_around (struct page_vma_mapped_walk *pvmw) +static inline bool lru_gen_look_around (struct page_vma_mapped_walk *pvmw) { +return false; }static inline void lru_gen_init_memcg (struct mem_cgroup *memcg)
mm/rmap.c +3 -6
@@ -896,13 +896,10 @@ static bool folio_referenced_one(struct folio *folio,return false; } -if (pvmw.pte) { -if (lru_gen_enabled () && -pte_young (ptep_get (pvmw.pte))) { -lru_gen_look_around (&pvmw); +if (lru_gen_enabled () && pvmw.pte) { +if (lru_gen_look_around (&pvmw)) referenced++; - } - + }else if (pvmw.pte) {if (ptep_clear_flush_young_notify (vma, address, pvmw.pte)) referenced++;
mm/vmscan.c +49 -42
222 lines not on this page · raw message
Signature · 1 line
2.47.0.105.g07ac214952-goog