mm: mglru: fix swappiness behavior

From: Kairui Song <[email protected]>

Change lruvec_evictable_size() to return the number of evictable anonymous and file pages separately instead of a single total, so that callers can weigh the two types independently.

Update lruvec_is_sizable() and get_nr_to_scan() accordingly; the summed totals are unchanged, so there is no functional change.

Signed-off-by Kairui Song <[email protected]>

Signed-off-by Barry Song (Xiaomi) <[email protected]>

---
 mm/vmscan.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

mm/vmscan.c +13 -10

index fc823fe3bd1f..3be0d6c04a82 100644
@@ -4332,32 +4332,35 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
        sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
 }
 
-static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
+static void lruvec_evictable_size(struct lruvec *lruvec, int swappiness,
+                                 unsigned long nr[])
 {
        int gen, type, zone;
-       unsigned long seq, total = 0;
+       unsigned long seq;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
        DEFINE_MAX_SEQ(lruvec);
        DEFINE_MIN_SEQ(lruvec);
 
+       nr[LRU_GEN_ANON] = 0;
+       nr[LRU_GEN_FILE] = 0;
+
        for_each_evictable_type(type, swappiness) {
                for (seq = min_seq[type]; seq <= max_seq; seq++) {
                        gen = lru_gen_from_seq(seq);
                        for (zone = 0; zone < MAX_NR_ZONES; zone++)
-                               total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+                               nr[type] += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
                }
        }
-
-       return total;
 }
 
 static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
 {
-       unsigned long total;
+       unsigned long total, sizes[ANON_AND_FILE];
        int swappiness = get_swappiness(lruvec, sc);
        struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 
-       total = lruvec_evictable_size(lruvec, swappiness);
+       lruvec_evictable_size(lruvec, swappiness, sizes);
+       total = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
 
        /* whether the size is big enough to be helpful */
        return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;

@@ -5181,7 +5184,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
 static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
                           struct mem_cgroup *memcg, int swappiness)
 {
-       unsigned long nr_to_scan, evictable;
+       unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
        struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 
        /*

@@ -5197,7 +5200,8 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
                        return 0;
        }
 
-       evictable = lruvec_evictable_size(lruvec, swappiness);
+       lruvec_evictable_size(lruvec, swappiness, sizes);
+       evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
 
        /* try to scrape all its memory if this memcg was deleted */
        if (!mem_cgroup_online(memcg))
--

2.34.1

From: Kairui Song <[email protected]>

The search for the oldest populated generation of a type, previously embedded in try_to_inc_min_seq(), is extracted into the new lruvec_populated_min_seq() helper. It returns the oldest populated seq of a given type, bounded by the given max_seq so the search never goes beyond it.

No functional change.

Signed-off-by Kairui Song <[email protected]>

Signed-off-by Barry Song (Xiaomi) <[email protected]>

---
 mm/vmscan.c | 43 +++++++++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 14 deletions(-)

mm/vmscan.c +29 -15

index 3be0d6c04a82..500ef23949fd 100644
@@ -4131,30 +4131,45 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
        return true;
 }
 
+/* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
+static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
+                                             unsigned long max_seq)
+{
+       int gen, zone;
+       unsigned long min_seq;
+       struct lru_gen_folio *lrugen = &lruvec->lrugen;
+
+       lockdep_assert_held(&lruvec->lru_lock);
+
+       min_seq = READ_ONCE(lrugen->min_seq[type]);
+       while (min_seq <= max_seq) {
+               gen = lru_gen_from_seq(min_seq);
+               for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+                       if (!list_empty(&lrugen->folios[gen][type][zone]))
+                               return min_seq;
+               }
+               min_seq++;
+       }
+
+       return max_seq;
+}
+
 static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
 {
-       int gen, type, zone;
+       int type;
        bool seq_inc_flag = false;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
        DEFINE_MIN_SEQ(lruvec);
+       unsigned long seq;
 
        VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
 
-       /* find the oldest populated generation */
+       /* Requires at least MIN_NR_GENS gens for basic functionality */
+       seq = lrugen->max_seq - MIN_NR_GENS;
        for_each_evictable_type(type, swappiness) {
-               while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
-                       gen = lru_gen_from_seq(min_seq[type]);
-
-                       for (zone = 0; zone < MAX_NR_ZONES; zone++) {
-                               if (!list_empty(&lrugen->folios[gen][type][zone]))
-                                       goto next;
-                       }
-
-                       min_seq[type]++;
+               min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+               if (min_seq[type] > lrugen->min_seq[type])
                        seq_inc_flag = true;
-               }
-next:
-               ;
        }
 
        /*
--

2.34.1

From: Kairui Song <[email protected]>

Currently MGLRU tries to keep the gen numbers of the two types in sync, avoiding reducing the gen number of one type when the other type has more gens, as described in the comment in try_to_inc_min_seq().

This makes scan_folios() stuck reclaiming the type with fewer gens: it always scanned the oldest gen of the type, even when that gen was empty, so reclaim made no progress on that type unless forced aging created a new gen and eliminated the dead empty gen.

There are other approaches, like lifting the gen number synchronization [1]. However, that would break the gen layout used by lru_gen_folio_seq(): newly added folios would no longer have a well-defined hotness according to their gen number, which reduces accuracy. It is also not pressure-aware, since with gentle reclaim we still want to respect the gen layout (hotness).

Instead, reclaim from whichever populated oldest gen is left, but only when the default reclaim priority can't satisfy the reclaimer's requirement. This is typical for swappiness limitations: e.g., low swappiness asks the reclaimer to reclaim from file, but file gens are drained, so the reclaim cycle fails and priority is raised.

At the default priority the scan quota is 1/4096 of the evictable size, and the old behavior is kept there to respect the hotness distribution. Once priority is raised, the quota is at least 1/2048, so the hotness bias introduced by reclaiming from a younger gen is bounded by that quota, which seems fair. Global reclaim for MGLRU may start with DEF_PRIORITY / 2 (1/64); the ratio is still reasonable at that point, and when global pressure is more critical, we should reclaim harder.

Note that scan_folios() still refuses to reclaim a type left with only MIN_NR_GENS gens, so a completely drained type still requires aging. This change only avoids scanning empty gens when populated reclaimable gens remain.

Link https://lore.kernel.org/linux-mm/[email protected]/ [1] open here

Signed-off-by Kairui Song <[email protected]>

Signed-off-by Barry Song <[email protected]>

---
 mm/vmscan.c | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

mm/vmscan.c +19 -12

index 500ef23949fd..d7eb15533de1 100644
@@ -3451,11 +3451,9 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
 }
 
 /* protect pages accessed multiple times through file descriptors */
-static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, int old_gen)
 {
-       int type = folio_is_file_lru(folio);
-       struct lru_gen_folio *lrugen = &lruvec->lrugen;
-       int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+       int new_gen;
        bool gen_increased;
 
        new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);

@@ -4836,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
  ******************************************************************************/
 
 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
-                      int tier_idx)
+                      int min_gen, int tier_idx)
 {
        int gen = folio_lru_gen(folio);
        int type = folio_is_file_lru(folio);

@@ -4854,14 +4852,14 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
                return false;
 
        /* promoted */
-       if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
+       if (gen != min_gen) {
                list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
                return true;
        }
 
        /* protected */
        if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
-               gen = folio_inc_gen(lruvec, folio);
+               gen = folio_inc_gen(lruvec, folio, min_gen);
                list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
 
                /* don't count the workingset being lazily promoted */

@@ -4876,7 +4874,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
 
        /* ineligible */
        if (zone > sc->reclaim_idx) {
-               gen = folio_inc_gen(lruvec, folio);
+               gen = folio_inc_gen(lruvec, folio, min_gen);
                list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
                return true;
        }

@@ -4919,17 +4917,27 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
        int scanned = 0;
        int isolated = 0;
        int skipped = 0;
+       unsigned long min_seq;
        unsigned long remaining = nr_to_scan;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
 
        VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
        VM_WARN_ON_ONCE(!list_empty(list));
 
+       /*
+        * Respect the gen hotness distribution at low priority to avoid
+        * aging and scan cost, but prefer to satisfy the reclaimers request
+        * when under pressure to respect swappiness more and reduce IO cost.
+        */
+       if (sc->priority == DEF_PRIORITY)
+               min_seq = lrugen->min_seq[type];
+       else
+               min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
+
        if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
                return 0;
 
-       gen = lru_gen_from_seq(lrugen->min_seq[type]);
-
+       gen = lru_gen_from_seq(min_seq);
        for (i = MAX_NR_ZONES; i > 0; i--) {
                LIST_HEAD(moved);
                int skipped_zone = 0;

@@ -4947,7 +4955,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 
                        scanned += delta;
 
-                       if (sort_folio(lruvec, folio, sc, tier))
+                       if (sort_folio(lruvec, folio, sc, gen, tier))
                                sorted += delta;
                        else if (isolate_folio(lruvec, folio, sc)) {
                                list_add(&folio->lru, list);
--

2.34.1

From: Kairui Song <[email protected]>

When a type is drained down to MIN_NR_GENS gens, the reclaimer will fail to make any progress on that type without aging.

However aging comes with a cost, and it will take time to finish. Even if we do aging more proactively, when under great pressure, gens will still be drained very fast so the new gens produced by aging can't catch up with the consumption. With MIN_NR_GENS setting blocking the reclaim, we will run into OOM. This OOM can be reproduced by the previous test program [1] used for fixing another aging problem [2]. When running in great pressure, certain workloads will keep seeing premature OOM with MGLRU, and never see any OOM with classical LRU.

This might get worse if we implement async aging, or avoid hotness inversion in aging, as they will slow down the aging process even more.

To fix this, let's first drop the get_nr_gens() == MIN_NR_GENS early return and let the raised-priority scan bound cover max_seq itself.

Mapped folios will still survive as the rmap PTE check will keep them. Unmapped cache folios may thrashing more when under great pressure, which is really a problem that should be fixed with refault distance and frequecy aware promotoin instead [3].

Link https://github.com/ryncsn/emm-test-project/tree/master/file-anon-mix-pressure [1]

Link https://lore.kernel.org/linux-mm/[email protected]/ [2] open here

Link https://lore.kernel.org/linux-mm/[email protected]/ [3] open here

Signed-off-by Kairui Song <[email protected]>

Signed-off-by Barry Song (Xiaomi) <[email protected]>

---
 mm/vmscan.c | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

mm/vmscan.c +25 -10

index d7eb15533de1..2cccaa5dd4e7 100644
@@ -4834,7 +4834,7 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
  ******************************************************************************/
 
 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
-                      int min_gen, int tier_idx)
+                      int min_gen, int max_gen, int tier_idx)
 {
        int gen = folio_lru_gen(folio);
        int type = folio_is_file_lru(folio);

@@ -4851,12 +4851,22 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
        if (!folio_evictable(folio))
                return false;
 
-       /* promoted */
+       /* promoted, or moved due to reparenting. */
        if (gen != min_gen) {
                list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
                return true;
        }
 
+       /*
+        * We are reclaiming the last gen left. Folios in this gen can no
+        * longer be promoted or protected, so just isolate them. The one
+        * exception is lru_gen_reparent_memcg(), which is handled above.
+        */
+       if (min_gen == max_gen) {
+               VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+               return false;
+       }
+
        /* protected */
        if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
                gen = folio_inc_gen(lruvec, folio, min_gen);

@@ -4911,15 +4921,16 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                       struct list_head *list, int *isolatedp)
 {
        int i;
-       int gen;
        enum node_stat_item item;
        int sorted = 0;
        int scanned = 0;
        int isolated = 0;
        int skipped = 0;
+       int gen, max_gen;
        unsigned long min_seq;
        unsigned long remaining = nr_to_scan;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
+       DEFINE_MAX_SEQ(lruvec);
 
        VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
        VM_WARN_ON_ONCE(!list_empty(list));

@@ -4932,18 +4943,24 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
        if (sc->priority == DEF_PRIORITY)
                min_seq = lrugen->min_seq[type];
        else
-               min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq - MIN_NR_GENS + 1);
-
-       if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
-               return 0;
+               min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
 
        gen = lru_gen_from_seq(min_seq);
+       max_gen = lru_gen_from_seq(max_seq);
+
        for (i = MAX_NR_ZONES; i > 0; i--) {
                LIST_HEAD(moved);
                int skipped_zone = 0;
                int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES;
                struct list_head *head = &lrugen->folios[gen][type][zone];
 
+               /*
+                * If there is only one gen left, rotating the unreclaimable
+                * zone become meaningless, just skip them.
+                */
+               if (gen == max_gen && zone > sc->reclaim_idx)
+                       continue;
+
                while (!list_empty(head)) {
                        struct folio *folio = lru_to_folio(head);
                        int delta = folio_nr_pages(folio);

@@ -4954,8 +4971,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                        VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
                        scanned += delta;
-
-                       if (sort_folio(lruvec, folio, sc, gen, tier))
+                       if (sort_folio(lruvec, folio, sc, gen, max_gen, tier))
                                sorted += delta;
                        else if (isolate_folio(lruvec, folio, sc)) {
                                list_add(&folio->lru, list);
--

2.34.1

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm/mglru: pinned min_seq in higher zones stalls aging in scan_folios() --

commit 1b2f683d6b8bd1e171eb9677315ba8a3d9431b9b
Author: Kairui Song <[email protected]>

mm/mglru: skip zones above reclaim_idx instead of rotating them

This commit restricts scan_folios() and lruvec_populated_min_seq() during eviction to zones at or below sc->reclaim_idx, and removes the ineligible-zone rotation branch from sort_folio(). try_to_inc_min_seq() continues to check all zones up to MAX_NR_ZONES before advancing min_seq.

Quoted 31 lines

mm/vmscan.c +3 -3

index a365a4bc8c988..09ea72b354a56 100644
@@ -4132,7 +4132,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 
 /* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
 static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
-                                           unsigned long max_seq)
+                                           int zone_limit, unsigned long max_seq)
 {
      int gen, zone;
      unsigned long min_seq;

@@ -4143,7 +4143,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
      min_seq = READ_ONCE(lrugen->min_seq[type]);
      while (min_seq <= max_seq) {
              gen = lru_gen_from_seq(min_seq);
-             for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+             for (zone = 0; zone < zone_limit; zone++) {
                      if (!list_empty(&lrugen->folios[gen][type][zone]))
                              return min_seq;
              }

@@ -4166,7 +4166,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
      /* Requires at least MIN_NR_GENS gens for basic functionality */
      seq = lrugen->max_seq - MIN_NR_GENS;
      for_each_evictable_type(type, swappiness) {
-             min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+             min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
              if (min_seq[type] > lrugen->min_seq[type])
                      seq_inc_flag = true;
      }

[ ... ]

Quoted 61 lines

mm/vmscan.c +8 -15

@@ -4863,10 +4864,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
       * longer be promoted or protected, so just isolate them. The one
       * exception is lru_gen_reparent_memcg(), which is handled above.
       */
-     if (min_gen == max_gen) {
-             VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+     if (min_gen == max_gen)
              return false;
-     }
 
      /* protected */
      if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {

@@ -4883,13 +4882,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
              return true;
      }
 
-     /* ineligible */
-     if (zone > sc->reclaim_idx) {
-             gen = folio_inc_gen(lruvec, folio, min_gen);
-             list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
-             return true;
-     }
-
      return false;
 }
 

@@ -4921,16 +4913,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                     struct scan_control *sc, int type, int tier,
                     struct list_head *list, int *isolatedp)
 {
-     int i;
      enum node_stat_item item;
+     int zone_idx;
      int sorted = 0;
      int scanned = 0;
      int isolated = 0;
      int skipped = 0;
      int gen, max_gen;
-     unsigned long min_seq;
+     unsigned long seq;
      unsigned long remaining = nr_to_scan;
      struct lru_gen_folio *lrugen = &lruvec->lrugen;
+     struct pglist_data *pgdat = lruvec_pgdat(lruvec);
      DEFINE_MAX_SEQ(lruvec);
 
      VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);

@@ -4942,24 +4935,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
       * when under pressure to respect swappiness more and reduce IO cost.
       */
      if (sc->priority == DEF_PRIORITY)
-             min_seq = lrugen->min_seq[type];
+             seq = lrugen->min_seq[type];
      else
-             min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
+             seq = lrugen->max_seq;
 
-     gen = lru_gen_from_seq(min_seq);
+     seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+     gen = lru_gen_from_seq(seq);
      max_gen = lru_gen_from_seq(max_seq);
 

mm/vmscan.c static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 4946

Quoted 3 lines

mm/vmscan.c +2 -1

-     for (i = MAX_NR_ZONES; i > 0; i--) {
+     /* Scan eligible zones in descending order. */
+     for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {

[Severity: High]
During zone-constrained reclaim in mm/vmscan.c (where sc->reclaim_idx is below MAX_NR_ZONES - 1 and zones above sc->reclaim_idx are populated, such as GFP_DMA32 reclaim with a populated ZONE_NORMAL or GFP_KERNEL reclaim with a populated ZONE_MOVABLE), can keeping MAX_NR_ZONES in try_to_inc_min_seq() while removing the ineligible-zone rotation from sort_folio() stall aging and memcg rotation?

Quoted 110 lines · On Sat, Oct 24, 2026 at 6:03 AM <[email protected]> wrote:

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm/mglru: pinned min_seq in higher zones stalls aging in scan_folios() --

commit 1b2f683d6b8bd1e171eb9677315ba8a3d9431b9b
Author: Kairui Song <[email protected]>

mm/mglru: skip zones above reclaim_idx instead of rotating them

This commit restricts scan_folios() and lruvec_populated_min_seq() during eviction to zones at or below sc->reclaim_idx, and removes the ineligible-zone rotation branch from sort_folio(). try_to_inc_min_seq() continues to check all zones up to MAX_NR_ZONES before advancing min_seq.

Quoted 31 lines

mm/vmscan.c +3 -3

index a365a4bc8c988..09ea72b354a56 100644
@@ -4132,7 +4132,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 
 /* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
 static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
-                                           unsigned long max_seq)
+                                           int zone_limit, unsigned long max_seq)
 {
      int gen, zone;
      unsigned long min_seq;

@@ -4143,7 +4143,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
      min_seq = READ_ONCE(lrugen->min_seq[type]);
      while (min_seq <= max_seq) {
              gen = lru_gen_from_seq(min_seq);
-             for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+             for (zone = 0; zone < zone_limit; zone++) {
                      if (!list_empty(&lrugen->folios[gen][type][zone]))
                              return min_seq;
              }

@@ -4166,7 +4166,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
      /* Requires at least MIN_NR_GENS gens for basic functionality */
      seq = lrugen->max_seq - MIN_NR_GENS;
      for_each_evictable_type(type, swappiness) {
-             min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+             min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
              if (min_seq[type] > lrugen->min_seq[type])
                      seq_inc_flag = true;
      }

[ ... ]

Quoted 61 lines

mm/vmscan.c +8 -15

@@ -4863,10 +4864,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
       * longer be promoted or protected, so just isolate them. The one
       * exception is lru_gen_reparent_memcg(), which is handled above.
       */
-     if (min_gen == max_gen) {
-             VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+     if (min_gen == max_gen)
              return false;
-     }
 
      /* protected */
      if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {

@@ -4883,13 +4882,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
              return true;
      }
 
-     /* ineligible */
-     if (zone > sc->reclaim_idx) {
-             gen = folio_inc_gen(lruvec, folio, min_gen);
-             list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
-             return true;
-     }
-
      return false;
 }
 

@@ -4921,16 +4913,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                     struct scan_control *sc, int type, int tier,
                     struct list_head *list, int *isolatedp)
 {
-     int i;
      enum node_stat_item item;
+     int zone_idx;
      int sorted = 0;
      int scanned = 0;
      int isolated = 0;
      int skipped = 0;
      int gen, max_gen;
-     unsigned long min_seq;
+     unsigned long seq;
      unsigned long remaining = nr_to_scan;
      struct lru_gen_folio *lrugen = &lruvec->lrugen;
+     struct pglist_data *pgdat = lruvec_pgdat(lruvec);
      DEFINE_MAX_SEQ(lruvec);
 
      VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);

@@ -4942,24 +4935,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
       * when under pressure to respect swappiness more and reduce IO cost.
       */
      if (sc->priority == DEF_PRIORITY)
-             min_seq = lrugen->min_seq[type];
+             seq = lrugen->min_seq[type];
      else
-             min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
+             seq = lrugen->max_seq;
 
-     gen = lru_gen_from_seq(min_seq);
+     seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+     gen = lru_gen_from_seq(seq);
      max_gen = lru_gen_from_seq(max_seq);
 

Quoted 11 lines

mm/vmscan.c static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 4946

Quoted 3 lines

mm/vmscan.c +2 -1

-     for (i = MAX_NR_ZONES; i > 0; i--) {
+     /* Scan eligible zones in descending order. */
+     for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {

[Severity: High]
During zone-constrained reclaim in mm/vmscan.c (where sc->reclaim_idx is below MAX_NR_ZONES - 1 and zones above sc->reclaim_idx are populated, such as GFP_DMA32 reclaim with a populated ZONE_NORMAL or GFP_KERNEL reclaim with a populated ZONE_MOVABLE), can keeping MAX_NR_ZONES in try_to_inc_min_seq() while removing the ineligible-zone rotation from sort_folio() stall aging and memcg rotation?

Yeah it can stall aging, but this series lifted the restriction of MIN_NR_GEN on eviction, and eviction will start the the last populated gen of a zone instead of min_seq, and we have more proactive aging when the gens are imbalanced, so this won't be a problem, it won't cause OOM. I'm not sure if it will cause any performance issue though, if this is concerning we can keep the old behavior, or trigger aging when the populated gen that satisfy a reclain is below (MIN_NR_GEN -
1) or imbalanced, this seems a better solution to me, still avoid a lot of redundant work while ensure there is no reclaim effenciency issue. We already have the right helper to do so in this series, so seem not hard or dirty.

Quoted 124 lines · Kairui Song

Quoted 110 lines · On Sat, Oct 24, 2026 at 6:03 AM <[email protected]> wrote:

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] mm/mglru: pinned min_seq in higher zones stalls aging in scan_folios() --

commit 1b2f683d6b8bd1e171eb9677315ba8a3d9431b9b
Author: Kairui Song <[email protected]>

mm/mglru: skip zones above reclaim_idx instead of rotating them

This commit restricts scan_folios() and lruvec_populated_min_seq() during eviction to zones at or below sc->reclaim_idx, and removes the ineligible-zone rotation branch from sort_folio(). try_to_inc_min_seq() continues to check all zones up to MAX_NR_ZONES before advancing min_seq.

Quoted 31 lines

mm/vmscan.c +3 -3

index a365a4bc8c988..09ea72b354a56 100644
@@ -4132,7 +4132,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 
  /* Similar to lruvec_evictable_size, return the oldest populated seq of the type */
  static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
-                                           unsigned long max_seq)
+                                           int zone_limit, unsigned long max_seq)
  {
       int gen, zone;
       unsigned long min_seq;

@@ -4143,7 +4143,7 @@ static unsigned long lruvec_populated_min_seq(struct lruvec *lruvec, int type,
       min_seq = READ_ONCE(lrugen->min_seq[type]);
       while (min_seq <= max_seq) {
               gen = lru_gen_from_seq(min_seq);
-             for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+             for (zone = 0; zone < zone_limit; zone++) {
                       if (!list_empty(&lrugen->folios[gen][type][zone]))
                               return min_seq;
               }

@@ -4166,7 +4166,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
       /* Requires at least MIN_NR_GENS gens for basic functionality */
       seq = lrugen->max_seq - MIN_NR_GENS;
       for_each_evictable_type(type, swappiness) {
-             min_seq[type] = lruvec_populated_min_seq(lruvec, type, seq + 1);
+             min_seq[type] = lruvec_populated_min_seq(lruvec, type, MAX_NR_ZONES, seq + 1);
               if (min_seq[type] > lrugen->min_seq[type])
                       seq_inc_flag = true;
       }

[ ... ]

Quoted 61 lines

mm/vmscan.c +5 -13

@@ -4863,10 +4864,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
        * longer be promoted or protected, so just isolate them. The one
        * exception is lru_gen_reparent_memcg(), which is handled above.
        */
-     if (min_gen == max_gen) {
-             VM_WARN_ON_ONCE(zone > sc->reclaim_idx);
+     if (min_gen == max_gen)
               return false;
-     }
 
       /* protected */
       if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {

@@ -4883,13 +4882,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
               return true;
       }
 
-     /* ineligible */
-     if (zone > sc->reclaim_idx) {
-             gen = folio_inc_gen(lruvec, folio, min_gen);
-             list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
-             return true;
-     }
-
       return false;
  }
 

@@ -4921,16 +4913,17 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                      struct scan_control *sc, int type, int tier,
                      struct list_head *list, int *isolatedp)
  {
-     int i;
       enum node_stat_item item;
+     int zone_idx;
       int sorted = 0;
       int scanned = 0;
       int isolated = 0;
       int skipped = 0;
       int gen, max_gen;
-     unsigned long min_seq;
+     unsigned long seq;
       unsigned long remaining = nr_to_scan;
       struct lru_gen_folio *lrugen = &lruvec->lrugen;
+     struct pglist_data *pgdat = lruvec_pgdat(lruvec);
       DEFINE_MAX_SEQ(lruvec);
 
       VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);

@@ -4942,24 +4935,22 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
        * when under pressure to respect swappiness more and reduce IO cost.
        */
       if (sc->priority == DEF_PRIORITY)
-             min_seq = lrugen->min_seq[type];
+             seq = lrugen->min_seq[type];

      else
-             min_seq = lruvec_populated_min_seq(lruvec, type, lrugen->max_seq);
+             seq = lrugen->max_seq;

-     gen = lru_gen_from_seq(min_seq);
+     seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+     gen = lru_gen_from_seq(seq);
      max_gen = lru_gen_from_seq(max_seq);

Quoted 11 lines

Quoted 3 lines

-     for (i = MAX_NR_ZONES; i > 0; i--) {
+     /* Scan eligible zones in descending order. */
+     for (zone_idx = sc->reclaim_idx; zone_idx >= 0; zone_idx--) {

[Severity: High]
During zone-constrained reclaim in mm/vmscan.c (where sc->reclaim_idx is below MAX_NR_ZONES - 1 and zones above sc->reclaim_idx are populated, such as GFP_DMA32 reclaim with a populated ZONE_NORMAL or GFP_KERNEL reclaim with a populated ZONE_MOVABLE), can keeping MAX_NR_ZONES in try_to_inc_min_seq() while removing the ineligible-zone rotation from sort_folio() stall aging and memcg rotation?

Quoted 11 lines

Yeah it can stall aging, but this series lifted the restriction of MIN_NR_GEN on eviction, and eviction will start the the last populated gen of a zone instead of min_seq, and we have more proactive aging when the gens are imbalanced, so this won't be a problem, it won't cause OOM. I'm not sure if it will cause any performance issue though, if this is concerning we can keep the old behavior, or trigger aging when the populated gen that satisfy a reclain is below (MIN_NR_GEN -
1) or imbalanced, this seems a better solution to me, still avoid a lot of redundant work while ensure there is no reclaim effenciency issue. We already have the right helper to do so in this series, so seem not hard or dirty.

Thanks, that makes sense regarding OOM.
Could lru_gen_imbalanced() trigger aging soon enough when few populated generations are eligible for reclaim, or could reclaim efficiency suffer while min_seq remains pinned?

Would the additional trigger you suggested for this case be worthwhile?

Thanks,
Kunwu

From: Kairui Song <[email protected]>

No feature change, just shuffle some code, prepare for later commits.

Signed-off-by Kairui Song <[email protected]>

Signed-off-by Barry Song (Xiaomi) <[email protected]>

---
 mm/vmscan.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

mm/vmscan.c +13 -15

index d3caed8aa789..0930bee5906e 100644
@@ -5215,20 +5215,6 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
                           struct mem_cgroup *memcg, int swappiness)
 {
        unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
-       struct pglist_data *pgdat = lruvec_pgdat(lruvec);
-
-       /*
-        * Proactive reclaim initiated by userspace for anonymous memory only.
-        * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
-        * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
-        * no swap), return 0 to skip the scan entirely, avoiding useless scan
-        * work when there is nothing eligible to reclaim.
-        */
-       if (swappiness == SWAPPINESS_ANON_ONLY) {
-               WARN_ON_ONCE(!sc->proactive);
-               if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc))
-                       return 0;
-       }
 
        lruvec_evictable_size(lruvec, swappiness, sizes);
        evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];

@@ -5285,6 +5271,19 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
        int swappiness = get_swappiness(lruvec, sc);
        struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 
+       /*
+        * Proactive reclaim initiated by userspace for anonymous memory only.
+        * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
+        * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
+        * no swap), return 0 to skip the scan entirely, avoiding useless scan
+        * work when there is nothing eligible to reclaim.
+        */
+       if (swappiness == SWAPPINESS_ANON_ONLY) {
+               WARN_ON_ONCE(!sc->proactive);
+               if (!can_reclaim_anon_pages(memcg, lruvec_pgdat(lruvec)->node_id, sc))
+                       return false;
+       }
+
        nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
        while (nr_to_scan > 0) {
                int delta;
--

2.34.1

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/mglru: lru_gen_balance_scan() budgets un-scannable gens and zones --

commit ce196f9c0d172466d44f73b512ed408a9ecb6e36
Author: Kairui Song <[email protected]>

mm/mglru: balance anon and file scan budget by refault rate

This commit replaces get_type_to_scan() and isolate_folios() with lru_gen_balance_scan() and evict_folio_lists() to split the scan budget proportionally between anon and file LRUs based on per-type refault rates and LRU sizes.

Quoted 5 lines

mm/vmscan.c +0 -0

index cc20ba8c6cf2c..fed39b0422e01 100644
@@ -5019,79 +5019,110 @@ static int get_tier_idx(struct lruvec *lruvec, int type)

[ ... ]

Quoted 3 lines

mm/vmscan.c +2 -0

+static void lru_gen_balance_scan(struct lruvec *lruvec, int swappiness,
+             unsigned long total_scan, unsigned long nr[])
 {

[ ... ]

Quoted 45 lines

mm/vmscan.c +24 -17

-     /*
-      * We are running out of the current reclaim type. Fall back to
-      * the other type if allowed.
+      * The more extreme the swappiness, the wider the IO cost gap between
+      * anon and file, so the split should follow the IO cost ratio rather
+      * than the sizes. With a mild swappiness, balance the scan by the
+      * actual LRU size more.
       */
-     if (!scanned && !tried && type_fallback_allowed) {
-             type = !type;
-             tried = true;
-             type_fallback_allowed = false;
-             goto retry;
-     }
+     size_bias = max(1, min(swappiness, MAX_SWAPPINESS - swappiness));
+     anon_weight = nr[LRU_GEN_ANON] + nr[LRU_GEN_FILE] / size_bias;
+     file_weight = nr[LRU_GEN_FILE] + nr[LRU_GEN_ANON] / size_bias;
+
+     anon_factor = (u64)swappiness * refault_rate[LRU_GEN_FILE] * anon_weight;
+     file_factor = (u64)(MAX_SWAPPINESS - swappiness) * refault_rate[LRU_GEN_ANON] * file_weight;
+
+     /* Cross-multiply the ratio above. */
+     anon_scan = mul_u64_u64_div_u64(total_scan, anon_factor, anon_factor + file_factor);
+     file_scan = (u64)total_scan - anon_scan;
+
      /*
-      * We scanned some folios but failed to isolate any due to promotions,
-      * protections, or races. Retry once to avoid a larger loop.
+      * Bound each type by its evictable size. Hand the exceeding part to
+      * the other type. Any excess left when total_scan exceeds both sizes
+      * is dropped, since it only stresses the scanner with no gain.
       */
-     if (scanned && !tried) {
-             tried = true;
-             goto retry;
-     }
-
-     return total_scanned;
+     if (anon_scan > nr[LRU_GEN_ANON])
+             file_scan += anon_scan - nr[LRU_GEN_ANON];
+     if (file_scan > nr[LRU_GEN_FILE])
+             anon_scan += file_scan - nr[LRU_GEN_FILE];
+     nr[LRU_GEN_ANON] = min_t(u64, anon_scan, nr[LRU_GEN_ANON]);
+     nr[LRU_GEN_FILE] = min_t(u64, file_scan, nr[LRU_GEN_FILE]);
 }

[ ... ]

Quoted 61 lines

mm/vmscan.c +34 -11

@@ -5212,22 +5244,22 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
      return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
 }
 
-static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
-                        struct mem_cgroup *memcg, int swappiness)
+static void lru_gen_prepare_scan(struct lruvec *lruvec, struct scan_control *sc,
+                              struct mem_cgroup *memcg, int swappiness,
+                              unsigned long nr_to_scan[])
 {
-     unsigned long nr_to_scan, evictable, sizes[ANON_AND_FILE];
+     unsigned long total;
 
-     lruvec_evictable_size(lruvec, swappiness, sizes);
-     evictable = sizes[LRU_GEN_ANON] + sizes[LRU_GEN_FILE];
+     lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+     total = nr_to_scan[LRU_GEN_ANON] + nr_to_scan[LRU_GEN_FILE];
 
-     /* try to scrape all its memory if this memcg was deleted */
+     /* Just scrape all pages calculated above if this memcg was deleted */
      if (!mem_cgroup_online(memcg))
-             return evictable;
-
-     nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
-     nr_to_scan >>= sc->priority;
+             return;
 
-     return nr_to_scan;
+     total = apply_proportional_protection(memcg, sc, total);
+     total >>= sc->priority;
+     lru_gen_balance_scan(lruvec, swappiness, total, nr_to_scan);
 }
 
 static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)

@@ -5260,6 +5292,30 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
      return true;
 }
 
+static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec,
+                          struct scan_control *sc, int swappiness, unsigned long max_batch)
+{
+     int type;
+     unsigned long batch, scanned, total_scanned = 0;
+
+     /*
+      * Scan both types with the given budgets; the caller checks if
+      * reclaim is satisfied after each pass.
+      */
+     for (type = LRU_GEN_FILE; type >= LRU_GEN_ANON; type--) {
+             if (!nr_to_scan[type])
+                     continue;
+
+             batch = min(nr_to_scan[type], max_batch);
+             scanned = evict_folios(batch, lruvec, sc, type, swappiness);
+
+             nr_to_scan[type] -= min(nr_to_scan[type], scanned);
+             total_scanned += scanned;
+     }
+
+     return total_scanned;
+}

[ ... ]

Quoted 13 lines

mm/vmscan.c +5 -2

@@ -5948,8 +6001,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
              if (sc->nr_reclaimed >= nr_to_reclaim)
                      return 0;
 
-             nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
-             if (!evict_folios(nr_batch, lruvec, sc, swappiness))
+             lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+             lru_gen_balance_scan(lruvec, swappiness,
+                                  nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
+
+             if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
                      return 0;
 

mm/vmscan.c static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co · line 6011

Quoted 1 line

mm/vmscan.c +0 -0

              cond_resched();

[Severity: Medium]
Can budgeting with lruvec_evictable_size() in lru_gen_balance_scan() cause lost scan budget and premature reclaim termination in try_to_shrink_lruvec() and run_eviction() when one type has no folios in scannable generations or eligible zones?

Quoted 13 lines

Quoted 13 lines

mm/vmscan.c +5 -2

@@ -5948,8 +6001,11 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
              if (sc->nr_reclaimed >= nr_to_reclaim)
                      return 0;
 
-             nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
-             if (!evict_folios(nr_batch, lruvec, sc, swappiness))
+             lruvec_evictable_size(lruvec, swappiness, nr_to_scan);
+             lru_gen_balance_scan(lruvec, swappiness,
+                                  nr_to_reclaim - sc->nr_reclaimed, nr_to_scan);
+
+             if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH))
                      return 0;
 

Quoted 7 lines

Review static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co · line 6011

Quoted 1 line

(quoted diff) +0 -0

              cond_resched();

[Severity: Medium]
Can budgeting with lruvec_evictable_size() in lru_gen_balance_scan() cause lost scan budget and premature reclaim termination in try_to_shrink_lruvec() and run_eviction() when one type has no folios in scannable generations or eligible zones?

At the moment it's like part of the design, prioiry will raise and in the end we will have no scannable generation limit, zone might worth be considered when getting the evictable size though?

Also, maybe we can improve the aging on imbalance to trigger when the scan budget is much larger than the scannable size (and priority is raised beyound default)?

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/mglru: scan_folios() scans expired gen < min_seq at DEF_PRIORITY --

commit 1329290ce5f86b7de9b77d6b22e4b265f16348ae
Author: Kairui Song <[email protected]>

mm/mglru: lift youngest gen protection for higher priority

This commit adjusts scan_folios() in mm/vmscan.c to choose the target generation sequence number based on reclaim priority rather than using lrugen->min_seq[type] at default priority. It aims to protect younger generations at default priority while allowing higher-priority reclaim to scan newer generations step by step.

Quoted 13 lines

mm/vmscan.c +2 -3

index fed39b0422e01..d1e14001f9346 100644
@@ -4930,12 +4930,14 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
      VM_WARN_ON_ONCE(!list_empty(list));
 
      /*
-      * Respect the gen hotness distribution at low priority to avoid
-      * aging and scan cost, but prefer to satisfy the reclaimers request
-      * when under pressure to respect swappiness more and reduce IO cost.
+      * Respect hotness at default priority, and only reach into the
+      * newest generation when under severe pressure, so hot folios are

mm/vmscan.c static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 4943

Quoted 10 lines

mm/vmscan.c +4 -1

+      * not evicted as soon as priority is raised.
       */
      if (sc->priority == DEF_PRIORITY)
-             seq = lrugen->min_seq[type];
+             seq = lrugen->max_seq - MIN_NR_GENS;
+     else if (sc->priority > DEF_PRIORITY / 2)
+             seq = lrugen->max_seq - 1;
      else
              seq = lrugen->max_seq;
 

[Severity: Medium]
Can this cause scan_folios() in mm/vmscan.c to scan an already-evicted generation when a type only has the minimum number of generations left?

In try_to_shrink_lruvec() -> evict_folio_lists() -> evict_folios() -> scan_folios() at default priority, when get_nr_gens(lruvec, type) is at the minimum (2 generations), lrugen->min_seq[type] is lrugen->max_seq - 1, while scan_folios() sets seq to lrugen->max_seq - 2 and passes it as max_seq to lruvec_populated_min_seq():

mm/vmscan.c:scan_folios() {
        if (sc->priority == DEF_PRIORITY)
                seq = lrugen->max_seq - MIN_NR_GENS;
        ...
        seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
        gen = lru_gen_from_seq(seq);
}

In lruvec_populated_min_seq(), because min_seq (lrugen->max_seq - 1) is greater than max_seq (lrugen->max_seq - 2), the while loop is skipped and max_seq is returned directly:

mm/vmscan.c:lruvec_populated_min_seq() {
        min_seq = READ_ONCE(lrugen->min_seq[type]);
        while (min_seq <= max_seq) {
                gen = lru_gen_from_seq(min_seq);
                for (zone = 0; zone < zone_limit; zone++) {
                        if (!list_empty(&lrugen->folios[gen][type][zone]))
                                return min_seq;
                }
                min_seq++;
        }

        return max_seq;
}

When scan_folios() previously scanned younger generations (lrugen->max_seq - 1 or lrugen->max_seq) at elevated priority while get_nr_gens(lruvec, type) had fewer than the maximum number of generations, sort_folio() can move folios with a stale folio_lru_gen() value (such as after __lru_gen_reparent_memcg() or when inc_min_seq() skipped draining old_gen) into that expired lrugen->folios[gen][type][zone] list:

mm/vmscan.c:sort_folio() {
        /* promoted, or moved due to reparenting. */
        if (gen != min_gen) {
                list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
                return true;
        }
}

A subsequent default-priority call to scan_folios() will then visit that expired generation (< lrugen->min_seq[type]) and corrupt active/inactive size accounting.

In addition, in lru_gen_seq_write() -> run_cmd() -> run_eviction(), debugfs eviction runs at default priority. If a user requests evicting up to lrugen->max_seq - 3 when four generations exist and one type has already emptied lrugen->max_seq - 3 while the other type still has min_seq at lrugen->max_seq - 3, wouldn't setting seq to lrugen->max_seq - 2 in scan_folios() evict folios from generation lrugen->max_seq - 2, exceeding the requested seq bound?

Quoted 95 lines · On Sat, Oct 24, 2026 at 6:03 AM <[email protected]> wrote:

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] mm/mglru: scan_folios() scans expired gen < min_seq at DEF_PRIORITY --

commit 1329290ce5f86b7de9b77d6b22e4b265f16348ae
Author: Kairui Song <[email protected]>

mm/mglru: lift youngest gen protection for higher priority

This commit adjusts scan_folios() in mm/vmscan.c to choose the target generation sequence number based on reclaim priority rather than using lrugen->min_seq[type] at default priority. It aims to protect younger generations at default priority while allowing higher-priority reclaim to scan newer generations step by step.

Quoted 13 lines

mm/vmscan.c +2 -3

index fed39b0422e01..d1e14001f9346 100644
@@ -4930,12 +4930,14 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
      VM_WARN_ON_ONCE(!list_empty(list));
 
      /*
-      * Respect the gen hotness distribution at low priority to avoid
-      * aging and scan cost, but prefer to satisfy the reclaimers request
-      * when under pressure to respect swappiness more and reduce IO cost.
+      * Respect hotness at default priority, and only reach into the
+      * newest generation when under severe pressure, so hot folios are

mm/vmscan.c static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 4943

Quoted 10 lines

mm/vmscan.c +4 -1

+      * not evicted as soon as priority is raised.
       */
      if (sc->priority == DEF_PRIORITY)
-             seq = lrugen->min_seq[type];
+             seq = lrugen->max_seq - MIN_NR_GENS;
+     else if (sc->priority > DEF_PRIORITY / 2)
+             seq = lrugen->max_seq - 1;
      else
              seq = lrugen->max_seq;
 

[Severity: Medium]
Can this cause scan_folios() in mm/vmscan.c to scan an already-evicted generation when a type only has the minimum number of generations left?

In try_to_shrink_lruvec() -> evict_folio_lists() -> evict_folios() -> scan_folios() at default priority, when get_nr_gens(lruvec, type) is at the minimum (2 generations), lrugen->min_seq[type] is lrugen->max_seq - 1, while scan_folios() sets seq to lrugen->max_seq - 2 and passes it as max_seq to lruvec_populated_min_seq():

mm/vmscan.c:scan_folios() {
        if (sc->priority == DEF_PRIORITY)
                seq = lrugen->max_seq - MIN_NR_GENS;
        ...
        seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
        gen = lru_gen_from_seq(seq);
}

In lruvec_populated_min_seq(), because min_seq (lrugen->max_seq - 1) is greater than max_seq (lrugen->max_seq - 2), the while loop is skipped and max_seq is returned directly:

mm/vmscan.c:lruvec_populated_min_seq() {
        min_seq = READ_ONCE(lrugen->min_seq[type]);
        while (min_seq <= max_seq) {
                gen = lru_gen_from_seq(min_seq);
                for (zone = 0; zone < zone_limit; zone++) {
                        if (!list_empty(&lrugen->folios[gen][type][zone]))
                                return min_seq;
                }
                min_seq++;
        }

        return max_seq;
}

When scan_folios() previously scanned younger generations (lrugen->max_seq - 1 or lrugen->max_seq) at elevated priority while get_nr_gens(lruvec, type) had fewer than the maximum number of generations, sort_folio() can move folios with a stale folio_lru_gen() value (such as after __lru_gen_reparent_memcg() or when inc_min_seq() skipped draining old_gen) into that expired lrugen->folios[gen][type][zone] list:

mm/vmscan.c:sort_folio() {
        /* promoted, or moved due to reparenting. */
        if (gen != min_gen) {
                list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
                return true;
        }
}

A subsequent default-priority call to scan_folios() will then visit that expired generation (< lrugen->min_seq[type]) and corrupt active/inactive size accounting.

Quoted 8 lines

In addition, in lru_gen_seq_write() -> run_cmd() -> run_eviction(), debugfs eviction runs at default priority. If a user requests evicting up to lrugen->max_seq - 3 when four generations exist and one type has already emptied lrugen->max_seq - 3 while the other type still has min_seq at lrugen->max_seq - 3, wouldn't setting seq to lrugen->max_seq - 2 in scan_folios() evict folios from generation lrugen->max_seq - 2, exceeding the requested seq bound?

Looks a false positive, though it is tricky. A folio on a generation list is only ever promoted, and min_seq[type] is advanced only after the old generation has been drained by inc_min_seq() or verified empty by try_to_inc_min_seq(). So when a type is down to MIN_NR_GENS, the max_seq - MIN_NR_GENS target is empty, not a populated expired generation. Lazy folio flags and the non evictable skip can create temporary skew, and reparenting can look odd across generations, but that skew is transient and cannot survive until a DEF_PRIORITY scan targets that generation. sort_folio() also refuses to promote from the last live generation.

For the debugfs case, lruvec_populated_min_seq() starts from min_seq[type], so the type with min_seq == max_seq - 3 scans max_seq - 3 while that generation is populated. The type already at max_seq - 2 was scanned at max_seq - 2 before this patch too. Not enforcing the requested seq per type is a separate pre-existing debugfs issue, not caused by this change.

We used to scan only the oldest generation in scan_folios(). In evict_folios(), we checked whether lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]. If so, the folio was marked active and moved to younger generations.

This was itself buggy, as folios in generation min_seq[type] - 1 would not be marked active by folio_set_active(), potentially causing a cold/hot inversion.

Besides this issue, the old approach also differs from the current behavior, where we can scan fewer than two generations. Now that we can scan younger generations, including the youngest generation, this no longer prevents rejected folios from being placed in a generation below or at the populated generation. Such rejected folios would then be scanned again and again, resulting in extremely low pg_steal/pg_scan ratios. For example:
pgsteal_file: 2312605
pgscan_file: 36465693
reclamation efficiency: 6.34%

We pass scanned_seq to move_folios_to_lru() and make sure we either promote the folio or place it in the youngest generation when the populated generation is already the youngest one.
Also, folio_set_active() is no longer needed.

With this change, the results become:
pgsteal_file: 2683815
pgscan_file: 4224244
reclamation efficiency: 63.53%

The sys time for building the kernel also decreased from 4m30.532s to 3m50.188s while swappiness==1.

Signed-off-by Barry Song (Xiaomi) <[email protected]>

---
 include/linux/mm_inline.h | 22 +++++++++++++++++++++-
 mm/vmscan.c               | 24 ++++++++++++------------
 2 files changed, 33 insertions(+), 13 deletions(-)

include/linux/mm_inline.h +21 -1

index ab69b9930893..f6f4bc9a704f 100644
@@ -332,7 +332,8 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
        return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type]));
 }
 
-static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
+               unsigned long scanned_seq, bool reclaiming)
 {
        unsigned long seq;
        unsigned long flags;

@@ -347,6 +348,8 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
                return false;
 
        seq = lru_gen_folio_seq(lruvec, folio, reclaiming);
+       if (scanned_seq != -1UL)
+               seq = max(seq, min(scanned_seq + 1, lrugen->max_seq));
        gen = lru_gen_from_seq(seq);
        flags = (gen + 1UL) << LRU_GEN_PGOFF;
        /* see the comment on MIN_NR_GENS about PG_active */

@@ -362,6 +365,17 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
        return true;
 }
 
+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+       return __lru_gen_add_folio(lruvec, folio, -1UL, reclaiming);
+}
+
+static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio,
+               unsigned long scanned_seq)
+{
+       return __lru_gen_add_folio(lruvec, folio, scanned_seq, false);
+}
+
 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
 {
        unsigned long flags;

@@ -420,6 +434,12 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
        return false;
 }
 
+static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio,
+               unsigned long scanned_seq)
+{
+       return false;
+}
+
 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
 {
        return false;

mm/vmscan.c +12 -13

index 12aca97be56b..39f77a68be37 100644
@@ -1971,7 +1971,8 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file,
  *
  * Note: The caller must not hold any lruvec lock.
  */
-static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
+static unsigned int move_folios_to_lru(struct list_head *list, unsigned long seq,
+               bool do_rotate)
 {
        int nr_pages, nr_moved = 0;
        struct lruvec *lruvec = NULL;

@@ -2028,7 +2029,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
                    !folio_test_referenced(folio) &&
                    folio_ref_count(folio) == folio_expected_ref_count(folio))
                        lruvec_add_folio_tail(lruvec, folio);
-               else
+               else if (seq == -1UL || !lru_gen_add_rejected_folio(lruvec, folio, seq))
                        lruvec_add_folio(lruvec, folio);
 
                nr_pages = folio_nr_pages(folio);

@@ -2147,7 +2148,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
        nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
                                         lruvec_memcg(lruvec));
 
-       move_folios_to_lru(&folio_list, true);
+       move_folios_to_lru(&folio_list, -1UL, true);
 
        mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                        stat.nr_demoted);

@@ -2258,8 +2259,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
        /*
         * Move folios back to the lru list.
         */
-       nr_activate = move_folios_to_lru(&l_active, false);
-       nr_deactivate = move_folios_to_lru(&l_inactive, false);
+       nr_activate = move_folios_to_lru(&l_active, -1UL, false);
+       nr_deactivate = move_folios_to_lru(&l_inactive, -1UL, false);
 
        count_vm_events(PGDEACTIVATE, nr_deactivate);
        count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);

@@ -4910,7 +4911,8 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
 
 static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                       struct scan_control *sc, int type, int tier,
-                      struct list_head *list, int *isolatedp)
+                      struct list_head *list, int *isolatedp,
+                      unsigned long *scanned_seq)
 {
        enum node_stat_item item;
        int zone_idx;

@@ -4941,6 +4943,7 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                seq = lrugen->max_seq;
 
        seq = lruvec_populated_min_seq(lruvec, type, sc->reclaim_idx + 1, seq);
+       *scanned_seq = seq;
        gen = lru_gen_from_seq(seq);
        max_gen = lru_gen_from_seq(max_seq);
 

@@ -5135,6 +5138,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
        int tier, scanned, reclaimed;
        int isolated = 0, nr_isolated = 0;
        unsigned long total_reclaimed = 0;
+       unsigned long scanned_seq;
        bool skip_retry = false;
        struct mem_cgroup *memcg = lruvec_memcg(lruvec);
        struct pglist_data *pgdat = lruvec_pgdat(lruvec);

@@ -5146,7 +5150,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 
        tier = get_tier_idx(lruvec, type);
        scanned = scan_folios(nr_to_scan, lruvec, sc,
-                             type, tier, &list, &isolated);
+                             type, tier, &list, &isolated, &scanned_seq);
        nr_isolated = isolated;
        if (nr_isolated)
                __mod_node_page_state(pgdat, NR_ISOLATED_ANON + type,

@@ -5172,8 +5176,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                        type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
        list_for_each_entry_safe_reverse(folio, next, &list, lru) {
-               DEFINE_MIN_SEQ(lruvec);
-
                /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
                if (!folio_evictable(folio))
                        continue;

@@ -5192,11 +5194,9 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                 * so this effectively promotes them by at least one generation.
                 */
                folio_set_lru_refs(folio, 0);
-               if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
-                       folio_set_active(folio);
        }
 
-       move_folios_to_lru(&list, false);
+       move_folios_to_lru(&list, scanned_seq, false);
 
        walk = current->reclaim_state->mm_walk;
        if (walk && walk->batched) {
--

2.34.1