mm/mglru: speed up inc_min_seq() and fix cold/hot inversions

This is an aging speedup series split out from the MGLRU swappiness series [1], with the inc_min_seq changes separated to make them easier to review.

Currently, inc_min_seq() has two primary issues that affect both performance and folio hotness assessment:

1. It processes each folio one by one, while many operations can be batched or skipped. For example, a batch of folios can be moved together from the oldest generation to the second-oldest generation, and the associated counting can also be done in batches.

2. It may cause potential cold/hot inversion by placing promoted folios (which have been scanned and found to have young PTEs) behind non-promoted folios. A similar inversion can also occur among non-promoted folios, as tail folios from the oldest generation are placed before head folios when moving them to the second-oldest generation.

This series tries to batch operations as much as possible and fix the potential cold/hot inversion by keeping promoted folios ahead of non-promoted folios, while also preserving the order of non-promoted folios when moving them from the oldest generation to the second-oldest generation.

Minor issue: inc_min_seq() also counts protected folios improperly, as promoted folios should be skipped, as in sort_folio().

We need a stable workload with a stable number of folios to measure aging and evaluate the speedup in inc_min_seq(). So I asked ChatGPT to generate the microbenchmark below. It ages an LRU vec containing 512 MB of memory 100 times:

#define _GNU_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <sys/mman.h>

#define SIZE (512UL * 1024 * 1024)
#define LRU_GEN "/sys/kernel/debug/lru_gen"
#define TARGET_CGROUP "/system.slice/agetest.scope"
#define START_GEN 3
#define END_GEN 103

static long long nsec_diff(const struct timespec *start,
const struct timespec *end)
{
return (end->tv_sec - start->tv_sec) * 1000000000LL +
(end->tv_nsec - start->tv_nsec);
}

static int find_memcg_id(void)
{
FILE *fp;
char line[4096];
int memcg_id;

fp = fopen(LRU_GEN, "r");
if (!fp) {
perror("fopen lru_gen");
return -1;
}

while (fgets(line, sizeof(line), fp)) {
char *p;

if (strncmp(line, "memcg ", 6))
continue;

p = line + 6;

if (sscanf(p, "%d", &memcg_id) != 1)
continue;

/*
* The memcg path follows the numeric ID. */
p = strchr(p, ' ');
if (!p)
continue;

if (strstr(p, TARGET_CGROUP)) {
fclose(fp);
return memcg_id;
}
}

fclose(fp);

fprintf(stderr, "Cannot find %s\n", TARGET_CGROUP); return -1;
}

int main(void)
{
void *addr;
int memcg_id;
int fd;
long long total_ns = 0;

/*
* mmap 512 MB and touch every page.
*/
addr = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (addr == MAP_FAILED) {
perror("mmap");
return 1;
}

memset(addr, 0x55, SIZE);

printf("mmap: %p, size: %lu MB\n",
addr, SIZE / 1024 / 1024);

/*
* Find the memcg ID automatically.
*/
memcg_id = find_memcg_id();
if (memcg_id < 0)
return 1;

printf("memcg: %d (%s)\n", memcg_id, TARGET_CGROUP); printf("aging generation %d -> %d\n",
START_GEN, END_GEN);

fd = open(LRU_GEN, O_WRONLY);
if (fd < 0) {
perror("open lru_gen");
return 1;
}

for (int gen = START_GEN; gen <= END_GEN; gen++) {
char buf[128];
int len;
struct timespec start, end;
long long ns;

len = snprintf(buf, sizeof(buf),
"+ %d 0 %d\n", memcg_id, gen);

clock_gettime(CLOCK_MONOTONIC, &start);

if (write(fd, buf, len) != len) {
perror("write lru_gen");
close(fd);
return 1;
}

clock_gettime(CLOCK_MONOTONIC, &end);

ns = nsec_diff(&start, &end);
total_ns += ns;

printf("gen %3d: %8.3f ms\n",
gen, ns / 1000000.0);
fflush(stdout);
}

close(fd);

printf("\nTotal: %.3f ms\n",
total_ns / 1000000.0);
printf("Average: %.3f ms\n",
total_ns / (double)(END_GEN - START_GEN + 1) / 1000000.0);

while (1)
sleep(1);

return 0;
}

Run the above microbenchmark with:
systemd-run --scope --unit=agetest -p MemoryMax=1024M ./agetest

I’m seeing a significant speedup in inc_min_seq() on my x86 PC:

W/o patch:

Running scope as unit: agetest.scope
mmap: 0x72c1b5a00000, size: 512 MB
memcg: 12673 (/system.slice/agetest.scope)
aging generation 3 -> 103

gen   3:    7.433 ms
gen   4:    0.949 ms
gen   5:    2.535 ms
gen   6:    5.043 ms
gen   7:    5.041 ms
gen   8:    5.027 ms
...
gen 100:    5.035 ms
gen 101:    5.011 ms
gen 102:    5.029 ms
gen 103:    5.056 ms

Total:   503.946 ms
Average: 4.990 ms

W/ patch:

Running scope as unit: agetest.scope
mmap: 0x775ec5200000, size: 512 MB
memcg: 12717 (/system.slice/agetest.scope)
aging generation 3 -> 103

gen   3:    7.558 ms
gen   4:    0.916 ms
gen   5:    2.277 ms
gen   6:    2.328 ms
…
gen 100:    2.303 ms
gen 101:    2.297 ms
gen 102:    2.305 ms
gen 103:    2.312 ms

Total:   236.635 ms
Average: 2.343 ms

The average aging time drops from 4.990 ms to 2.343 ms!

Thanks, Xueyuan, for testing this on ARM[2]. It actually shows an even larger improvement.
Xueyuan tested this series on his arm64 machine (24 cores, 4K base pages) and reproduced the improvement:

THP=never (PTE):
baseline: 7.644 ms
patched: 2.964 ms (-61.2%)

THP=always (PMD):
baseline: 0.0373 ms
patched: 0.0292 ms (-21.8%)

[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/

-v3:
* Collect tags from Ridong, Lian and Kunwu, thanks!
* Add some comments as Ridong's suggestion in "enhance cold/hot inversion handling", thanks!

-v2:
* Collect tags from Kairui, Baoquan, and Xueyuan, thanks!
* Split __folio_inc_gen() into a separate patch and add VM_WARN_ON checks for the generation and active state, per Baoquan and Kairui, thanks!
* Make delta a long instead of unsigned long, per Lian Wang, thanks!
* Fix checkpatch issues by converting macros to static inline functions, per Kairui, thanks!
* Drop the batched protected update since it doesn't show a significant performance improvement, following Baoquan's comments.
https://lore.kernel.org/linux-mm/[email protected]/

-v1:
https://lore.kernel.org/linux-mm/[email protected]/

Barry Song (Xiaomi) (7):
mm/mglru: separate folio generation update from LRU accounting mm/mglru: batch update lrugen->nr_pages in inc_min_seq() mm/mglru: enhance cold/hot inversion handling in inc_min_seq() mm/mglru: exclude folios promoted by aging from protected in

    inc_min_seq()

mm/mglru: make LRU folio prefetch helper an inline function mm/mglru: move folios from oldest gen to second-oldest gen from head
to tail
mm/mglru: batch move folios to the second-oldest gen's LRU

 mm/vmscan.c | 136 ++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 106 insertions(+), 30 deletions(-)

2.34.1

folio_inc_gen() currently updates both the folio's generation and the LRU size accounting. This makes it difficult to batch the LRU size updates when moving multiple folios.

Extract the generation update into __folio_inc_gen(), which only updates the folio's generation and reports whether the generation was actually increased. Keep folio_inc_gen() as the wrapper that performs the LRU size accounting when needed.

This separates the per-folio generation update from LRU accounting and allows the latter to be batched by subsequent changes.

---
 mm/vmscan.c | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

index f11491ee9ed5..54bce2f608ef 100644
@@ -3295,21 +3295,21 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma
 	return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
 }
 
-/* protect pages accessed multiple times through file descriptors */
-static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
 {
-	int type = folio_is_file_lru(folio);
-	struct lru_gen_folio *lrugen = &lruvec->lrugen;
-	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
 	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
+	int new_gen;
 
 	VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
 
 	do {
 		new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
 		/* folio_update_gen() has promoted this page? */
-		if (new_gen >= 0 && new_gen != old_gen)
+		if (new_gen >= 0 && new_gen != old_gen) {
+			if (increased)
+				*increased = false;
 			return new_gen;
+		}
 
 		new_gen = (old_gen + 1) % MAX_NR_GENS;
 

@@ -3317,8 +3317,22 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
 		new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
 	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
 
-	lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+	if (increased)
+		*increased = true;
+	return new_gen;
+}
+
+/* protect pages accessed multiple times through file descriptors */
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+{
+	int type = folio_is_file_lru(folio);
+	struct lru_gen_folio *lrugen = &lruvec->lrugen;
+	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+	bool gen_increased;
 
+	new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
+	if (gen_increased)
+		lru_gen_update_size(lruvec, folio, old_gen, new_gen);
 	return new_gen;
 }
 

2.34.1

LGTM.

Currently, folio_inc_gen() updates lrugen->nr_pages for every folio as it advances generations. Instead, accumulate the size changes and update lrugen->nr_pages in a batch after scanning the entire oldest generation, or when the scan stops because remaining reaches zero.

Since we only move folios from the oldest generation to the second oldest generation, the active/inactive state cannot change. We can therefore skip __lru_update_size().

---
 mm/vmscan.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

index 54bce2f608ef..27494505cecc 100644
@@ -3918,6 +3918,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	int hist = lru_hist_from_seq(lrugen->min_seq[type]);
 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+	int target_gen = (old_gen + 1) % MAX_NR_GENS;
 
 	/* For file type, skip the check if swappiness is anon only */
 	if (type && (swappiness == SWAPPINESS_ANON_ONLY))

@@ -3927,35 +3928,47 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	if (!type && !swappiness)
 		goto done;
 
+	VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS);
+	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) !=
+			lru_gen_is_active(lruvec, target_gen));
 	/* prevent cold/hot inversion if the type is evictable */
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 		struct list_head *head = &lrugen->folios[old_gen][type][zone];
+		long delta = 0;
 
 		while (!list_empty(head)) {
 			struct folio *folio = lru_to_folio(head);
+			long nr_pages = folio_nr_pages(folio);
 			int refs = folio_lru_refs(folio);
 			bool workingset = folio_test_workingset(folio);
+			bool gen_increased;
 
 			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
-			new_gen = folio_inc_gen(lruvec, folio);
+			new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
 			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
-
+			if (gen_increased)
+				delta += nr_pages;
 			/* don't count the workingset being lazily promoted */
 			if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
 				int tier = lru_tier_from_refs(refs, workingset);
-				int delta = folio_nr_pages(folio);
 
 				WRITE_ONCE(lrugen->protected[hist][type][tier],
-					   lrugen->protected[hist][type][tier] + delta);
+					   lrugen->protected[hist][type][tier] + nr_pages);
 			}
 
 			if (!--remaining)
-				return false;
+				break;
 		}
+		WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
+			   lrugen->nr_pages[old_gen][type][zone] - delta);
+		WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
+			   lrugen->nr_pages[target_gen][type][zone] + delta);
+		if (!remaining)
+			return false;
 	}
 done:
 	reset_ctrl_pos(lruvec, type, true);

2.34.1

Make sense to me.

LGTM. One nit follows.

Thanks very much!

We have already said this twice—once in the changelog and once with the VM_WARN_ON() in C :-)

+ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) !=
+ lru_gen_is_active(lruvec, target_gen));

I guess we don't need to say it a third time, do we? It should be self-documenting.

For a possible future change, I know Kairui may change the definition of active/inactive—for example, by no longer using generations to define them, but instead using refs. Kairui knows the MGLRU code inside out, though, so I'm not too concerned. :-)

Best Regards
Barry

During aging, a folio's generation may already have been updated by folio_update_gen(), even though it has not yet been moved to the corresponding generation list. Such folios are hotter than those already in that generation.

It makes sense for inc_min_seq() to increment the generation of folios that were never promoted during aging and move them to the tail of the new oldest generation. However, folios that were already promoted should instead be moved to the head of their updated generation, just as sort_folio() does in scan_folios().

Otherwise, promoted folios could end up behind folios that were never promoted, effectively inverting their hot/cold ordering.

---
 mm/vmscan.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

index 27494505cecc..95d48f9efc09 100644
@@ -3949,9 +3949,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
 			new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
-			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
-			if (gen_increased)
+			/*
+			 * If gen_increased is false, this is a promotion. Put folios
+			 * at the head of the promoted gen. Otherwise, put them at
+			 * the tail of the second-oldest gen.
+			 */
+			if (gen_increased) {
 				delta += nr_pages;
+				list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
+			} else {
+				list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]);
+			}
 			/* don't count the workingset being lazily promoted */
 			if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
 				int tier = lru_tier_from_refs(refs, workingset);

2.34.1

Good catch. Make sense to me.

Some folios may have been promoted during aging, so don't count them as protected, similar to sort_folio().

---
 mm/vmscan.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

index 95d48f9efc09..ca57f6095b37 100644
@@ -3957,17 +3957,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			if (gen_increased) {
 				delta += nr_pages;
 				list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
+
+				/* don't count the workingset being lazily promoted */
+				if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
+					int tier = lru_tier_from_refs(refs, workingset);
+
+					WRITE_ONCE(lrugen->protected[hist][type][tier],
+						   lrugen->protected[hist][type][tier] + nr_pages);
+				}
 			} else {
 				list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]);
 			}
-			/* don't count the workingset being lazily promoted */
-			if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
-				int tier = lru_tier_from_refs(refs, workingset);
-
-				WRITE_ONCE(lrugen->protected[hist][type][tier],
-					   lrugen->protected[hist][type][tier] + nr_pages);
-			}
-
 			if (!--remaining)
 				break;
 		}

2.34.1

prefetchw_prev_lru_folio() is currently implemented as a macro with a potentially unused argument. This makes the helper harder to read and can also trigger checkpatch warnings about unused macro arguments.

Make it a static inline function and remove the unnecessary _field argument. The helper always prefetches the previous folio's flags, so there is no need to make the field configurable.

---
 mm/vmscan.c | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

index ca57f6095b37..1907a946840d 100644
@@ -182,17 +182,21 @@ struct scan_control {
 };
 
 #ifdef ARCH_HAS_PREFETCHW
-#define prefetchw_prev_lru_folio(_folio, _base, _field)			\
-	do {								\
-		if ((_folio)->lru.prev != _base) {			\
-			struct folio *prev;				\
-									\
-			prev = lru_to_folio(&(_folio->lru));		\
-			prefetchw(&prev->_field);			\
-		}							\
-	} while (0)
+static inline void prefetchw_prev_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+	if (folio->lru.prev != base) {
+		struct folio *prev;
+
+		prev = lru_to_folio(&folio->lru);
+		prefetchw(&prev->flags);
+	}
+}
 #else
-#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
+static inline void prefetchw_prev_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+}
 #endif
 
 /*

@@ -1695,7 +1699,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
 		struct folio *folio;
 
 		folio = lru_to_folio(src);
-		prefetchw_prev_lru_folio(folio, src, flags);
+		prefetchw_prev_lru_folio(folio, src);
 
 		nr_pages = folio_nr_pages(folio);
 		total_scan += nr_pages;

2.34.1

LGTM.

For reclamation, it makes sense to reclaim folios from tail to head, as folios near the head are relatively hot. However, when moving folios from the oldest generation to the second-oldest generation, using the tail-to-head order would effectively cause a cold/hot inversion.

---
 mm/vmscan.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

index 1907a946840d..76dfa9575852 100644
@@ -192,11 +192,27 @@ static inline void prefetchw_prev_lru_folio(struct folio *folio,
 		prefetchw(&prev->flags);
 	}
 }
+
+static inline void prefetchw_next_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+	if (folio->lru.next != base) {
+		struct folio *next;
+
+		next = list_entry(folio->lru.next, struct folio, lru);
+		prefetchw(&next->flags);
+	}
+}
 #else
 static inline void prefetchw_prev_lru_folio(struct folio *folio,
 		struct list_head *base)
 {
 }
+
+static inline void prefetchw_next_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+}
 #endif
 
 /*

@@ -3938,10 +3954,11 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	/* prevent cold/hot inversion if the type is evictable */
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 		struct list_head *head = &lrugen->folios[old_gen][type][zone];
+		struct list_head *pos = head->next;
 		long delta = 0;
 
-		while (!list_empty(head)) {
-			struct folio *folio = lru_to_folio(head);
+		while (pos != head) {
+			struct folio *folio = list_entry(pos, struct folio, lru);
 			long nr_pages = folio_nr_pages(folio);
 			int refs = folio_lru_refs(folio);
 			bool workingset = folio_test_workingset(folio);

@@ -3952,6 +3969,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
+			prefetchw_next_lru_folio(folio, head);
+			pos = pos->next;
 			new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
 			/*
 			 * If gen_increased is false, this is a promotion. Put folios

2.34.1

Hi Baolin,

Thanks for your review.

I had this discussion with Kairui and thought it might be useful to keep it here:

https://lore.kernel.org/linux-mm/CAMgjq7AJnWGGH=FNuV7ynXjbffktsDRgdcyagLUJUwJf1ukaJQ@mail.gmail.com/

It might be arch-dependent. Some architectures could benefit significantly from prefetching, while others might see little to no impact.

for my x86 test, it has very slight improvement:

***** no-prefetch:

agetest:
...

gen 100:    2.421 ms
gen 101:    2.424 ms
gen 102:    2.420 ms
gen 103:    2.413 ms

Total:   248.418 ms
Average: 2.460 ms

agetest:
...

gen 100:    2.393 ms
gen 101:    2.396 ms
gen 102:    2.395 ms
gen 103:    2.392 ms

Total:   245.627 ms
Average: 2.432 ms

agetest:
...

gen 100:    2.433 ms
gen 101:    2.427 ms
gen 102:    2.432 ms
gen 103:    2.450 ms

Total:   249.186 ms
Average: 2.467 ms

**** has-prefetch:

agetest:
....

gen 100:    2.314 ms
gen 101:    2.310 ms
gen 102:    2.321 ms
gen 103:    2.303 ms

Total:   237.619 ms
Average: 2.353 ms

agetest:

gen 100:    2.342 ms
gen 101:    2.343 ms
gen 102:    2.339 ms
gen 103:    2.335 ms

Total:   239.929 ms
Average: 2.376 ms

agetest:

gen 100:    2.344 ms
gen 101:    2.347 ms
gen 102:    2.352 ms
gen 103:    2.348 ms

Total:   241.188 ms
Average: 2.388 ms

Basically, it’s 2.3xx vs. 2.4xx, lower is better.

Best Regards
Barry

Could you add this info to the commit message? (IIUC, someone tried to remove the prefetch in MM, since they found that prefetch doesn't seem to help much on modern CPUs.)

On Fri, Sep 4, 2026 at 10:47 AM Baolin Wang <[email protected]> wrote:
[...]

Sure. I guess we could ask Andrew for his kind help in adding the following to the Changelog of this patch.

Hi Andrew,

Could you please update the changelog and add the description below at the end? Hopefully, this would be easier for you than handling a new version. If you would prefer a new version instead, please let me know.

The impact of the added prefetching might be arch-dependent. Some architectures could benefit more from prefetching, while others might see little to no impact. In my x86 test, it shows a very slight improvement:

***** no-prefetch:

agetest:
...
gen 100:    2.421 ms
gen 101:    2.424 ms
gen 102:    2.420 ms
gen 103:    2.413 ms

Total:   248.418 ms
Average: 2.460 ms

agetest:
...
gen 100:    2.393 ms
gen 101:    2.396 ms
gen 102:    2.395 ms
gen 103:    2.392 ms

Total:   245.627 ms
Average: 2.432 ms

agetest:
...
gen 100:    2.433 ms
gen 101:    2.427 ms
gen 102:    2.432 ms
gen 103:    2.450 ms

Total:   249.186 ms
Average: 2.467 ms

**** has-prefetch:

agetest:
....
gen 100:    2.314 ms
gen 101:    2.310 ms
gen 102:    2.321 ms
gen 103:    2.303 ms

Total:   237.619 ms
Average: 2.353 ms

agetest:
gen 100:    2.342 ms
gen 101:    2.343 ms
gen 102:    2.339 ms
gen 103:    2.335 ms

Total:   239.929 ms
Average: 2.376 ms

agetest:

gen 100:    2.344 ms
gen 101:    2.347 ms
gen 102:    2.352 ms
gen 103:    2.348 ms

Total:   241.188 ms
Average: 2.388 ms

Basically, it’s 2.3xx vs. 2.4xx, lower is better.

No probs.

Detect folios that need to move from the oldest generation to the second-oldest generation, and batch-move them together. This can significantly reduce the sys time of inc_min_seq(), especially when the other type is significantly behind the preferred type.

---
 mm/vmscan.c | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

index 76dfa9575852..79defbb44c6e 100644
@@ -3931,6 +3931,19 @@ static void clear_mm_walk(void)
 		kfree(walk);
 }
 
+static inline void flush_lru_batch(struct list_head *head, struct list_head **batch_end,
+				   struct list_head *dst)
+{
+	LIST_HEAD(movable);
+
+	if (!*batch_end)
+		return;
+
+	list_cut_position(&movable, head, *batch_end);
+	list_splice_tail_init(&movable, dst);
+	*batch_end = NULL;
+}
+
 static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 {
 	int zone;

@@ -3953,8 +3966,10 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			lru_gen_is_active(lruvec, target_gen));
 	/* prevent cold/hot inversion if the type is evictable */
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+		struct list_head *target_list = &lrugen->folios[target_gen][type][zone];
 		struct list_head *head = &lrugen->folios[old_gen][type][zone];
 		struct list_head *pos = head->next;
+		struct list_head *batch_end = NULL;
 		long delta = 0;
 
 		while (pos != head) {

@@ -3979,7 +3994,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			 */
 			if (gen_increased) {
 				delta += nr_pages;
-				list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
+				batch_end = &folio->lru;
 
 				/* don't count the workingset being lazily promoted */
 				if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {

@@ -3989,11 +4004,14 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 						   lrugen->protected[hist][type][tier] + nr_pages);
 				}
 			} else {
+				flush_lru_batch(head, &batch_end, target_list);
 				list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]);
 			}
 			if (!--remaining)
 				break;
 		}
+		flush_lru_batch(head, &batch_end, target_list);
+
 		WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
 			   lrugen->nr_pages[old_gen][type][zone] - delta);
 		WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],

2.34.1

I have the same taste as Baoquan[1]. :) Anyway, it's up to you.

[1] https://lore.kernel.org/linux-mm/ao6zFSzx6mOl874A@fedora/#t