Skip to contentSkip to conversation

mm: vmscan: retry folios written back while isolated for traditional LRU

PATCH Superseded Ridong Chen 2 participants 5 messages Sep 13 – Sep 16

Subsystems (3)

v8 of 9 Superseded Latest is v9 →

lore
Share & tools
lore
#0 lore raw

PATCH v8 1/1 mm: vmscan: retry folios written back while isolated for traditional LRU

From: Ridong Chen <[email protected]>

As commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") mentioned:

The page reclaim isolates a batch of folios from the tail of one of the LRU lists and works on those folios one by one. For a suitable swap-backed folio, if the swap device is async, it queues that folio for writeback. After the page reclaim finishes an entire batch, it puts back the folios it queued for writeback to the head of the original LRU list.

In the meantime, the page writeback flushes the queued folios also by batches. Its batching logic is independent from that of the page reclaim. For each of the folios it writes back, the page writeback calls folio_rotate_reclaimable() which tries to rotate a folio to the tail.

folio_rotate_reclaimable() only works for a folio after the page reclaim has put it back. If an async swap device is fast enough, the page writeback can finish with that folio while the page reclaim is still working on the rest of the batch containing it. In this case, that folio will remain at the head and the page reclaim will not retry it before reaching there".

The commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") only fixed the issue for mglru. However, this issue also exists in the traditional active/inactive LRU and was found at [1].

It can be reproduced with below steps:

1. Compile with CONFIG_TRANSPARENT_HUGEPAGE=y
2. Mount memcg v1, and create memcg named test_memcg and set limit_in_bytes=1G, memsw.limit_in_bytes=2G.
3. Create a 1G swap file, and allocate 1.35G anon memory in test_memcg.

It was found that:

cat memory.usage_in_bytes
1073700864
cat memory.memsw.usage_in_bytes
1413124096

free -h

                total        used        free
  Mem:           1.6Gi       1.2Gi       299Mi
  Swap:          1.0Gi       678Mi       346Mi

As shown above, the test_memcg charged about 324M swap (memsw.usage minus usage), but almost 678M swap memory was used, which means that 350M+ may be wasted because other memcgs can not use these swap memory.

This issue should be fixed in the same way as mglru. Therefore, the common logic was extracted to the 'find_folios_written_back' function firstly, which is then reused in the 'shrink_inactive_list' function. Finally, retry reclaiming those folios that may have missed the rotation for traditional LRU.

After change, the same test case only wasted about 2M swap. The swap device usage matches what the memcg actually charged.

cat memory.usage_in_bytes
1070301184
cat memory.memsw.usage_in_bytes
1412448256

free -h

                total        used        free
  Mem:           1.6Gi       1.2Gi       299Mi
  Swap:          1.0Gi       327Mi       696Mi

[1] https://lore.kernel.org/linux-kernel/[email protected]/
[2] https://lore.kernel.org/linux-kernel/CAGsJ_4zqL8ZHNRZ44o_CC69kE7DBVXvbZfvmQxMGiFqRxqHQdA@mail.gmail.com/

---
v8: Rebase to the -next tree, adapt to the current reclaim API, and retest.

[v7]: https://lore.kernel.org/linux-mm/[email protected]/

 mm/vmscan.c | 104 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 73 insertions(+), 31 deletions(-)
mm/vmscan.c +73 -31
index d1495a7d469d..6e13e570aab8 100644
@@ -181,6 +181,10 @@ struct scan_control {
 	struct reclaim_state reclaim_state;
 };
 
+static void find_folios_written_back(struct list_head *list,
+				     struct list_head *clean, struct lruvec *lruvec,
+				     int type, bool skip_retry);
+
 #ifdef ARCH_HAS_PREFETCHW
 static inline void prefetchw_prev_lru_folio(struct folio *folio,
 		struct list_head *base)
@@ -2013,14 +2017,16 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
 		enum lru_list lru)
 {
 	LIST_HEAD(folio_list);
+	LIST_HEAD(clean_list);
 	unsigned long nr_scanned;
-	unsigned int nr_reclaimed = 0;
-	unsigned long nr_taken;
+	unsigned int nr_reclaimed, total_reclaimed = 0;
+	unsigned long nr_taken, isolated;
 	struct reclaim_stat stat;
 	bool file = is_file_lru(lru);
 	enum node_stat_item item;
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 	bool stalled = false;
+	bool skip_retry = false;
 
 	while (unlikely(too_many_isolated(pgdat, file, sc))) {
 		if (stalled)
@@ -2052,25 +2058,42 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
 	if (nr_taken == 0)
 		return 0;
 
+	isolated = nr_taken;
+retry:
 	nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
 					 lruvec_memcg(lruvec));
+	total_reclaimed += nr_reclaimed;
+
+	/* Retry pass is only meant for clean folios without new isolation */
+	if (isolated)
+		handle_reclaim_writeback(isolated, pgdat, sc, &stat);
+	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
+			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
+
+	find_folios_written_back(&folio_list, &clean_list, lruvec, file, skip_retry);
 
 	move_folios_to_lru(&folio_list);
 
 	mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
 					stat.nr_demoted);
-	mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
 	item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
 	mod_lruvec_state(lruvec, item, nr_reclaimed);
 	mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
-	if (nr_scanned > nr_reclaimed)
+
+	if (!list_empty(&clean_list)) {
+		list_splice_init(&clean_list, &folio_list);
+		skip_retry = true;
+		/* Retry folios were already isolated and accounted above */
+		isolated = 0;
+		goto retry;
+	}
+
+	mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
+	if (nr_scanned > total_reclaimed)
 		mod_lruvec_state(lruvec, PGROTATE_ANON + file,
-				 nr_scanned - nr_reclaimed);
+				 nr_scanned - total_reclaimed);
 
-	handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
-	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
-	return nr_reclaimed;
+	return total_reclaimed;
 }
 
 /*
@@ -5006,8 +5029,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 {
 	LIST_HEAD(list);
 	LIST_HEAD(clean);
-	struct folio *folio;
-	struct folio *next;
 	enum node_stat_item item;
 	struct reclaim_stat stat;
 	struct lru_gen_mm_walk *walk;
@@ -5046,26 +5067,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			type_scanned, reclaimed, &stat, sc->priority,
 			type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
-	list_for_each_entry_safe_reverse(folio, next, &list, lru) {
-		DEFINE_MIN_SEQ(lruvec);
-
-		/* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
-		if (!folio_evictable(folio))
-			continue;
-
-		/* retry folios that may have missed folio_rotate_reclaimable() */
-		if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
-		    !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
-			list_move(&folio->lru, &clean);
-			continue;
-		}
-
-		/* don't add rejected folios to the oldest generation */
-		if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
-			folio_set_lru_refs(folio, 0);
-			folio_set_active(folio);
-		}
-	}
+	find_folios_written_back(&list, &clean, lruvec, type, skip_retry);
 
 	move_folios_to_lru(&list);
 
@@ -6108,6 +6110,46 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
 
 #endif /* CONFIG_LRU_GEN */
 
+/**
+ * find_folios_written_back - Find and move the written back folios to a new list.
+ * @list: folios list
+ * @clean: the written back folios list
+ * @lruvec: the lruvec
+ * @type: LRU type (only used for CONFIG_LRU_GEN)
+ * @skip_retry: whether skip retry.
+ */
+static void find_folios_written_back(struct list_head *list,
+				     struct list_head *clean, struct lruvec *lruvec,
+				     int type, bool skip_retry)
+{
+	struct folio *folio;
+	struct folio *next;
+
+	list_for_each_entry_safe_reverse(folio, next, list, lru) {
+#ifdef CONFIG_LRU_GEN
+		DEFINE_MIN_SEQ(lruvec);
+#endif
+		/* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+		if (!folio_evictable(folio))
+			continue;
+
+		/* retry folios that may have missed folio_rotate_reclaimable() */
+		if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
+		    !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
+			list_move(&folio->lru, clean);
+			continue;
+		}
+#ifdef CONFIG_LRU_GEN
+		/* don't add rejected folios to the oldest generation */
+		if (lruvec->lrugen.enabled &&
+		    lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+			folio_set_lru_refs(folio, 0);
+			folio_set_active(folio);
+		}
+#endif
+	}
+}
+
 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
 {
 	unsigned long nr[NR_LRU_LISTS];
Signature · 1 line
2.34.1
#1 lore raw
Quoted 27 lines · Ridong Chen

From: Ridong Chen <[email protected]>

As commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") mentioned:

The page reclaim isolates a batch of folios from the tail of one of the LRU lists and works on those folios one by one. For a suitable swap-backed folio, if the swap device is async, it queues that folio for writeback. After the page reclaim finishes an entire batch, it puts back the folios it queued for writeback to the head of the original LRU list.

In the meantime, the page writeback flushes the queued folios also by batches. Its batching logic is independent from that of the page reclaim. For each of the folios it writes back, the page writeback calls folio_rotate_reclaimable() which tries to rotate a folio to the tail.

folio_rotate_reclaimable() only works for a folio after the page reclaim has put it back. If an async swap device is fast enough, the page writeback can finish with that folio while the page reclaim is still working on the rest of the batch containing it. In this case, that folio will remain at the head and the page reclaim will not retry it before reaching there".

The commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") only fixed the issue for mglru. However, this issue also exists in the traditional active/inactive LRU and was found at [1].

Quoted 7 lines

It can be reproduced with below steps:

1. Compile with CONFIG_TRANSPARENT_HUGEPAGE=y
2. Mount memcg v1, and create memcg named test_memcg and set

    limit_in_bytes=1G, memsw.limit_in_bytes=2G.

3. Create a 1G swap file, and allocate 1.35G anon memory in test_memcg.

Hi all,

I am raising this issue again. It has been a long time since the last version [1].

It was suspected that Kirill's "[PATCH 0/8] mm: Remove PG_reclaim" would solve this issue, but the issue remains.

I am providing the reproducer(offered by Xuedong Zhao) in the hope that it will help fix this issue.

memcg_malloc.c:

```
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define ONE_GB (1024 * 1024 * 1024)
#define SIXTY_FOUR_MB (64 * 1024 * 1024)

/* non-zero fill: zero pages can be deduped/never written to swap, which hides
 * the "written-back-while-isolated" leak. Use a real byte pattern. */
#define FILL_BYTE 0xAB

void allocate_memory(size_t size_in_bytes) {
    size_t total_allocated = 0;
    char *memory;

    while (total_allocated + ONE_GB <= size_in_bytes) {
        memory = (char *)malloc(ONE_GB);
        if (memory == NULL) {
            perror("malloc");
            exit(EXIT_FAILURE);
        }
        memset(memory, FILL_BYTE, ONE_GB);
        total_allocated += ONE_GB;
        printf("Allocated %zu GB\n", total_allocated / ONE_GB);
        sleep(1);
    }

    while (total_allocated + SIXTY_FOUR_MB <= size_in_bytes) {
        memory = (char *)malloc(SIXTY_FOUR_MB);
        if (memory == NULL) {
            perror("malloc");
            exit(EXIT_FAILURE);
        }
        memset(memory, FILL_BYTE, SIXTY_FOUR_MB);
        total_allocated += SIXTY_FOUR_MB;
        printf("Allocated %zu MB\n", total_allocated / (1024 * 1024));
        sleep(1);
    }

    size_t remaining = size_in_bytes - total_allocated;
    if (remaining > 0) {
        memory = (char *)malloc(remaining);
        if (memory == NULL) {
            perror("malloc");
            exit(EXIT_FAILURE);
        }
        memset(memory, FILL_BYTE, remaining);
        total_allocated += remaining;
        printf("Allocated remaining %zu bytes\n", remaining);
        sleep(1);
    }

    printf("Total allocated: %zu bytes\n", total_allocated);
}

int main(int argc, char *argv[]) {
    if (argc != 2) {
        fprintf(stderr, "Usage: %s <size_in_gb>\n", argv[0]);
        return EXIT_FAILURE;
    }

    double size_in_gb = atof(argv[1]);
    if (size_in_gb <= 0) {
        fprintf(stderr, "Invalid size: %s\n", argv[1]);
        return EXIT_FAILURE;
    }

    size_t size_in_bytes = (size_t)(size_in_gb * ONE_GB);
    allocate_memory(size_in_bytes);

    sleep(3600);
    return EXIT_SUCCESS;
}
```

test.sh:

```
#!/bin/bash
set -e

# Variables
MEMCG_NAME="test_memcg"
MEM_LIMIT="1G"
MEMSW_LIMIT="2G"
PROGRAM_PATH="./memcg_malloc"
PROGRAM_ARGS="1.35"

# ---- swapfile setup --------------------------------------------------------
SWAPFILE="/swapfile"
SWAP_SIZE="1G"          # size of the swap device backing the test

setup_swap() {
    # already have swap on? then nothing to do
    if [ "$(swapon --show --noheadings | wc -l)" -gt 0 ]; then
        echo "swap already active:"; swapon --show
        return
    fi

    if [ ! -f "$SWAPFILE" ]; then
        echo "Creating ${SWAP_SIZE} swapfile at ${SWAPFILE}"
        # fallocate is fast; fall back to dd if the fs doesn't support it
        fallocate -l "$SWAP_SIZE" "$SWAPFILE" 2>/dev/null || \
            dd if=/dev/zero of="$SWAPFILE" bs=1M count=$((8*1024)) status=progress
        chmod 600 "$SWAPFILE"
        mkswap "$SWAPFILE"
    fi
    swapon "$SWAPFILE"
    echo "swap enabled:"; swapon --show
}

setup_swap

# ---- reclaim preconditions -------------------------------------------------
# This bug is in the *traditional* active/inactive LRU; MGLRU already fixed it
# in commit 359a5e1416ca, so it must be disabled to reproduce.
[ -f /sys/kernel/mm/lru_gen/enabled ] && echo 0 > /sys/kernel/mm/lru_gen/enabled
# Large anon folios make the reclaim/writeback batching race easy to hit.
echo always > /sys/kernel/mm/transparent_hugepage/enabled
echo "lru_gen: $(cat /sys/kernel/mm/lru_gen/enabled 2>/dev/null)  thp: $(cat /sys/kernel/mm/transparent_hugepage/enabled)"

# Create the cgroup slice if it doesn't exist
if ! systemctl list-units --full -all | grep -q "${MEMCG_NAME}.slice"; then
    echo "Creating cgroup slice ${MEMCG_NAME}.slice"
    systemctl set-property --runtime -- ${MEMCG_NAME}.slice MemoryMax=${MEM_LIMIT}
    systemctl set-property --runtime -- ${MEMCG_NAME}.slice 
MemorySwapMax=${MEMSW_LIMIT}
fi

# Start the slice to apply the properties
systemctl start ${MEMCG_NAME}.slice

# Run the program in the cgroup
echo "Running programs in cgroup slice ${MEMCG_NAME}.slice"
systemd-run --unit=${MEMCG_NAME}_proc1 --slice=${MEMCG_NAME}.slice ${PROGRAM_PATH} ${PROGRAM_ARGS} &

# Pause to let reclaim settle under the 1G limit
sleep 60

# ---- measurement (cgroup v1) -----------------------------------------------
echo "########## measurement ##########"

CG=/sys/fs/cgroup/memory/${MEMCG_NAME}.slice/${MEMCG_NAME}_proc1.service

usage=$(cat "$CG/memory.usage_in_bytes" 2>/dev/null || echo 0)
memsw=$(cat "$CG/memory.memsw.usage_in_bytes" 2>/dev/null || echo 0)

# v1: swap charged to the memcg is memsw.usage - usage
swap_charged=$((memsw - usage))

# swap actually consumed on the device: /proc/swaps "Used" column is in KiB
dev_used_kb=$(awk 'NR>1 {sum += $4} END {print sum+0}' /proc/swaps)
dev_used=$((dev_used_kb * 1024))

# the bug wastes swap: slots written back while isolated are charged on the
# device but never reused, so device usage outruns what the cgroup accounts for
waste=$((dev_used - swap_charged))

to_mib() { awk -v b="$1" 'BEGIN { printf "%.0f MiB", b/1024/1024 }'; }

echo "memory.usage_in_bytes        : ${usage} bytes ($(to_mib ${usage}))"
echo "memory.memsw.usage_in_bytes  : ${memsw} bytes ($(to_mib ${memsw}))"
echo "swap charged (memsw - usage) : ${swap_charged} bytes ($(to_mib ${swap_charged}))"
echo "swap device used             : ${dev_used} bytes ($(to_mib ${dev_used}))"
echo "wasted swap (device - cg)    : ${waste} bytes ($(to_mib ${waste}))"
echo "-----"
free -h
echo "--- /proc/swaps ---"; cat /proc/swaps

# Wait for the processes to complete
wait

# Clean up
echo "Cleaning up"
# Reset failed state if the slice is still loaded
if systemctl list-units --full -all | grep -q "${MEMCG_NAME}.slice"; then
    systemctl reset-failed ${MEMCG_NAME}.slice
fi
systemctl stop ${MEMCG_NAME}.slice

echo "Done"

```

Result shown as:

```
...
########## measurement ##########
memory.usage_in_bytes        : 1070014464 bytes (1020 MiB)
memory.memsw.usage_in_bytes  : 1413173248 bytes (1348 MiB)
swap charged (memsw - usage) : 343158784 bytes (327 MiB)
swap device used             : 344248320 bytes (328 MiB)
wasted swap (device - cg)    : 1089536 bytes (1 MiB)
-----
               total        used        free      shared  buff/cache   available
Mem:           1.6Gi       1.2Gi       316Mi       0.0Ki        85Mi       287Mi
Swap:          1.0Gi       328Mi       695Mi
...
```

[1] https://lore.kernel.org/linux-mm/[email protected]/#r

Signature · 2 lines
Best regards
Ridong
reviewer
#2 lore raw
In reply to Ridong Chen
Quoted 151 lines · Ridong Chen

From: Ridong Chen <[email protected]>

As commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") mentioned:

The page reclaim isolates a batch of folios from the tail of one of the LRU lists and works on those folios one by one. For a suitable swap-backed folio, if the swap device is async, it queues that folio for writeback. After the page reclaim finishes an entire batch, it puts back the folios it queued for writeback to the head of the original LRU list.

In the meantime, the page writeback flushes the queued folios also by batches. Its batching logic is independent from that of the page reclaim. For each of the folios it writes back, the page writeback calls folio_rotate_reclaimable() which tries to rotate a folio to the tail.

folio_rotate_reclaimable() only works for a folio after the page reclaim has put it back. If an async swap device is fast enough, the page writeback can finish with that folio while the page reclaim is still working on the rest of the batch containing it. In this case, that folio will remain at the head and the page reclaim will not retry it before reaching there".

The commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") only fixed the issue for mglru. However, this issue also exists in the traditional active/inactive LRU and was found at [1].

It can be reproduced with below steps:

1. Compile with CONFIG_TRANSPARENT_HUGEPAGE=y
2. Mount memcg v1, and create memcg named test_memcg and set limit_in_bytes=1G, memsw.limit_in_bytes=2G.
3. Create a 1G swap file, and allocate 1.35G anon memory in test_memcg.

It was found that:

cat memory.usage_in_bytes
1073700864
cat memory.memsw.usage_in_bytes
1413124096

free -h

                total        used        free
  Mem:           1.6Gi       1.2Gi       299Mi
  Swap:          1.0Gi       678Mi       346Mi

As shown above, the test_memcg charged about 324M swap (memsw.usage minus usage), but almost 678M swap memory was used, which means that 350M+ may be wasted because other memcgs can not use these swap memory.

This issue should be fixed in the same way as mglru. Therefore, the common logic was extracted to the 'find_folios_written_back' function firstly, which is then reused in the 'shrink_inactive_list' function. Finally, retry reclaiming those folios that may have missed the rotation for traditional LRU.

After change, the same test case only wasted about 2M swap. The swap device usage matches what the memcg actually charged.

cat memory.usage_in_bytes
1070301184
cat memory.memsw.usage_in_bytes
1412448256

free -h

                total        used        free
  Mem:           1.6Gi       1.2Gi       299Mi
  Swap:          1.0Gi       327Mi       696Mi

[1] https://lore.kernel.org/linux-kernel/[email protected]/
[2] https://lore.kernel.org/linux-kernel/CAGsJ_4zqL8ZHNRZ44o_CC69kE7DBVXvbZfvmQxMGiFqRxqHQdA@mail.gmail.com/

---
v8: Rebase to the -next tree, adapt to the current reclaim API, and retest.

[v7]: https://lore.kernel.org/linux-mm/[email protected]/

 mm/vmscan.c | 104 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 73 insertions(+), 31 deletions(-)
mm/vmscan.c +28 -4
index d1495a7d469d..6e13e570aab8 100644
@@ -181,6 +181,10 @@ struct scan_control {
        struct reclaim_state reclaim_state;
 };
 
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry);
+
 #ifdef ARCH_HAS_PREFETCHW
 static inline void prefetchw_prev_lru_folio(struct folio *folio,
                struct list_head *base)
@@ -2013,14 +2017,16 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
                enum lru_list lru)
 {
        LIST_HEAD(folio_list);
+       LIST_HEAD(clean_list);
        unsigned long nr_scanned;
-       unsigned int nr_reclaimed = 0;
-       unsigned long nr_taken;
+       unsigned int nr_reclaimed, total_reclaimed = 0;
+       unsigned long nr_taken, isolated;
        struct reclaim_stat stat;
        bool file = is_file_lru(lru);
        enum node_stat_item item;
        struct pglist_data *pgdat = lruvec_pgdat(lruvec);
        bool stalled = false;
+       bool skip_retry = false;
 
        while (unlikely(too_many_isolated(pgdat, file, sc))) {
                if (stalled)
@@ -2052,25 +2058,42 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
        if (nr_taken == 0)
                return 0;
 
+       isolated = nr_taken;
+retry:
        nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
                                         lruvec_memcg(lruvec));
+       total_reclaimed += nr_reclaimed;
+
+       /* Retry pass is only meant for clean folios without new isolation */
+       if (isolated)
+               handle_reclaim_writeback(isolated, pgdat, sc, &stat);
+       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
+                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
+
+       find_folios_written_back(&folio_list, &clean_list, lruvec, file, skip_retry);
 
        move_folios_to_lru(&folio_list);
 
        mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                        stat.nr_demoted);
-       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
        item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
        mod_lruvec_state(lruvec, item, nr_reclaimed);
        mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
-       if (nr_scanned > nr_reclaimed)
+
+       if (!list_empty(&clean_list)) {
+               list_splice_init(&clean_list, &folio_list);
+               skip_retry = true;
+               /* Retry folios were already isolated and accounted above */
+               isolated = 0;
+               goto retry;
+       }
+

mm/vmscan.c static unsigned long shrink_inactive_list(unsigned long nr_to_scan, · line 2073

Quoted 10 lines
mm/vmscan.c +3 -5
+       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
+       if (nr_scanned > total_reclaimed)
                mod_lruvec_state(lruvec, PGROTATE_ANON + file,
-                                nr_scanned - nr_reclaimed);
+                                nr_scanned - total_reclaimed);
 
-       handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
-       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
-       return nr_reclaimed;

this is a bit weird to move the tracepoint, shouldn't it just trace the total number of twice reclamation?

Quoted 26 lines
mm/vmscan.c +1 -11
+       return total_reclaimed;
 }
 
 /*
@@ -5006,8 +5029,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 {
        LIST_HEAD(list);
        LIST_HEAD(clean);
-       struct folio *folio;
-       struct folio *next;
        enum node_stat_item item;
        struct reclaim_stat stat;
        struct lru_gen_mm_walk *walk;
@@ -5046,26 +5067,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                        type_scanned, reclaimed, &stat, sc->priority,
                        type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
-       list_for_each_entry_safe_reverse(folio, next, &list, lru) {
-               DEFINE_MIN_SEQ(lruvec);
-
-               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
-               if (!folio_evictable(folio))
-                       continue;
-
-               /* retry folios that may have missed folio_rotate_reclaimable() */
-               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&

mm/vmscan.c static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 5067

Quoted 10 lines
mm/vmscan.c +0 -10
-                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
-                       list_move(&folio->lru, &clean);
-                       continue;
-               }
-
-               /* don't add rejected folios to the oldest generation */
-               if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
-                       folio_set_lru_refs(folio, 0);
-                       folio_set_active(folio);
-               }

Baolin has a patch which modifies this.
so probably you are not based on the mm-new?

https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git/commit/?id=519f7768585e372b97e568baf9d4648e6beb6863

Quoted 35 lines
mm/vmscan.c +27 -1
-       }
+       find_folios_written_back(&list, &clean, lruvec, type, skip_retry);
 
        move_folios_to_lru(&list);
 
@@ -6108,6 +6110,46 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
 
 #endif /* CONFIG_LRU_GEN */
 
+/**
+ * find_folios_written_back - Find and move the written back folios to a new list.
+ * @list: folios list
+ * @clean: the written back folios list
+ * @lruvec: the lruvec
+ * @type: LRU type (only used for CONFIG_LRU_GEN)
+ * @skip_retry: whether skip retry.
+ */
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry)
+{
+       struct folio *folio;
+       struct folio *next;
+
+       list_for_each_entry_safe_reverse(folio, next, list, lru) {
+#ifdef CONFIG_LRU_GEN
+               DEFINE_MIN_SEQ(lruvec);
+#endif
+               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+               if (!folio_evictable(folio))
+                       continue;
+
+               /* retry folios that may have missed folio_rotate_reclaimable() */
+               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
+                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {

mm/vmscan.c static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control * · line 6148

Quoted 10 lines
mm/vmscan.c +10 -0
+                       list_move(&folio->lru, clean);
+                       continue;
+               }
+#ifdef CONFIG_LRU_GEN
+               /* don't add rejected folios to the oldest generation */
+               if (lruvec->lrugen.enabled &&
+                   lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+                       folio_set_lru_refs(folio, 0);
+                       folio_set_active(folio);
+               }

as above. please take a look at Baolin's patch:
https://lore.kernel.org/9214e36bf738fcfba86acc8cea85dff4010f66b0.1788918714.git.baolin.wang@linux.alibaba.com

Best Regards
Barry

#3 lore raw
Quoted 248 lines · Barry Song
Quoted 151 lines · Ridong Chen

From: Ridong Chen <[email protected]>

As commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") mentioned:

The page reclaim isolates a batch of folios from the tail of one of the LRU lists and works on those folios one by one. For a suitable swap-backed folio, if the swap device is async, it queues that folio for writeback. After the page reclaim finishes an entire batch, it puts back the folios it queued for writeback to the head of the original LRU list.

In the meantime, the page writeback flushes the queued folios also by batches. Its batching logic is independent from that of the page reclaim. For each of the folios it writes back, the page writeback calls folio_rotate_reclaimable() which tries to rotate a folio to the tail.

folio_rotate_reclaimable() only works for a folio after the page reclaim has put it back. If an async swap device is fast enough, the page writeback can finish with that folio while the page reclaim is still working on the rest of the batch containing it. In this case, that folio will remain at the head and the page reclaim will not retry it before reaching there".

The commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") only fixed the issue for mglru. However, this issue also exists in the traditional active/inactive LRU and was found at [1].

It can be reproduced with below steps:

1. Compile with CONFIG_TRANSPARENT_HUGEPAGE=y
2. Mount memcg v1, and create memcg named test_memcg and set

    limit_in_bytes=1G, memsw.limit_in_bytes=2G.

3. Create a 1G swap file, and allocate 1.35G anon memory in test_memcg.

It was found that:

cat memory.usage_in_bytes
1073700864
cat memory.memsw.usage_in_bytes
1413124096

free -h

                 total        used        free
   Mem:           1.6Gi       1.2Gi       299Mi
   Swap:          1.0Gi       678Mi       346Mi

As shown above, the test_memcg charged about 324M swap (memsw.usage minus usage), but almost 678M swap memory was used, which means that 350M+ may be wasted because other memcgs can not use these swap memory.

This issue should be fixed in the same way as mglru. Therefore, the common logic was extracted to the 'find_folios_written_back' function firstly, which is then reused in the 'shrink_inactive_list' function. Finally, retry reclaiming those folios that may have missed the rotation for traditional LRU.

After change, the same test case only wasted about 2M swap. The swap device usage matches what the memcg actually charged.

cat memory.usage_in_bytes
1070301184
cat memory.memsw.usage_in_bytes
1412448256

free -h

                 total        used        free
   Mem:           1.6Gi       1.2Gi       299Mi
   Swap:          1.0Gi       327Mi       696Mi

[1] https://lore.kernel.org/linux-kernel/[email protected]/
[2] https://lore.kernel.org/linux-kernel/CAGsJ_4zqL8ZHNRZ44o_CC69kE7DBVXvbZfvmQxMGiFqRxqHQdA@mail.gmail.com/

---
v8: Rebase to the -next tree, adapt to the current reclaim API, and retest.

[v7]: https://lore.kernel.org/linux-mm/[email protected]/

  mm/vmscan.c | 104 ++++++++++++++++++++++++++++++++++++----------------
  1 file changed, 73 insertions(+), 31 deletions(-)
mm/vmscan.c +28 -4
index d1495a7d469d..6e13e570aab8 100644
@@ -181,6 +181,10 @@ struct scan_control {
         struct reclaim_state reclaim_state;
  };
 
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry);
+
  #ifdef ARCH_HAS_PREFETCHW
  static inline void prefetchw_prev_lru_folio(struct folio *folio,
                 struct list_head *base)
@@ -2013,14 +2017,16 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
                 enum lru_list lru)
  {
         LIST_HEAD(folio_list);
+       LIST_HEAD(clean_list);
         unsigned long nr_scanned;
-       unsigned int nr_reclaimed = 0;
-       unsigned long nr_taken;
+       unsigned int nr_reclaimed, total_reclaimed = 0;
+       unsigned long nr_taken, isolated;
         struct reclaim_stat stat;
         bool file = is_file_lru(lru);
         enum node_stat_item item;
         struct pglist_data *pgdat = lruvec_pgdat(lruvec);
         bool stalled = false;
+       bool skip_retry = false;
 
         while (unlikely(too_many_isolated(pgdat, file, sc))) {
                 if (stalled)
@@ -2052,25 +2058,42 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
         if (nr_taken == 0)
                 return 0;
 
+       isolated = nr_taken;
+retry:
         nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
                                          lruvec_memcg(lruvec));
+       total_reclaimed += nr_reclaimed;
+
+       /* Retry pass is only meant for clean folios without new isolation */
+       if (isolated)
+               handle_reclaim_writeback(isolated, pgdat, sc, &stat);
+       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
+                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
+
+       find_folios_written_back(&folio_list, &clean_list, lruvec, file, skip_retry);
 
         move_folios_to_lru(&folio_list);
 
         mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                         stat.nr_demoted);
-       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
         item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
         mod_lruvec_state(lruvec, item, nr_reclaimed);
         mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
-       if (nr_scanned > nr_reclaimed)
+
+       if (!list_empty(&clean_list)) {
+               list_splice_init(&clean_list, &folio_list);
+               skip_retry = true;
+               /* Retry folios were already isolated and accounted above */
+               isolated = 0;
+               goto retry;
+       }
+

mm/vmscan.c static unsigned long shrink_inactive_list(unsigned long nr_to_scan, · line 2073

Quoted 10 lines
mm/vmscan.c +3 -5
+       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
+       if (nr_scanned > total_reclaimed)
                 mod_lruvec_state(lruvec, PGROTATE_ANON + file,
-                                nr_scanned - nr_reclaimed);
+                                nr_scanned - total_reclaimed);
 
-       handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
-       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
-       return nr_reclaimed;

this is a bit weird to move the tracepoint, shouldn't it just trace the total number of twice reclamation?

Quoted 26 lines
mm/vmscan.c +1 -11
+       return total_reclaimed;
  }
 
  /*
@@ -5006,8 +5029,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
  {
         LIST_HEAD(list);
         LIST_HEAD(clean);
-       struct folio *folio;
-       struct folio *next;
         enum node_stat_item item;
         struct reclaim_stat stat;
         struct lru_gen_mm_walk *walk;
@@ -5046,26 +5067,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                         type_scanned, reclaimed, &stat, sc->priority,
                         type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
-       list_for_each_entry_safe_reverse(folio, next, &list, lru) {
-               DEFINE_MIN_SEQ(lruvec);
-
-               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
-               if (!folio_evictable(folio))
-                       continue;
-
-               /* retry folios that may have missed folio_rotate_reclaimable() */
-               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&

mm/vmscan.c static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 5067

Quoted 10 lines
mm/vmscan.c +0 -10
-                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
-                       list_move(&folio->lru, &clean);
-                       continue;
-               }
-
-               /* don't add rejected folios to the oldest generation */
-               if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
-                       folio_set_lru_refs(folio, 0);
-                       folio_set_active(folio);
-               }

Baolin has a patch which modifies this.
so probably you are not based on the mm-new?

https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git/commit/?id=519f7768585e372b97e568baf9d4648e6beb6863

Quoted 40 lines
mm/vmscan.c +32 -1
-       }
+       find_folios_written_back(&list, &clean, lruvec, type, skip_retry);
 
         move_folios_to_lru(&list);
 
@@ -6108,6 +6110,46 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
 
  #endif /* CONFIG_LRU_GEN */
 
+/**
+ * find_folios_written_back - Find and move the written back folios to a new list.
+ * @list: folios list
+ * @clean: the written back folios list
+ * @lruvec: the lruvec
+ * @type: LRU type (only used for CONFIG_LRU_GEN)
+ * @skip_retry: whether skip retry.
+ */
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry)
+{
+       struct folio *folio;
+       struct folio *next;
+
+       list_for_each_entry_safe_reverse(folio, next, list, lru) {
+#ifdef CONFIG_LRU_GEN
+               DEFINE_MIN_SEQ(lruvec);
+#endif
+               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+               if (!folio_evictable(folio))
+                       continue;
+
+               /* retry folios that may have missed folio_rotate_reclaimable() */
+               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
+                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
+                       list_move(&folio->lru, clean);
+                       continue;
+               }
+#ifdef CONFIG_LRU_GEN
+               /* don't add rejected folios to the oldest generation */
Quoted 10 lines

mm/vmscan.c static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control * · line 6148

Quoted 5 lines
mm/vmscan.c +5 -0
+               if (lruvec->lrugen.enabled &&
+                   lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+                       folio_set_lru_refs(folio, 0);
+                       folio_set_active(folio);
+               }

as above. please take a look at Baolin's patch:
https://lore.kernel.org/9214e36bf738fcfba86acc8cea85dff4010f66b0.1788918714.git.baolin.wang@linux.alibaba.com

Thanks, I rebased on the next branch.

I will rebase on mm-new.

Signature · 2 lines
Best regards
Ridong
#4 lore raw
In reply to Barry Song
Quoted 158 lines · Barry Song
Quoted 157 lines · Ridong Chen

From: Ridong Chen <[email protected]>

As commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") mentioned:

The page reclaim isolates a batch of folios from the tail of one of the LRU lists and works on those folios one by one. For a suitable swap-backed folio, if the swap device is async, it queues that folio for writeback. After the page reclaim finishes an entire batch, it puts back the folios it queued for writeback to the head of the original LRU list.

In the meantime, the page writeback flushes the queued folios also by batches. Its batching logic is independent from that of the page reclaim. For each of the folios it writes back, the page writeback calls folio_rotate_reclaimable() which tries to rotate a folio to the tail.

folio_rotate_reclaimable() only works for a folio after the page reclaim has put it back. If an async swap device is fast enough, the page writeback can finish with that folio while the page reclaim is still working on the rest of the batch containing it. In this case, that folio will remain at the head and the page reclaim will not retry it before reaching there".

The commit 359a5e1416ca ("mm: multi-gen LRU: retry folios written back while isolated") only fixed the issue for mglru. However, this issue also exists in the traditional active/inactive LRU and was found at [1].

It can be reproduced with below steps:

1. Compile with CONFIG_TRANSPARENT_HUGEPAGE=y
2. Mount memcg v1, and create memcg named test_memcg and set

    limit_in_bytes=1G, memsw.limit_in_bytes=2G.

3. Create a 1G swap file, and allocate 1.35G anon memory in test_memcg.

It was found that:

cat memory.usage_in_bytes
1073700864
cat memory.memsw.usage_in_bytes
1413124096

free -h

                 total        used        free
   Mem:           1.6Gi       1.2Gi       299Mi
   Swap:          1.0Gi       678Mi       346Mi

As shown above, the test_memcg charged about 324M swap (memsw.usage minus usage), but almost 678M swap memory was used, which means that 350M+ may be wasted because other memcgs can not use these swap memory.

This issue should be fixed in the same way as mglru. Therefore, the common logic was extracted to the 'find_folios_written_back' function firstly, which is then reused in the 'shrink_inactive_list' function. Finally, retry reclaiming those folios that may have missed the rotation for traditional LRU.

After change, the same test case only wasted about 2M swap. The swap device usage matches what the memcg actually charged.

cat memory.usage_in_bytes
1070301184
cat memory.memsw.usage_in_bytes
1412448256

free -h

                 total        used        free
   Mem:           1.6Gi       1.2Gi       299Mi
   Swap:          1.0Gi       327Mi       696Mi

[1] https://lore.kernel.org/linux-kernel/[email protected]/
[2] https://lore.kernel.org/linux-kernel/CAGsJ_4zqL8ZHNRZ44o_CC69kE7DBVXvbZfvmQxMGiFqRxqHQdA@mail.gmail.com/

---
v8: Rebase to the -next tree, adapt to the current reclaim API, and retest.

[v7]: https://lore.kernel.org/linux-mm/[email protected]/

  mm/vmscan.c | 104 ++++++++++++++++++++++++++++++++++++----------------
  1 file changed, 73 insertions(+), 31 deletions(-)
mm/vmscan.c +31 -5
index d1495a7d469d..6e13e570aab8 100644
@@ -181,6 +181,10 @@ struct scan_control {
         struct reclaim_state reclaim_state;
  };
 
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry);
+
  #ifdef ARCH_HAS_PREFETCHW
  static inline void prefetchw_prev_lru_folio(struct folio *folio,
                 struct list_head *base)
@@ -2013,14 +2017,16 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
                 enum lru_list lru)
  {
         LIST_HEAD(folio_list);
+       LIST_HEAD(clean_list);
         unsigned long nr_scanned;
-       unsigned int nr_reclaimed = 0;
-       unsigned long nr_taken;
+       unsigned int nr_reclaimed, total_reclaimed = 0;
+       unsigned long nr_taken, isolated;
         struct reclaim_stat stat;
         bool file = is_file_lru(lru);
         enum node_stat_item item;
         struct pglist_data *pgdat = lruvec_pgdat(lruvec);
         bool stalled = false;
+       bool skip_retry = false;
 
         while (unlikely(too_many_isolated(pgdat, file, sc))) {
                 if (stalled)
@@ -2052,25 +2058,42 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
         if (nr_taken == 0)
                 return 0;
 
+       isolated = nr_taken;
+retry:
         nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
                                          lruvec_memcg(lruvec));
+       total_reclaimed += nr_reclaimed;
+
+       /* Retry pass is only meant for clean folios without new isolation */
+       if (isolated)
+               handle_reclaim_writeback(isolated, pgdat, sc, &stat);
+       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
+                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
+
+       find_folios_written_back(&folio_list, &clean_list, lruvec, file, skip_retry);
 
         move_folios_to_lru(&folio_list);
 
         mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                         stat.nr_demoted);
-       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
         item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
         mod_lruvec_state(lruvec, item, nr_reclaimed);
         mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
-       if (nr_scanned > nr_reclaimed)
+
+       if (!list_empty(&clean_list)) {
+               list_splice_init(&clean_list, &folio_list);
+               skip_retry = true;
+               /* Retry folios were already isolated and accounted above */
+               isolated = 0;
+               goto retry;
+       }
+
+       mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
+       if (nr_scanned > total_reclaimed)
                 mod_lruvec_state(lruvec, PGROTATE_ANON + file,
-                                nr_scanned - nr_reclaimed);
+                                nr_scanned - total_reclaimed);
 
Quoted 8 lines

mm/vmscan.c static unsigned long shrink_inactive_list(unsigned long nr_to_scan, · line 2073

Quoted 4 lines
mm/vmscan.c +0 -4
-       handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
-       trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-                       nr_scanned, nr_reclaimed, &stat, sc->priority, file);
-       return nr_reclaimed;

this is a bit weird to move the tracepoint, shouldn't it just trace the total number of twice reclamation?

Sorry for misssing this comment.

I think we should move the tracepoint, since the 'stat' would be cleared by the time it is passed to shrink_folio_list, just as evict_folios does.

Quoted 94 lines
Quoted 26 lines
mm/vmscan.c +1 -11
+       return total_reclaimed;
  }
 
  /*
@@ -5006,8 +5029,6 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
  {
         LIST_HEAD(list);
         LIST_HEAD(clean);
-       struct folio *folio;
-       struct folio *next;
         enum node_stat_item item;
         struct reclaim_stat stat;
         struct lru_gen_mm_walk *walk;
@@ -5046,26 +5067,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
                         type_scanned, reclaimed, &stat, sc->priority,
                         type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
-       list_for_each_entry_safe_reverse(folio, next, &list, lru) {
-               DEFINE_MIN_SEQ(lruvec);
-
-               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
-               if (!folio_evictable(folio))
-                       continue;
-
-               /* retry folios that may have missed folio_rotate_reclaimable() */
-               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&

mm/vmscan.c static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · line 5067

Quoted 10 lines
mm/vmscan.c +0 -10
-                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
-                       list_move(&folio->lru, &clean);
-                       continue;
-               }
-
-               /* don't add rejected folios to the oldest generation */
-               if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
-                       folio_set_lru_refs(folio, 0);
-                       folio_set_active(folio);
-               }

Baolin has a patch which modifies this.
so probably you are not based on the mm-new?

https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git/commit/?id=519f7768585e372b97e568baf9d4648e6beb6863

Quoted 35 lines
mm/vmscan.c +27 -1
-       }
+       find_folios_written_back(&list, &clean, lruvec, type, skip_retry);
 
         move_folios_to_lru(&list);
 
@@ -6108,6 +6110,46 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
 
  #endif /* CONFIG_LRU_GEN */
 
+/**
+ * find_folios_written_back - Find and move the written back folios to a new list.
+ * @list: folios list
+ * @clean: the written back folios list
+ * @lruvec: the lruvec
+ * @type: LRU type (only used for CONFIG_LRU_GEN)
+ * @skip_retry: whether skip retry.
+ */
+static void find_folios_written_back(struct list_head *list,
+                                    struct list_head *clean, struct lruvec *lruvec,
+                                    int type, bool skip_retry)
+{
+       struct folio *folio;
+       struct folio *next;
+
+       list_for_each_entry_safe_reverse(folio, next, list, lru) {
+#ifdef CONFIG_LRU_GEN
+               DEFINE_MIN_SEQ(lruvec);
+#endif
+               /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+               if (!folio_evictable(folio))
+                       continue;
+
+               /* retry folios that may have missed folio_rotate_reclaimable() */
+               if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
+                   !folio_test_dirty(folio) && !folio_test_writeback(folio)) {

mm/vmscan.c static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control * · line 6148

Quoted 10 lines
mm/vmscan.c +10 -0
+                       list_move(&folio->lru, clean);
+                       continue;
+               }
+#ifdef CONFIG_LRU_GEN
+               /* don't add rejected folios to the oldest generation */
+               if (lruvec->lrugen.enabled &&
+                   lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+                       folio_set_lru_refs(folio, 0);
+                       folio_set_active(folio);
+               }

as above. please take a look at Baolin's patch:
https://lore.kernel.org/9214e36bf738fcfba86acc8cea85dff4010f66b0.1788918714.git.baolin.wang@linux.alibaba.com

Best Regards
Barry

Signature · 2 lines
Best regards
Ridong