mm/vmscan: move tracepoints to mm/ and add MGLRU tracepoints

From: Ridong Chen <[email protected]>

memcontrol.c includes <trace/events/vmscan.h> but does not use any of the tracepoints or macros it defines: all tracepoints invoked here come from memcg.h and writeback.h.

Drop it ahead of moving the vmscan trace header under mm/, after which <trace/events/vmscan.h> will no longer exist.

No functional change.

Acked-by Muchun Song <[email protected]>

Signed-off-by Ridong Chen <[email protected]>

Reviewed-by Barry Song <[email protected]>

---
 mm/memcontrol.c | 2 --
 1 file changed, 2 deletions(-)

mm/memcontrol.c +0 -2

index 84f4a8fd4d16..8a35f5b635e0 100644
@@ -77,8 +77,6 @@
 #include <trace/events/memcg.h>
 #undef CREATE_TRACE_POINTS
 
-#include <trace/events/vmscan.h>
-
 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
 EXPORT_SYMBOL(memory_cgrp_subsys);
 

Signature · 1 line

2.43.0

From: Ridong Chen <[email protected]>

Move include/trace/events/vmscan.h to mm/trace_vmscan.h so that the tracepoints can be extended to reference vmscan-internal structures such as struct scan_control, which is not visible outside mm/. This mirrors what several other subsystems already do with their trace headers (e.g. fs/fuse/fuse_trace.h, fs/gfs2/trace_gfs2.h, fs/lockd/trace.h). As they do, set TRACE_INCLUDE_PATH to "." and add mm/ to the include path of vmscan.o, the only user that defines CREATE_TRACE_POINTS.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

Reviewed-by Barry Song <[email protected]>

From: Ridong Chen <[email protected]>

Move include/trace/events/vmscan.h to mm/trace_vmscan.h so that the tracepoints can be extended to reference vmscan-internal structures such as struct scan_control, which is not visible outside mm/. This mirrors what several other subsystems already do with their trace headers (e.g. fs/fuse/fuse_trace.h, fs/gfs2/trace_gfs2.h, fs/lockd/trace.h). As they do, set TRACE_INCLUDE_PATH to "." and add mm/ to the include path of vmscan.o, the only user that defines CREATE_TRACE_POINTS.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

Reviewed-by Barry Song <[email protected]>

---
 MAINTAINERS                                        | 2 ++
 mm/Makefile                                        | 1 +
 mm/shrinker.c                                      | 2 +-
 include/trace/events/vmscan.h => mm/trace_vmscan.h | 4 ++++
 mm/vmscan.c                                        | 2 +-
 5 files changed, 9 insertions(+), 2 deletions(-)
 rename include/trace/events/vmscan.h => mm/trace_vmscan.h (99%)

MAINTAINERS +2 -0

index aaf2825ac945..2ad01107fa05 100644
@@ -17653,6 +17653,7 @@ F:	Documentation/admin-guide/mm/multigen_lru.rst
 F:	Documentation/mm/multigen_lru.rst
 F:	include/linux/mm_inline.h
 F:	include/linux/mmzone.h
+F:	mm/trace_vmscan.h
 F:	mm/vmscan.c
 F:	mm/workingset.c
 

@@ -17802,6 +17803,7 @@ R:	Shakeel Butt <[email protected]>
 R:	Lorenzo Stoakes <[email protected]>
 L:	[email protected]
 S:	Maintained
+F:	mm/trace_vmscan.h
 F:	mm/vmscan.c
 F:	mm/workingset.c
 

mm/Makefile +0 -0

index 2a3ec53d62ee..277b97f7f8d7 100644

mm/Makefile +0 -0

 mmu-y			:= nommu.o
 mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \

mm/shrinker.c +1 -1

index 4fb424e721bc..4fbdade6d2f6 100644
@@ -3,7 +3,7 @@
 #include <linux/rwsem.h>
 #include <linux/shrinker.h>
 #include <linux/rculist.h>
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
 
 #include "internal.h"
 

mm/trace_vmscan.h renamed from include/trace/events/vmscan.h +4 -0

similarity index 99%
rename from include/trace/events/vmscan.h
rename to mm/trace_vmscan.h
index 8a872990b4be..ad90f0e9bf2e 100644
@@ -648,4 +648,8 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless,
 #endif /* _TRACE_VMSCAN_H */
 
 /* This part must be outside protection */
+#undef TRACE_INCLUDE_FILE
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_vmscan
 #include <trace/define_trace.h>

mm/vmscan.c +1 -1

index 9959ca158ce8..79d2d614afd0 100644
@@ -71,7 +71,7 @@
 #include "swap.h"
 
 #define CREATE_TRACE_POINTS
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
 
 struct scan_control {
 	/* How many pages shrink_list() should reclaim */

Signature · 1 line

2.43.0

Quoted 48 lines · Ridong Chen

From: Ridong Chen <[email protected]>

Move include/trace/events/vmscan.h to mm/trace_vmscan.h so that the tracepoints can be extended to reference vmscan-internal structures such as struct scan_control, which is not visible outside mm/. This mirrors what several other subsystems already do with their trace headers (e.g. fs/fuse/fuse_trace.h, fs/gfs2/trace_gfs2.h, fs/lockd/trace.h). As they do, set TRACE_INCLUDE_PATH to "." and add mm/ to the include path of vmscan.o, the only user that defines CREATE_TRACE_POINTS.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

Reviewed-by Barry Song <[email protected]>

---
  MAINTAINERS                                        | 2 ++
  mm/Makefile                                        | 1 +
  mm/shrinker.c                                      | 2 +-
  include/trace/events/vmscan.h => mm/trace_vmscan.h | 4 ++++
  mm/vmscan.c                                        | 2 +-
  5 files changed, 9 insertions(+), 2 deletions(-)

rename include/trace/events/vmscan.h => mm/trace_vmscan.h (99%)

MAINTAINERS +2 -0

index aaf2825ac945..2ad01107fa05 100644
@@ -17653,6 +17653,7 @@ F:	Documentation/admin-guide/mm/multigen_lru.rst
  F:	Documentation/mm/multigen_lru.rst
  F:	include/linux/mm_inline.h
  F:	include/linux/mmzone.h
+F:	mm/trace_vmscan.h
  F:	mm/vmscan.c
  F:	mm/workingset.c
 

@@ -17802,6 +17803,7 @@ R:	Shakeel Butt <[email protected]>
  R:	Lorenzo Stoakes <[email protected]>
  L:	[email protected]
  S:	Maintained
+F:	mm/trace_vmscan.h
  F:	mm/vmscan.c
  F:	mm/workingset.c
 

mm/Makefile +0 -0

index 2a3ec53d62ee..277b97f7f8d7 100644

mm/Makefile KCOV_INSTRUMENT_vmstat.o := n · line 35

Quoted 4 lines

mm/Makefile +1 -0

@@ -32,6 +32,7 @@ KCOV_INSTRUMENT_vmstat.o := n
  KCOV_INSTRUMENT_failslab.o := n
 
  CFLAGS_init-mm.o += -Wno-override-init
+CFLAGS_vmscan.o += -I$(src)

One nit:

Does it really need the above line? It affects mm/shrinker.c too, did your shrinker.o building fail?

Seems it's redundant because kbuild adds -I$(src) -I$(obj) for every object in the directory since commit cdd750bfb1f7 ("kbuild: remove 'addtree' and 'flags' magic for header search paths"). Please correct me if I am wrong.

Other than the nit, this patch looks good to me,

Reviewed-by Baoquan He <[email protected]>

Thanks.

Quoted 48 lines

mm/Makefile +0 -0

  mmu-y			:= nommu.o
  mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \

mm/shrinker.c +1 -1

index 4fb424e721bc..4fbdade6d2f6 100644
@@ -3,7 +3,7 @@
  #include <linux/rwsem.h>
  #include <linux/shrinker.h>
  #include <linux/rculist.h>
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
 
  #include "internal.h"
 

mm/trace_vmscan.h renamed from include/trace/events/vmscan.h +4 -0

similarity index 99%
rename from include/trace/events/vmscan.h
rename to mm/trace_vmscan.h
index 8a872990b4be..ad90f0e9bf2e 100644
@@ -648,4 +648,8 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless,
  #endif /* _TRACE_VMSCAN_H */
 
  /* This part must be outside protection */
+#undef TRACE_INCLUDE_FILE
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_vmscan
  #include <trace/define_trace.h>

mm/vmscan.c +1 -1

index 9959ca158ce8..79d2d614afd0 100644
@@ -71,7 +71,7 @@
  #include "swap.h"
 
  #define CREATE_TRACE_POINTS
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
 
  struct scan_control {
  	/* How many pages shrink_list() should reclaim */

Signature · 1 line

2.43.0

Quoted 53 lines · Baoquan He

Quoted 46 lines · Ridong Chen

From: Ridong Chen <[email protected]>

Move include/trace/events/vmscan.h to mm/trace_vmscan.h so that the tracepoints can be extended to reference vmscan-internal structures such as struct scan_control, which is not visible outside mm/. This mirrors what several other subsystems already do with their trace headers (e.g. fs/fuse/fuse_trace.h, fs/gfs2/trace_gfs2.h, fs/lockd/trace.h). As they do, set TRACE_INCLUDE_PATH to "." and add mm/ to the include path of vmscan.o, the only user that defines CREATE_TRACE_POINTS.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

Reviewed-by Barry Song <[email protected]>

---
  MAINTAINERS                                        | 2 ++
  mm/Makefile                                        | 1 +
  mm/shrinker.c                                      | 2 +-
  include/trace/events/vmscan.h => mm/trace_vmscan.h | 4 ++++
  mm/vmscan.c                                        | 2 +-
  5 files changed, 9 insertions(+), 2 deletions(-)

rename include/trace/events/vmscan.h => mm/trace_vmscan.h (99%)

MAINTAINERS +2 -0

index aaf2825ac945..2ad01107fa05 100644
@@ -17653,6 +17653,7 @@ F:	Documentation/admin-guide/mm/multigen_lru.rst
  F:	Documentation/mm/multigen_lru.rst
  F:	include/linux/mm_inline.h
  F:	include/linux/mmzone.h
+F:	mm/trace_vmscan.h
  F:	mm/vmscan.c
  F:	mm/workingset.c

@@ -17802,6 +17803,7 @@ R:	Shakeel Butt <[email protected]>
  R:	Lorenzo Stoakes <[email protected]>
  L:	[email protected]
  S:	Maintained
+F:	mm/trace_vmscan.h
  F:	mm/vmscan.c
  F:	mm/workingset.c

mm/Makefile +0 -0

index 2a3ec53d62ee..277b97f7f8d7 100644

mm/Makefile KCOV_INSTRUMENT_vmstat.o := n · line 34

Quoted 3 lines

mm/Makefile +1 -0

@@ -32,6 +32,7 @@ KCOV_INSTRUMENT_vmstat.o := n
  KCOV_INSTRUMENT_failslab.o := n
  CFLAGS_init-mm.o += -Wno-override-init
+CFLAGS_vmscan.o += -I$(src)

One nit:

Quoted 8 lines

Does it really need the above line? It affects mm/shrinker.c too, did your shrinker.o building fail?

Seems it's redundant because kbuild adds -I$(src) -I$(obj) for every object in the directory since commit cdd750bfb1f7 ("kbuild: remove 'addtree' and 'flags' magic for header search paths"). Please correct me if I am wrong.

Hi Baoquan,

Thanks for the review.

The line is needed for vmscan.o, and mm/shrinker.c is fine without it, because the two files include trace_vmscan.h in different ways.

mm/shrinker.c only does a plain #include "trace_vmscan.h". The lookup starts from mm/, where the header lives, so no -I is needed.

mm/vmscan.c defines CREATE_TRACE_POINTS before including the header. In that case <trace/define_trace.h>, included at the end of trace_vmscan.h, includes the header again to generate the tracepoint definitions:

#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) /* -> "./trace_vmscan.h" */

This #include is in include/trace/define_trace.h, so the lookup starts from include/trace/ and only finds the header through -I. Without the flag, vmscan.o fails to build:

./include/trace/define_trace.h:118:42: fatal error: ./trace_vmscan.h: No such file or directory

As for cdd750bfb1f7, it doesn't add -I$(src) for every build. kbuild only adds -I$(src) -I$(obj) automatically when building with O= (see "ifdef building_out_of_srctree" in scripts/Makefile.lib). So with O= the line is indeed redundant and vmscan.o builds fine without it, but a plain in-tree "make" fails with the error above. The explicit flag keeps both working.

To be honest, I had the same worry at first: I thought every .c file that includes the header would need a Makefile change. That's why v3 used a relative TRACE_INCLUDE_PATH (../../mm) instead. Steven corrected me and pointed out that the proper way is TRACE_INCLUDE_PATH "." plus -I$(src) for the one object that defines CREATE_TRACE_POINTS [1]. That turned out to need only this one line. Thanks again to Steven for that.

[1] https://lore.kernel.org/all/[email protected]/

Quoted 47 lines

Other than the nit, this patch looks good to me,

Reviewed-by Baoquan He <[email protected]>

Thanks.

Quoted 42 lines

mm/Makefile +0 -0

  mmu-y			:= nommu.o
  mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \

mm/shrinker.c +1 -1

index 4fb424e721bc..4fbdade6d2f6 100644
@@ -3,7 +3,7 @@
  #include <linux/rwsem.h>
  #include <linux/shrinker.h>
  #include <linux/rculist.h>
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
  #include "internal.h"

mm/trace_vmscan.h renamed from include/trace/events/vmscan.h +4 -0

similarity index 99%
rename from include/trace/events/vmscan.h
rename to mm/trace_vmscan.h
index 8a872990b4be..ad90f0e9bf2e 100644
@@ -648,4 +648,8 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless,
  #endif /* _TRACE_VMSCAN_H */
  /* This part must be outside protection */
+#undef TRACE_INCLUDE_FILE
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_vmscan
  #include <trace/define_trace.h>

mm/vmscan.c +1 -1

index 9959ca158ce8..79d2d614afd0 100644
@@ -71,7 +71,7 @@
  #include "swap.h"
  #define CREATE_TRACE_POINTS
-#include <trace/events/vmscan.h>
+#include "trace_vmscan.h"
  struct scan_control {
  	/* How many pages shrink_list() should reclaim */

Signature · 1 line

2.43.0

Signature · 2 lines

Best regards
Ridong

From: Ridong Chen <[email protected]>

The definition of struct scan_control has lived inline at the top of mm/vmscan.c. Move it out to a new mm/vmscan.h so the reclaim-internal state can be referenced from other reclaim translation units without duplicating the definition.

The header uses an ordinary include guard and pulls in only what the struct needs (nodemask_t, gfp_t and struct reclaim_state).

mm/trace_vmscan.h includes the new header directly so that any translation unit pulling in the trace events (mm/vmscan.c, mm/shrinker.c) sees the full definition of struct scan_control regardless of include order.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

From: Ridong Chen <[email protected]>

The definition of struct scan_control has lived inline at the top of mm/vmscan.c. Move it out to a new mm/vmscan.h so the reclaim-internal state can be referenced from other reclaim translation units without duplicating the definition.

The header uses an ordinary include guard and pulls in only what the struct needs (nodemask_t, gfp_t and struct reclaim_state).

mm/trace_vmscan.h includes the new header directly so that any translation unit pulling in the trace events (mm/vmscan.c, mm/shrinker.c) sees the full definition of struct scan_control regardless of include order.

No functional change.

Signed-off-by Ridong Chen <[email protected]>

---
 MAINTAINERS       |   2 +
 mm/trace_vmscan.h |   2 +
 mm/vmscan.c       | 108 ------------------------------------------
 mm/vmscan.h       | 118 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 122 insertions(+), 108 deletions(-)
 create mode 100644 mm/vmscan.h

MAINTAINERS +2 -0

index 2ad01107fa05..724791bb779f 100644
@@ -17655,6 +17655,7 @@ F:	include/linux/mm_inline.h
 F:	include/linux/mmzone.h
 F:	mm/trace_vmscan.h
 F:	mm/vmscan.c
+F:	mm/vmscan.h
 F:	mm/workingset.c
 
 MEMORY MANAGEMENT - MIGRATION

@@ -17805,6 +17806,7 @@ L:	[email protected]
 S:	Maintained
 F:	mm/trace_vmscan.h
 F:	mm/vmscan.c
+F:	mm/vmscan.h
 F:	mm/workingset.c
 
 MEMORY MANAGEMENT - RMAP (REVERSE MAPPING)

mm/trace_vmscan.h +2 -0

index ad90f0e9bf2e..3917e1b62dfa 100644
@@ -11,6 +11,8 @@
 #include <linux/memcontrol.h>
 #include <trace/events/mmflags.h>
 
+#include "vmscan.h"
+
 #define RECLAIM_WB_ANON		0x0001u
 #define RECLAIM_WB_FILE		0x0002u
 #define RECLAIM_WB_MIXED	0x0010u

mm/vmscan.c +0 -108

index 79d2d614afd0..49677464fc9a 100644
@@ -73,114 +73,6 @@
 #define CREATE_TRACE_POINTS
 #include "trace_vmscan.h"
 
-struct scan_control {
-	/* How many pages shrink_list() should reclaim */
-	unsigned long nr_to_reclaim;
-
-	/*
-	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
-	 * are scanned.
-	 */
-	const nodemask_t *nodemask;
-
-	/*
-	 * The memory cgroup that hit its limit and as a result is the
-	 * primary target of this reclaim invocation.
-	 */
-	struct mem_cgroup *target_mem_cgroup;
-
-	/*
-	 * Scan pressure balancing between anon and file LRUs
-	 */
-	unsigned long	anon_cost;
-	unsigned long	file_cost;
-
-	/* Swappiness value for proactive reclaim. Always use sc_swappiness()! */
-	int *proactive_swappiness;
-
-	/* Can active folios be deactivated as part of reclaim? */
-#define DEACTIVATE_ANON 1
-#define DEACTIVATE_FILE 2
-	unsigned int may_deactivate:2;
-	unsigned int force_deactivate:1;
-	unsigned int skipped_deactivate:1;
-
-	/* zone_reclaim_mode, boost reclaim */
-	unsigned int may_writepage:1;
-
-	/* zone_reclaim_mode */
-	unsigned int may_unmap:1;
-
-	/* zone_reclaim_mode, boost reclaim, cgroup restrictions */
-	unsigned int may_swap:1;
-
-	/* Not allow cache_trim_mode to be turned on as part of reclaim? */
-	unsigned int no_cache_trim_mode:1;
-
-	/* Has cache_trim_mode failed at least once? */
-	unsigned int cache_trim_mode_failed:1;
-
-	/* Proactive reclaim invoked by userspace */
-	unsigned int proactive:1;
-
-	/*
-	 * Cgroup memory below memory.low is protected as long as we
-	 * don't threaten to OOM. If any cgroup is reclaimed at
-	 * reduced force or passed over entirely due to its memory.low
-	 * setting (memcg_low_skipped), and nothing is reclaimed as a
-	 * result, then go back for one more cycle that reclaims the protected
-	 * memory (memcg_low_reclaim) to avert OOM.
-	 */
-	unsigned int memcg_low_reclaim:1;
-	unsigned int memcg_low_skipped:1;
-
-	/* Shared cgroup tree walk failed, rescan the whole tree */
-	unsigned int memcg_full_walk:1;
-
-	unsigned int hibernation_mode:1;
-
-	/* One of the zones is ready for compaction */
-	unsigned int compaction_ready:1;
-
-	/* There is easily reclaimable cold cache in the current node */
-	unsigned int cache_trim_mode:1;
-
-	/* The file folios on the current node are dangerously low */
-	unsigned int file_is_tiny:1;
-
-	/* Always discard instead of demoting to lower tier memory */
-	unsigned int no_demotion:1;
-
-	/* Allocation order */
-	s8 order;
-
-	/* Scan (total_size >> priority) pages at once */
-	s8 priority;
-
-	/* The highest zone to isolate folios for reclaim from */
-	s8 reclaim_idx;
-
-	/* This context's GFP mask */
-	gfp_t gfp_mask;
-
-	/* Incremented by the number of inactive pages that were scanned */
-	unsigned long nr_scanned;
-
-	/* Number of pages freed so far during a call to shrink_zones() */
-	unsigned long nr_reclaimed;
-
-	struct {
-		unsigned int dirty;
-		unsigned int congested;
-		unsigned int writeback;
-		unsigned int immediate;
-		unsigned int taken;
-	} nr;
-
-	/* for recording the reclaimed slab by now */
-	struct reclaim_state reclaim_state;
-};
-
 #ifdef ARCH_HAS_PREFETCHW
 static inline void prefetchw_prev_lru_folio(struct folio *folio,
 		struct list_head *base)

mm/vmscan.h new file +118 -0

new file mode 100644
index 000000000000..adb22e80d760
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_VMSCAN_H
+#define _MM_VMSCAN_H
+
+#include <linux/gfp_types.h>
+#include <linux/nodemask.h>
+#include <linux/swap.h>
+#include <linux/types.h>
+
+struct scan_control {
+	/* How many pages shrink_list() should reclaim */
+	unsigned long nr_to_reclaim;
+
+	/*
+	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
+	 * are scanned.
+	 */
+	const nodemask_t *nodemask;
+
+	/*
+	 * The memory cgroup that hit its limit and as a result is the
+	 * primary target of this reclaim invocation.
+	 */
+	struct mem_cgroup *target_mem_cgroup;
+
+	/*
+	 * Scan pressure balancing between anon and file LRUs
+	 */
+	unsigned long	anon_cost;
+	unsigned long	file_cost;
+
+	/* Swappiness value for proactive reclaim. Always use sc_swappiness()! */
+	int *proactive_swappiness;
+
+	/* Can active folios be deactivated as part of reclaim? */
+#define DEACTIVATE_ANON 1
+#define DEACTIVATE_FILE 2
+	unsigned int may_deactivate:2;
+	unsigned int force_deactivate:1;
+	unsigned int skipped_deactivate:1;
+
+	/* zone_reclaim_mode, boost reclaim */
+	unsigned int may_writepage:1;
+
+	/* zone_reclaim_mode */
+	unsigned int may_unmap:1;
+
+	/* zone_reclaim_mode, boost reclaim, cgroup restrictions */
+	unsigned int may_swap:1;
+
+	/* Not allow cache_trim_mode to be turned on as part of reclaim? */
+	unsigned int no_cache_trim_mode:1;
+
+	/* Has cache_trim_mode failed at least once? */
+	unsigned int cache_trim_mode_failed:1;
+
+	/* Proactive reclaim invoked by userspace */
+	unsigned int proactive:1;
+
+	/*
+	 * Cgroup memory below memory.low is protected as long as we
+	 * don't threaten to OOM. If any cgroup is reclaimed at
+	 * reduced force or passed over entirely due to its memory.low
+	 * setting (memcg_low_skipped), and nothing is reclaimed as a
+	 * result, then go back for one more cycle that reclaims the protected
+	 * memory (memcg_low_reclaim) to avert OOM.
+	 */
+	unsigned int memcg_low_reclaim:1;
+	unsigned int memcg_low_skipped:1;
+
+	/* Shared cgroup tree walk failed, rescan the whole tree */
+	unsigned int memcg_full_walk:1;
+
+	unsigned int hibernation_mode:1;
+
+	/* One of the zones is ready for compaction */
+	unsigned int compaction_ready:1;
+
+	/* There is easily reclaimable cold cache in the current node */
+	unsigned int cache_trim_mode:1;
+
+	/* The file folios on the current node are dangerously low */
+	unsigned int file_is_tiny:1;
+
+	/* Always discard instead of demoting to lower tier memory */
+	unsigned int no_demotion:1;
+
+	/* Allocation order */
+	s8 order;
+
+	/* Scan (total_size >> priority) pages at once */
+	s8 priority;
+
+	/* The highest zone to isolate folios for reclaim from */
+	s8 reclaim_idx;
+
+	/* This context's GFP mask */
+	gfp_t gfp_mask;
+
+	/* Incremented by the number of inactive pages that were scanned */
+	unsigned long nr_scanned;
+
+	/* Number of pages freed so far during a call to shrink_zones() */
+	unsigned long nr_reclaimed;
+
+	struct {
+		unsigned int dirty;
+		unsigned int congested;
+		unsigned int writeback;
+		unsigned int immediate;
+		unsigned int taken;
+	} nr;
+
+	/* for recording the reclaimed slab by now */
+	struct reclaim_state reclaim_state;
+};
+
+#endif /* _MM_VMSCAN_H */

Signature · 1 line

2.43.0

From: Ridong Chen <[email protected]>

Several vmscan tracepoints spell out reclaim state as individual arguments, all of which the caller already holds in struct scan_control. Now that the struct is visible from the trace header, pass sc directly and pick the fields out in TP_fast_assign instead of spelling them out at every callsite:

- mm_vmscan_direct_reclaim_begin, mm_vmscan_memcg_reclaim_begin and mm_vmscan_node_reclaim_begin: gfp_mask and order.

- mm_vmscan_lru_isolate, mm_vmscan_lru_shrink_inactive and mm_vmscan_lru_shrink_active: reclaim_idx, order and priority.

- mm_vmscan_reclaim_pages: nr_scanned. nr_reclaimed stays a separate
argument: it is the return value of shrink_folio_list(), which does not update sc->nr_reclaimed.

- mm_vmscan_balance_pgdat_end: order and nr_reclaimed. highest_zoneidx stays a separate argument: it is a balance_pgdat() parameter, and sc.reclaim_idx may differ from it by the time the end event fires.

No functional change. The tracefs event format (TP_STRUCT__entry and TP_printk) is unchanged, so tracefs-based consumers are unaffected. BPF programs attached via raw_tp/tp_btf see the new TP_PROTO and need to be adjusted; tracepoints are not a stable ABI (see Documentation/bpf/bpf_design_QA.rst).

Signed-off-by Ridong Chen <[email protected]>

From: Ridong Chen <[email protected]>

Several vmscan tracepoints spell out reclaim state as individual arguments, all of which the caller already holds in struct scan_control. Now that the struct is visible from the trace header, pass sc directly and pick the fields out in TP_fast_assign instead of spelling them out at every callsite:

- mm_vmscan_direct_reclaim_begin, mm_vmscan_memcg_reclaim_begin and mm_vmscan_node_reclaim_begin: gfp_mask and order.

- mm_vmscan_lru_isolate, mm_vmscan_lru_shrink_inactive and mm_vmscan_lru_shrink_active: reclaim_idx, order and priority.

- mm_vmscan_reclaim_pages: nr_scanned. nr_reclaimed stays a separate
argument: it is the return value of shrink_folio_list(), which does not update sc->nr_reclaimed.

- mm_vmscan_balance_pgdat_end: order and nr_reclaimed. highest_zoneidx stays a separate argument: it is a balance_pgdat() parameter, and sc.reclaim_idx may differ from it by the time the end event fires.

No functional change. The tracefs event format (TP_STRUCT__entry and TP_printk) is unchanged, so tracefs-based consumers are unaffected. BPF programs attached via raw_tp/tp_btf see the new TP_PROTO and need to be adjusted; tracepoints are not a stable ABI (see Documentation/bpf/bpf_design_QA.rst).

Signed-off-by Ridong Chen <[email protected]>

---
 mm/trace_vmscan.h | 74 +++++++++++++++++++++++------------------------
 mm/vmscan.c       | 26 ++++++++---------
 2 files changed, 48 insertions(+), 52 deletions(-)

mm/trace_vmscan.h +36 -38

index 3917e1b62dfa..b2ad83e5f25e 100644
@@ -124,10 +124,9 @@ TRACE_EVENT(mm_vmscan_balance_pgdat_begin,
 
 TRACE_EVENT(mm_vmscan_balance_pgdat_end,
 
-	TP_PROTO(int nid, int order, int highest_zoneidx,
-		 unsigned long nr_reclaimed),
+	TP_PROTO(int nid, struct scan_control *sc, int highest_zoneidx),
 
-	TP_ARGS(nid, order, highest_zoneidx, nr_reclaimed),
+	TP_ARGS(nid, sc, highest_zoneidx),
 
 	TP_STRUCT__entry(
 		__field(int, nid)

@@ -138,9 +137,9 @@ TRACE_EVENT(mm_vmscan_balance_pgdat_end,
 
 	TP_fast_assign(
 		__entry->nid = nid;
-		__entry->order = order;
+		__entry->order = sc->order;
 		__entry->highest_zoneidx = highest_zoneidx;
-		__entry->nr_reclaimed = nr_reclaimed;
+		__entry->nr_reclaimed = sc->nr_reclaimed;
 	),
 
 	TP_printk("nid=%d order=%d highest_zoneidx=%-8s nr_reclaimed=%lu",

@@ -178,9 +177,9 @@ TRACE_EVENT(mm_vmscan_wakeup_kswapd,
 
 DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_begin_template,
 
-	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
+	TP_PROTO(struct scan_control *sc, struct mem_cgroup *memcg),
 
-	TP_ARGS(gfp_flags, order, memcg),
+	TP_ARGS(sc, memcg),
 
 	TP_STRUCT__entry(
 		__field(	unsigned long,	gfp_flags	)

@@ -189,8 +188,8 @@ DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_begin_template,
 	),
 
 	TP_fast_assign(
-		__entry->gfp_flags	= (__force unsigned long)gfp_flags;
-		__entry->order		= order;
+		__entry->gfp_flags	= (__force unsigned long)sc->gfp_mask;
+		__entry->order		= sc->order;
 		__entry->memcg_id	= mem_cgroup_id(memcg);
 	),
 

@@ -204,17 +203,17 @@ DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_begin_template,
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_direct_reclaim_begin,
 
-	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
+	TP_PROTO(struct scan_control *sc, struct mem_cgroup *memcg),
 
-	TP_ARGS(gfp_flags, order, memcg)
+	TP_ARGS(sc, memcg)
 );
 
 #ifdef CONFIG_MEMCG
 DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_reclaim_begin,
 
-	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
+	TP_PROTO(struct scan_control *sc, struct mem_cgroup *memcg),
 
-	TP_ARGS(gfp_flags, order, memcg)
+	TP_ARGS(sc, memcg)
 );
 #endif /* CONFIG_MEMCG */
 

@@ -350,15 +349,14 @@ TRACE_EVENT(mm_shrink_slab_end,
 );
 
 TRACE_EVENT(mm_vmscan_lru_isolate,
-	TP_PROTO(int highest_zoneidx,
-		int order,
-		unsigned long nr_requested,
-		unsigned long nr_scanned,
-		unsigned long nr_skipped,
-		unsigned long nr_taken,
-		int lru),
+	TP_PROTO(struct scan_control *sc,
+		 unsigned long nr_requested,
+		 unsigned long nr_scanned,
+		 unsigned long nr_skipped,
+		 unsigned long nr_taken,
+		 int lru),
 
-	TP_ARGS(highest_zoneidx, order, nr_requested, nr_scanned, nr_skipped, nr_taken, lru),
+	TP_ARGS(sc, nr_requested, nr_scanned, nr_skipped, nr_taken, lru),
 
 	TP_STRUCT__entry(
 		__field(int, highest_zoneidx)

@@ -371,8 +369,8 @@ TRACE_EVENT(mm_vmscan_lru_isolate,
 	),
 
 	TP_fast_assign(
-		__entry->highest_zoneidx = highest_zoneidx;
-		__entry->order = order;
+		__entry->highest_zoneidx = sc->reclaim_idx;
+		__entry->order = sc->order;
 		__entry->nr_requested = nr_requested;
 		__entry->nr_scanned = nr_scanned;
 		__entry->nr_skipped = nr_skipped;

@@ -419,11 +417,11 @@ TRACE_EVENT(mm_vmscan_write_folio,
 
 TRACE_EVENT(mm_vmscan_reclaim_pages,
 
-	TP_PROTO(int nid,
-		unsigned long nr_scanned, unsigned long nr_reclaimed,
-		struct reclaim_stat *stat),
+	TP_PROTO(int nid, struct scan_control *sc,
+		 unsigned long nr_reclaimed,
+		 struct reclaim_stat *stat),
 
-	TP_ARGS(nid, nr_scanned, nr_reclaimed, stat),
+	TP_ARGS(nid, sc, nr_reclaimed, stat),
 
 	TP_STRUCT__entry(
 		__field(int, nid)

@@ -441,7 +439,7 @@ TRACE_EVENT(mm_vmscan_reclaim_pages,
 
 	TP_fast_assign(
 		__entry->nid = nid;
-		__entry->nr_scanned = nr_scanned;
+		__entry->nr_scanned = sc->nr_scanned;
 		__entry->nr_reclaimed = nr_reclaimed;
 		__entry->nr_dirty = stat->nr_dirty;
 		__entry->nr_writeback = stat->nr_writeback;

@@ -466,9 +464,9 @@ TRACE_EVENT(mm_vmscan_lru_shrink_inactive,
 
 	TP_PROTO(int nid,
 		unsigned long nr_scanned, unsigned long nr_reclaimed,
-		struct reclaim_stat *stat, int priority, int file),
+		struct reclaim_stat *stat, struct scan_control *sc, int file),
 
-	TP_ARGS(nid, nr_scanned, nr_reclaimed, stat, priority, file),
+	TP_ARGS(nid, nr_scanned, nr_reclaimed, stat, sc, file),
 
 	TP_STRUCT__entry(
 		__field(int, nid)

@@ -498,7 +496,7 @@ TRACE_EVENT(mm_vmscan_lru_shrink_inactive,
 		__entry->nr_activate1 = stat->nr_activate[1];
 		__entry->nr_ref_keep = stat->nr_ref_keep;
 		__entry->nr_unmap_fail = stat->nr_unmap_fail;
-		__entry->priority = priority;
+		__entry->priority = sc->priority;
 		__entry->reclaim_flags = trace_reclaim_flags(file);
 	),
 

@@ -517,9 +515,9 @@ TRACE_EVENT(mm_vmscan_lru_shrink_active,
 
 	TP_PROTO(int nid, unsigned long nr_taken,
 		unsigned long nr_active, unsigned long nr_deactivated,
-		unsigned long nr_referenced, int priority, int file),
+		unsigned long nr_referenced, struct scan_control *sc, int file),
 
-	TP_ARGS(nid, nr_taken, nr_active, nr_deactivated, nr_referenced, priority, file),
+	TP_ARGS(nid, nr_taken, nr_active, nr_deactivated, nr_referenced, sc, file),
 
 	TP_STRUCT__entry(
 		__field(int, nid)

@@ -537,7 +535,7 @@ TRACE_EVENT(mm_vmscan_lru_shrink_active,
 		__entry->nr_active = nr_active;
 		__entry->nr_deactivated = nr_deactivated;
 		__entry->nr_referenced = nr_referenced;
-		__entry->priority = priority;
+		__entry->priority = sc->priority;
 		__entry->reclaim_flags = trace_reclaim_flags(file);
 	),
 

@@ -551,9 +549,9 @@ TRACE_EVENT(mm_vmscan_lru_shrink_active,
 
 TRACE_EVENT(mm_vmscan_node_reclaim_begin,
 
-	TP_PROTO(int nid, int order, gfp_t gfp_flags),
+	TP_PROTO(int nid, struct scan_control *sc),
 
-	TP_ARGS(nid, order, gfp_flags),
+	TP_ARGS(nid, sc),
 
 	TP_STRUCT__entry(
 		__field(int, nid)

@@ -563,8 +561,8 @@ TRACE_EVENT(mm_vmscan_node_reclaim_begin,
 
 	TP_fast_assign(
 		__entry->nid = nid;
-		__entry->order = order;
-		__entry->gfp_flags = (__force unsigned long)gfp_flags;
+		__entry->order = sc->order;
+		__entry->gfp_flags = (__force unsigned long)sc->gfp_mask;
 	),
 
 	TP_printk("nid=%d order=%d gfp_flags=%s",

mm/vmscan.c +12 -14

index 49677464fc9a..15e9184a580a 100644
@@ -1764,7 +1764,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
 		}
 	}
 	*nr_scanned = total_scan;
-	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
+	trace_mm_vmscan_lru_isolate(sc, nr_to_scan,
 				    total_scan, skipped, nr_taken, lru);
 	update_lru_sizes(lruvec, lru, nr_zone_taken);
 	return nr_taken;

@@ -2053,7 +2053,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
 
 	handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
+			nr_scanned, nr_reclaimed, &stat, sc, file);
 	return nr_reclaimed;
 }
 

@@ -2160,7 +2160,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
 		mod_lruvec_state(lruvec, PGROTATE_ANON + file, nr_rotated);
 
 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
-			nr_deactivate, nr_rotated, sc->priority, file);
+			nr_deactivate, nr_rotated, sc, file);
 }
 
 static unsigned int reclaim_folio_list(struct list_head *folio_list,

@@ -2183,7 +2183,7 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list,
 		list_del(&folio->lru);
 		folio_putback_lru(folio);
 	}
-	trace_mm_vmscan_reclaim_pages(pgdat->node_id, sc.nr_scanned, nr_reclaimed, &stat);
+	trace_mm_vmscan_reclaim_pages(pgdat->node_id, &sc, nr_reclaimed, &stat);
 
 	return nr_reclaimed;
 }

@@ -4849,9 +4849,9 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	mod_lruvec_state(lruvec, item, isolated);
 	mod_lruvec_state(lruvec, PGREFILL, sorted);
 	mod_lruvec_state(lruvec, PGSCAN_ANON + type, isolated);
-	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
-				scanned, skipped, isolated,
-				type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
+	trace_mm_vmscan_lru_isolate(sc, nr_to_scan,
+				    scanned, skipped, isolated,
+				    type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
 	*isolatedp = isolated;
 	return scanned;

@@ -4993,7 +4993,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	if (isolated)
 		handle_reclaim_writeback(isolated, pgdat, sc, &stat);
 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-			type_scanned, reclaimed, &stat, sc->priority,
+			type_scanned, reclaimed, &stat, sc,
 			type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
 	list_for_each_entry_safe_reverse(folio, next, &list, lru) {

@@ -6889,7 +6889,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 		return 1;
 
 	set_task_reclaim_state(current, &sc.reclaim_state);
-	trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, NULL);
+	trace_mm_vmscan_direct_reclaim_begin(&sc, NULL);
 
 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
 

@@ -6930,7 +6930,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
 	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
 
 	set_task_reclaim_state(current, &sc.reclaim_state);
-	trace_mm_vmscan_memcg_reclaim_begin(sc.gfp_mask, 0, memcg);
+	trace_mm_vmscan_memcg_reclaim_begin(&sc, memcg);
 	noreclaim_flag = memalloc_noreclaim_save();
 
 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);

@@ -7407,8 +7407,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 	psi_memstall_leave(&pflags);
 	set_task_reclaim_state(current, NULL);
 
-	trace_mm_vmscan_balance_pgdat_end(pgdat->node_id, sc.order,
-					  highest_zoneidx, sc.nr_reclaimed);
+	trace_mm_vmscan_balance_pgdat_end(pgdat->node_id, &sc, highest_zoneidx);
 
 	/*
 	 * Return the order kswapd stopped reclaiming at as

@@ -7886,8 +7885,7 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat,
 	unsigned int noreclaim_flag;
 	unsigned long pflags;
 
-	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, sc->order,
-					   sc->gfp_mask);
+	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, sc);
 
 	cond_resched();
 	if (!sc->proactive)

Signature · 1 line

2.43.0

From: Ridong Chen <[email protected]>

MGLRU's scan_folios() emits the classic-LRU tracepoint trace_mm_vmscan_lru_isolate(), which predates MGLRU. It reports the scan/isolate counts and the LRU type, but carries no generation or memcg context, so a trace of an MGLRU run cannot tell which memcg a given scan belongs to, nor how far reclaim has progressed through the generations.

Add mm_mglru_scan_folios next to it, reporting the same counters plus nr_sorted (folios moved to a younger generation by sort_folio()) and the context the classic tracepoint lacks: the node id and memcg id of the lruvec, and the max_seq, tier and min_seq of the type being scanned.

Now that struct scan_control is visible to the trace header, pass sc, lruvec, type and tier straight in and derive everything else in TP_fast_assign(), so nothing is computed at the call site when the event is disabled. The int fields are grouped at the end of the entry to avoid padding on 64-bit.

scan_folios() is the MGLRU-specific layer where folios are actually scanned, so it has no classic-LRU counterpart. The existing tracepoint is left unchanged.

Signed-off-by Ridong Chen <[email protected]>

---

From: Ridong Chen <[email protected]>

MGLRU's scan_folios() emits the classic-LRU tracepoint trace_mm_vmscan_lru_isolate(), which predates MGLRU. It reports the scan/isolate counts and the LRU type, but carries no generation or memcg context, so a trace of an MGLRU run cannot tell which memcg a given scan belongs to, nor how far reclaim has progressed through the generations.

Add mm_mglru_scan_folios next to it, reporting the same counters plus nr_sorted (folios moved to a younger generation by sort_folio()) and the context the classic tracepoint lacks: the node id and memcg id of the lruvec, and the max_seq, tier and min_seq of the type being scanned.

Now that struct scan_control is visible to the trace header, pass sc, lruvec, type and tier straight in and derive everything else in TP_fast_assign(), so nothing is computed at the call site when the event is disabled. The int fields are grouped at the end of the entry to avoid padding on 64-bit.

scan_folios() is the MGLRU-specific layer where folios are actually scanned, so it has no classic-LRU counterpart. The existing tracepoint is left unchanged.

Signed-off-by Ridong Chen <[email protected]>

---
 mm/trace_vmscan.h | 66 +++++++++++++++++++++++++++++++++++++++++++++++
 mm/vmscan.c       |  2 ++
 2 files changed, 68 insertions(+)

mm/trace_vmscan.h +16 -0

index b2ad83e5f25e..8148b8224422 100644
@@ -392,6 +392,72 @@ TRACE_EVENT(mm_vmscan_lru_isolate,
 		__print_symbolic(__entry->lru, LRU_NAMES))
 );
 
+#ifdef CONFIG_LRU_GEN
+TRACE_EVENT(mm_mglru_scan_folios,
+
+	TP_PROTO(struct lruvec *lruvec,
+		 struct scan_control *sc,
+		 unsigned long nr_requested,
+		 unsigned long nr_scanned,
+		 unsigned long nr_sorted,
+		 unsigned long nr_skipped,
+		 unsigned long nr_taken,
+		 int type,
+		 int tier),
+
+	TP_ARGS(lruvec, sc, nr_requested, nr_scanned, nr_sorted, nr_skipped,
+		nr_taken, type, tier),
+

mm/trace_vmscan.h +40 -0

+		__field(int, highest_zoneidx)
+		__field(int, order)
+		__field(int, lru)
+		__field(int, tier)
+	),
+
+	TP_fast_assign(
+		__entry->memcg_id = mem_cgroup_id(lruvec_memcg(lruvec));
+		__entry->nr_requested = nr_requested;
+		__entry->nr_scanned = nr_scanned;
+		__entry->nr_sorted = nr_sorted;
+		__entry->nr_skipped = nr_skipped;
+		__entry->nr_taken = nr_taken;
+		__entry->max_seq = lruvec->lrugen.max_seq;
+		__entry->min_seq = lruvec->lrugen.min_seq[type];
+		__entry->nid = lruvec_pgdat(lruvec)->node_id;
+		__entry->highest_zoneidx = sc->reclaim_idx;
+		__entry->order = sc->order;
+		__entry->lru = type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
+		__entry->tier = tier;
+	),
+
+	TP_printk("nid=%d memcg_id=%llu classzone=%d order=%d nr_requested=%lu nr_scanned=%lu nr_sorted=%lu nr_skipped=%lu nr_taken=%lu lru=%s max_seq=%lu tier=%d min_seq=%lu",
+		__entry->nid,
+		__entry->memcg_id,
+		__entry->highest_zoneidx,
+		__entry->order,
+		__entry->nr_requested,
+		__entry->nr_scanned,
+		__entry->nr_sorted,
+		__entry->nr_skipped,
+		__entry->nr_taken,
+		__print_symbolic(__entry->lru, LRU_NAMES),
+		__entry->max_seq,
+		__entry->tier,
+		__entry->min_seq)
+);
+
+#endif /* CONFIG_LRU_GEN */
+
 TRACE_EVENT(mm_vmscan_write_folio,
 
 	TP_PROTO(struct folio *folio),

mm/vmscan.c +2 -0

index 15e9184a580a..ff51fee872f8 100644
@@ -4852,6 +4852,8 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	trace_mm_vmscan_lru_isolate(sc, nr_to_scan,
 				    scanned, skipped, isolated,
 				    type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
+	trace_mm_mglru_scan_folios(lruvec, sc, nr_to_scan, scanned, sorted,
+				   skipped, isolated, type, tier);
 
 	*isolatedp = isolated;
 	return scanned;

Signature · 1 line

2.43.0

From: Ridong Chen <[email protected]>

Aging in MGLRU advances max_seq via inc_max_seq(), creating a new youngest generation. There is currently no tracepoint on this path, so the moment a new generation is created, and how the min_seq of each type trails behind it, cannot be observed as it happens.

Add mm_mglru_inc_max_seq, emitted right after max_seq is bumped while the lruvec lock is still held, with the node id, memcg id, the new max_seq, the anon and file min_seq, and the number of pages in each generation for both types, summed over zones and clamped to >= 0 the same way the debugfs lru_gen file reports them. The nr_anon/nr_file arrays are indexed by generation slot, so the emitted max_seq/min_seq say which slot holds which seq. They are printed with __print_array(), so the output is hex and its length follows MAX_NR_GENS automatically.

Only the lruvec is passed in; all of the above is collected in TP_fast_assign(), so inc_max_seq() does no extra work when the event is disabled. Paired with mm_mglru_scan_folios it makes the full aging-to-eviction window observable per memcg.

Signed-off-by Ridong Chen <[email protected]>

---
 mm/trace_vmscan.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 mm/vmscan.c       |  2 ++
 2 files changed, 49 insertions(+)

mm/trace_vmscan.h +25 -0

index 8148b8224422..cddf42b5849a 100644
@@ -456,6 +456,53 @@ TRACE_EVENT(mm_mglru_scan_folios,
                __entry->min_seq)
 );
 
+TRACE_EVENT(mm_mglru_inc_max_seq,
+
+       TP_PROTO(struct lruvec *lruvec),
+
+       TP_ARGS(lruvec),
+
+       TP_STRUCT__entry(
+               __field(u64, memcg_id)
+               __field(unsigned long, max_seq)
+               __field(unsigned long, anon_min_seq)
+               __field(unsigned long, file_min_seq)
+               __array(unsigned long, nr_anon, MAX_NR_GENS)
+               __array(unsigned long, nr_file, MAX_NR_GENS)
+               __field(int, nid)
+       ),
+
+       TP_fast_assign(
+               struct lru_gen_folio *lrugen = &lruvec->lrugen;
+               int gen, zone;
+
+               __entry->memcg_id = mem_cgroup_id(lruvec_memcg(lruvec));
+               __entry->max_seq = lrugen->max_seq;
+               __entry->anon_min_seq = lrugen->min_seq[LRU_GEN_ANON];
+               __entry->file_min_seq = lrugen->min_seq[LRU_GEN_FILE];
+               /* summed over zones and clamped to >= 0, as debugfs lru_gen does */

Quoted 67 lines · Ridong Chen

From: Ridong Chen <[email protected]>

Aging in MGLRU advances max_seq via inc_max_seq(), creating a new youngest generation. There is currently no tracepoint on this path, so the moment a new generation is created, and how the min_seq of each type trails behind it, cannot be observed as it happens.

Add mm_mglru_inc_max_seq, emitted right after max_seq is bumped while the lruvec lock is still held, with the node id, memcg id, the new max_seq, the anon and file min_seq, and the number of pages in each generation for both types, summed over zones and clamped to >= 0 the same way the debugfs lru_gen file reports them. The nr_anon/nr_file arrays are indexed by generation slot, so the emitted max_seq/min_seq say which slot holds which seq. They are printed with __print_array(), so the output is hex and its length follows MAX_NR_GENS automatically.

Only the lruvec is passed in; all of the above is collected in TP_fast_assign(), so inc_max_seq() does no extra work when the event is disabled. Paired with mm_mglru_scan_folios it makes the full aging-to-eviction window observable per memcg.

Signed-off-by Ridong Chen <[email protected]>

---
 mm/trace_vmscan.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 mm/vmscan.c       |  2 ++
 2 files changed, 49 insertions(+)

mm/trace_vmscan.h +31 -0

index 8148b8224422..cddf42b5849a 100644
@@ -456,6 +456,53 @@ TRACE_EVENT(mm_mglru_scan_folios,
                __entry->min_seq)
 );
 
+TRACE_EVENT(mm_mglru_inc_max_seq,
+
+       TP_PROTO(struct lruvec *lruvec),
+
+       TP_ARGS(lruvec),
+
+       TP_STRUCT__entry(
+               __field(u64, memcg_id)
+               __field(unsigned long, max_seq)
+               __field(unsigned long, anon_min_seq)
+               __field(unsigned long, file_min_seq)
+               __array(unsigned long, nr_anon, MAX_NR_GENS)
+               __array(unsigned long, nr_file, MAX_NR_GENS)
+               __field(int, nid)
+       ),
+
+       TP_fast_assign(
+               struct lru_gen_folio *lrugen = &lruvec->lrugen;
+               int gen, zone;
+
+               __entry->memcg_id = mem_cgroup_id(lruvec_memcg(lruvec));
+               __entry->max_seq = lrugen->max_seq;
+               __entry->anon_min_seq = lrugen->min_seq[LRU_GEN_ANON];
+               __entry->file_min_seq = lrugen->min_seq[LRU_GEN_FILE];
+               /* summed over zones and clamped to >= 0, as debugfs lru_gen does */
+               for (gen = 0; gen < MAX_NR_GENS; gen++) {
+                       __entry->nr_anon[gen] = 0;
+                       __entry->nr_file[gen] = 0;
+                       for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+                               __entry->nr_anon[gen] +=
+                                       max(lrugen->nr_pages[gen][LRU_GEN_ANON][zone], 0L);

mm/trace_vmscan.h TRACE_EVENT(mm_mglru_scan_folios, · line 493

Quoted 4 lines

mm/trace_vmscan.h +4 -0

+                               __entry->nr_file[gen] +=
+                                       max(lrugen->nr_pages[gen][LRU_GEN_FILE][zone], 0L);
+                       }
+               }

I feel this is wrong. Can we calculate the newest and oldest generations from seq, then print them in order from newest to oldest?

The nr_pages[] array isn't ordered from newest to oldest or vice versa, as we have:

static inline int lru_gen_from_seq(unsigned long seq)

{
        return seq % MAX_NR_GENS;
}

And I added a helper here:
https://lore.kernel.org/linux-mm/[email protected]/

(quoted diff) +10 -0

+static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
+ int type, unsigned long seq)
+{
+     int gen = lru_gen_from_seq(seq);
+     unsigned long size = 0;
+
+     for (int zone = 0; zone < MAX_NR_ZONES; zone++)
+          size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+     return size;
+}

Maybe you could port it here as a separate patch so it can land earlier as part of your tracepoints series. We could then use it in the next version of the swappiness series.

Best,
Barry

Quoted 68 lines · Barry Song

Quoted 67 lines · Ridong Chen

From: Ridong Chen <[email protected]>

Aging in MGLRU advances max_seq via inc_max_seq(), creating a new youngest generation. There is currently no tracepoint on this path, so the moment a new generation is created, and how the min_seq of each type trails behind it, cannot be observed as it happens.

Add mm_mglru_inc_max_seq, emitted right after max_seq is bumped while the lruvec lock is still held, with the node id, memcg id, the new max_seq, the anon and file min_seq, and the number of pages in each generation for both types, summed over zones and clamped to >= 0 the same way the debugfs lru_gen file reports them. The nr_anon/nr_file arrays are indexed by generation slot, so the emitted max_seq/min_seq say which slot holds which seq. They are printed with __print_array(), so the output is hex and its length follows MAX_NR_GENS automatically.

Only the lruvec is passed in; all of the above is collected in TP_fast_assign(), so inc_max_seq() does no extra work when the event is disabled. Paired with mm_mglru_scan_folios it makes the full aging-to-eviction window observable per memcg.

Signed-off-by Ridong Chen <[email protected]>

---
  mm/trace_vmscan.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++
  mm/vmscan.c       |  2 ++
  2 files changed, 49 insertions(+)

mm/trace_vmscan.h +31 -0

index 8148b8224422..cddf42b5849a 100644
@@ -456,6 +456,53 @@ TRACE_EVENT(mm_mglru_scan_folios,
                 __entry->min_seq)
  );
 
+TRACE_EVENT(mm_mglru_inc_max_seq,
+
+       TP_PROTO(struct lruvec *lruvec),
+
+       TP_ARGS(lruvec),
+
+       TP_STRUCT__entry(
+               __field(u64, memcg_id)
+               __field(unsigned long, max_seq)
+               __field(unsigned long, anon_min_seq)
+               __field(unsigned long, file_min_seq)
+               __array(unsigned long, nr_anon, MAX_NR_GENS)
+               __array(unsigned long, nr_file, MAX_NR_GENS)
+               __field(int, nid)
+       ),
+
+       TP_fast_assign(
+               struct lru_gen_folio *lrugen = &lruvec->lrugen;
+               int gen, zone;
+
+               __entry->memcg_id = mem_cgroup_id(lruvec_memcg(lruvec));
+               __entry->max_seq = lrugen->max_seq;
+               __entry->anon_min_seq = lrugen->min_seq[LRU_GEN_ANON];
+               __entry->file_min_seq = lrugen->min_seq[LRU_GEN_FILE];
+               /* summed over zones and clamped to >= 0, as debugfs lru_gen does */
+               for (gen = 0; gen < MAX_NR_GENS; gen++) {
+                       __entry->nr_anon[gen] = 0;
+                       __entry->nr_file[gen] = 0;
+                       for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+                               __entry->nr_anon[gen] +=
+                                       max(lrugen->nr_pages[gen][LRU_GEN_ANON][zone], 0L);

Quoted 8 lines

mm/trace_vmscan.h TRACE_EVENT(mm_mglru_scan_folios, · line 493

Quoted 4 lines

mm/trace_vmscan.h +4 -0

+                               __entry->nr_file[gen] +=
+                                       max(lrugen->nr_pages[gen][LRU_GEN_FILE][zone], 0L);
+                       }
+               }

I feel this is wrong. Can we calculate the newest and oldest generations from seq, then print them in order from newest to oldest?

Agree, from newest to oldest or oldest to newest makes it clearer.

Printing from lrugen->max_seq - MAX_NR_GENS + 1, or from MAX_NR_GENS, both looks good.

Quoted 23 lines

The nr_pages[] array isn't ordered from newest to oldest or vice versa, as we have:

static inline int lru_gen_from_seq(unsigned long seq)

{
         return seq % MAX_NR_GENS;
}

And I added a helper here:
https://lore.kernel.org/linux-mm/[email protected]/

(quoted diff) +10 -0

+static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
+ int type, unsigned long seq)
+{
+     int gen = lru_gen_from_seq(seq);
+     unsigned long size = 0;
+
+     for (int zone = 0; zone < MAX_NR_ZONES; zone++)
+          size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+     return size;
+}

Quoted 5 lines

Maybe you could port it here as a separate patch so it can land earlier as part of your tracepoints series. We could then use it in the next version of the swappiness series.