mm/mglru: performance optimizations
Subsystems (4)
v3 cover mm/mglru: performance optimizations
This series improves performance for some previously reported test cases. Most of the code changes gathered here has been floating on the mailing list [1][2]. They are now properly organized and have gone through various benchmarks on client and server devices, including Android, FIO, memcached, multiple VMs and MongoDB.
In addition to the warning [3] fixed in v2, this version fixes another warning [4] reported by syzbot.
[1] https://lore.kernel.org/CAOUHufahuWcKf5f1Sg3emnqX+cODuR=2TQo7T4Gr-QYLujn4RA@mail.gmail.com/
[2] https://lore.kernel.org/CAOUHufawNerxqLm7L9Yywp3HJFiYVrYO26ePUb1jH-qxNGWzyA@mail.gmail.com/
[3] https://lore.kernel.org/[email protected]/
[4] https://lore.kernel.org/[email protected]/
Yu Zhao (6):
mm/mglru: clean up workingset
mm/mglru: optimize deactivation
mm/mglru: rework aging feedback
mm/mglru: rework type selection
mm/mglru: rework refault detection
mm/mglru: rework workingset protection
include/linux/mm_inline.h | 94 +++++---- include/linux/mmzone.h | 88 +++++--- mm/swap.c | 70 +++++-- mm/vmscan.c | 426 +++++++++++++++++--------------------- mm/workingset.c | 67 +++--- 5 files changed, 375 insertions(+), 370 deletions(-)
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 1/6 mm/mglru: clean up workingset
Move VM_BUG_ON_FOLIO() to cover both the default and MGLRU paths. Also use a pair of rcu_read_lock() and rcu_read_unlock() within each path, to improve readability.
This change should not have any side effects.
--- mm/workingset.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)
mm/workingset.c +12 -13
@@ -428,17 +428,17 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset,struct pglist_data *pgdat;unsigned long eviction; -rcu_read_lock (); -if (lru_gen_enabled ()) { -bool recent =lru_gen_test_recent (shadow, file, - &eviction_lruvec, &eviction, workingset); +bool recent; +rcu_read_lock (); + recent =lru_gen_test_recent (shadow, file, &eviction_lruvec, + &eviction, workingset);rcu_read_unlock ();return recent; } - +rcu_read_lock ();unpack_shadow (shadow, &memcgid, &pgdat, &eviction, workingset); eviction <<= bucket_order;
@@ -459,14 +459,12 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset,* configurations instead. */ eviction_memcg =mem_cgroup_from_id (memcgid); -if (!mem_cgroup_disabled () && - (!eviction_memcg || !mem_cgroup_tryget (eviction_memcg))) { -rcu_read_unlock (); +if (!mem_cgroup_tryget (eviction_memcg)) + eviction_memcg =NULL ; +rcu_read_unlock (); + +if (!mem_cgroup_disabled () && !eviction_memcg)return false; - } - -rcu_read_unlock (); -/* * Flush stats (and potentially sleep) outside the RCU read section. *
@@ -544,6 +542,8 @@ void workingset_refault(struct folio *folio, void *shadow)bool workingset;long nr; +VM_BUG_ON_FOLIO (!folio_test_locked (folio), folio); +if (lru_gen_enabled ()) {lru_gen_refault (folio, shadow);return ;
@@ -558,7 +558,6 @@ void workingset_refault(struct folio *folio, void *shadow)* is actually experiencing the refault event. Make sure the folio is * locked to guarantee folio_memcg() stability throughout. */ -VM_BUG_ON_FOLIO (!folio_test_locked (folio), folio); nr =folio_nr_pages (folio); memcg =folio_memcg (folio); pgdat =folio_pgdat (folio);
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 2/6 mm/mglru: optimize deactivation
In reply to Yu ZhaoDo not shuffle a folio in the deactivation paths if it is already in the oldest generation. This reduces the LRU lock contention.
Before this patch, the contention is reproducible by FIO, e.g.,
fio -filename=/dev/nvme1n1p2 -direct=0 -thread -size=1024G \
-rwmixwrite=30 --norandommap --randrepeat=0 -ioengine=sync \
-bs=4k -numjobs=400 -runtime=25000 --time_based \
-group_reporting -name=mglru98.96%--_raw_spin_lock_irqsave
folio_lruvec_lock_irqsave
|
--98.78%--folio_batch_move_lru
|
--98.63%--deactivate_file_folio
mapping_try_invalidate
invalidate_mapping_pages
invalidate_bdev
blkdev_common_ioctl
blkdev_ioctlAfter this patch, deactivate_file_folio() bails out early without taking the LRU lock.
A side effect is that a folio can be left at the head of the oldest generation, rather than the tail. If reclaim happens at the same time, it cannot reclaim this folio immediately. Since there is no known correlation between truncation and reclaim, this side effect is considered insignificant.
--- mm/swap.c | 49 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-)
mm/swap.c +41 -8
@@ -379,11 +379,12 @@ static void __lru_cache_activate_folio(struct folio *folio) }#ifdef CONFIG_LRU_GEN -static void folio_inc_refs (struct folio *folio) + +static void lru_gen_inc_refs (struct folio *folio) {unsigned long new_flags, old_flags =READ_ONCE (folio->flags); -if (folio_test_unevictable (folio)) +if (!folio_test_lru (folio) ||folio_test_unevictable (folio))return ;if (!folio_test_referenced (folio)) {
@@ -406,10 +407,33 @@ static void folio_inc_refs(struct folio *folio) new_flags |= old_flags & ~LRU_REFS_MASK; }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); } -#else -static void folio_inc_refs (struct folio *folio) + +static bool lru_gen_clear_refs (struct folio *folio) { +struct lru_gen_folio *lrugen; +int type =folio_is_file_lru (folio); + +if (!folio_test_lru (folio) ||folio_test_unevictable (folio)) +return true; + +set_mask_bits (&folio->flags,LRU_REFS_MASK |LRU_REFS_FLAGS ,0 ); + + lrugen = &folio_lruvec (folio)->lrugen; +/* whether can do without shuffling under the LRU lock */ +return folio_lru_gen (folio) ==lru_gen_from_seq (READ_ONCE (lrugen->min_seq[type])); } + +#else /* !CONFIG_LRU_GEN */ + +static void lru_gen_inc_refs (struct folio *folio) +{ +} + +static bool lru_gen_clear_refs (struct folio *folio) +{ +return false; +} +#endif /* CONFIG_LRU_GEN */ /**
@@ -428,7 +452,7 @@ static void folio_inc_refs(struct folio *folio)void folio_mark_accessed (struct folio *folio) {if (lru_gen_enabled ()) { -folio_inc_refs (folio); +lru_gen_inc_refs (folio);return ; }
@@ -524,7 +548,7 @@ void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)*/ static void lru_deactivate_file (struct lruvec *lruvec,struct folio *folio) { -bool active =folio_test_active (folio); +bool active =folio_test_active (folio) ||lru_gen_enabled ();long nr_pages =folio_nr_pages (folio);if (folio_test_unevictable (folio))
@@ -589,7 +613,10 @@ static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)lruvec_del_folio (lruvec, folio);folio_clear_active (folio); -folio_clear_referenced (folio); +if (lru_gen_enabled ()) +lru_gen_clear_refs (folio); +else +folio_clear_referenced (folio);/* * Lazyfree folios are clean anonymous folios. They have * the swapbacked flag cleared, to distinguish them from normal
@@ -657,6 +684,9 @@ void deactivate_file_folio(struct folio *folio)if (folio_test_unevictable (folio))return ; +if (lru_gen_enabled () &&lru_gen_clear_refs (folio)) +return ; +folio_batch_add_and_move (folio, lru_deactivate_file, true); }
@@ -670,7 +700,10 @@ void deactivate_file_folio(struct folio *folio)*/ void folio_deactivate (struct folio *folio) { -if (folio_test_unevictable (folio) || !(folio_test_active (folio) ||lru_gen_enabled ())) +if (folio_test_unevictable (folio)) +return ; + +if (lru_gen_enabled () ?lru_gen_clear_refs (folio) : !folio_test_active (folio))return ;folio_batch_add_and_move (folio, lru_deactivate, true);
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 3/6 mm/mglru: rework aging feedback
In reply to Yu ZhaoThe aging feedback is based on both the number of generations and the distribution of folios in each generation. The number of generations is currently the distance between max_seq and anon min_seq. This is because anon min_seq is not allowed to move past file min_seq. The rationale for that is that file is always evictable whereas anon is not. However, for use cases where anon is a lot cheaper than file:
1. Anon in the second oldest generation can be a better choice than file in the oldest generation.
2. A large amount of file in the oldest generation can skew the distribution, making should_run_aging() return false negative.
Allow anon and file min_seq to move independently, and use solely the number of generations as the feedback for aging. Specifically, when both anon and file are evictable, anon min_seq can now be greater than file min_seq, and therefore the number of generations becomes the distance between max_seq and min(min_seq[0],min_seq[1]). And should_run_aging() returns true if and only if the number of generations is less than MAX_NR_GENS.
As the first step to the final optimization, this change by itself should not have userspace-visiable effects beyond performance. The next twos patch will take advantage of this change; the last patch in this series will better distribute folios across MAX_NR_GENS.
--- include/linux/mmzone.h | 6 +- mm/vmscan.c | 217 +++++++++++++++++------------------------ 2 files changed, 91 insertions(+), 132 deletions(-)
include/linux/mmzone.h +3 -3
@@ -446,8 +446,8 @@ struct lru_gen_folio {unsigned long avg_refaulted[ANON_AND_FILE ][MAX_NR_TIERS ];/* the exponential moving average of evicted+protected */ unsigned long avg_total[ANON_AND_FILE ][MAX_NR_TIERS ]; -/* the first tier doesn't need protection, hence the minus one */ -unsigned long protected[NR_HIST_GENS ][ANON_AND_FILE ][MAX_NR_TIERS -1 ]; +/* can only be modified under the LRU lock */ +unsigned long protected[NR_HIST_GENS ][ANON_AND_FILE ][MAX_NR_TIERS ];/* can be modified without holding the LRU lock */ atomic_long_t evicted[NR_HIST_GENS ][ANON_AND_FILE ][MAX_NR_TIERS ];atomic_long_t refaulted[NR_HIST_GENS ][ANON_AND_FILE ][MAX_NR_TIERS ];
@@ -498,7 +498,7 @@ struct lru_gen_mm_walk {int mm_stats[NR_MM_STATS ];/* total batched items */ int batched; -bool can_swap; +int swappiness;bool force_scan; };
mm/vmscan.c +88 -129
519 lines not on this page · raw message
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 4/6 mm/mglru: rework type selection
In reply to Yu ZhaoWith anon and file min_seq being able to move independently, rework type selection so that it is based on the total refaults from all tiers of each type. Also allow a type to be selected until that type reaches MIN_NR_GENS, and therefore abs_diff(min_seq[0],min_seq[1]) now can be 2 (MAX_NR_GENS-MIN_NR_GENS) instead of 1.
Since some tiers of a selected type can have higher refaults than the first tier of the other type, use a less larger gain factor 2:3 instead of 1:2, in order for those tiers in the selected type to be better protected.
As an intermediate step to the final optimization, this change by itself should not have userspace-visiable effects beyond performance.
--- mm/vmscan.c | 82 +++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 56 deletions(-)
mm/vmscan.c +26 -56
@@ -3089,15 +3089,20 @@ struct ctrl_pos {static void read_ctrl_pos (struct lruvec *lruvec,int type,int tier,int gain,struct ctrl_pos *pos) { +int i;struct lru_gen_folio *lrugen = &lruvec->lrugen;int hist =lru_hist_from_seq (lrugen->min_seq[type]); - pos->refaulted = lrugen->avg_refaulted[type][tier] + -atomic_long_read (&lrugen->refaulted[hist][type][tier]); - pos->total = lrugen->avg_total[type][tier] + - lrugen->protected[hist][type][tier] + -atomic_long_read (&lrugen->evicted[hist][type][tier]); pos->gain = gain; + pos->refaulted = pos->total =0 ; + +for (i = tier %MAX_NR_TIERS ; i <=min (tier,MAX_NR_TIERS -1 ); i++) { + pos->refaulted += lrugen->avg_refaulted[type][i] + +atomic_long_read (&lrugen->refaulted[hist][type][i]); + pos->total += lrugen->avg_total[type][i] + + lrugen->protected[hist][type][i] + +atomic_long_read (&lrugen->evicted[hist][type][i]); + } }static void reset_ctrl_pos (struct lruvec *lruvec,int type,bool carryover)
@@ -4480,13 +4485,13 @@ static int get_tier_idx(struct lruvec *lruvec, int type)struct ctrl_pos sp, pv;/* -* To leave a margin for fluctuations, use a larger gain factor (1:2). +* To leave a margin for fluctuations, use a larger gain factor (2:3). * This value is chosen because any other tier would have at least twice * as many refaults as the first tier. */ -read_ctrl_pos (lruvec, type,0 ,1 , &sp); +read_ctrl_pos (lruvec, type,0 ,2 , &sp);for (tier =1 ; tier <MAX_NR_TIERS ; tier++) { -read_ctrl_pos (lruvec, type, tier,2 , &pv); +read_ctrl_pos (lruvec, type, tier,3 , &pv);if (!positive_ctrl_err (&sp, &pv))break ; }
@@ -4494,68 +4499,34 @@ static int get_tier_idx(struct lruvec *lruvec, int type)return tier -1 ; } -static int get_type_to_scan (struct lruvec *lruvec,int swappiness,int *tier_idx) +static int get_type_to_scan (struct lruvec *lruvec,int swappiness) { -int type, tier;struct ctrl_pos sp, pv; -int gain[ANON_AND_FILE ] = { swappiness,MAX_SWAPPINESS - swappiness }; +if (!swappiness) +return LRU_GEN_FILE ; + +if (swappiness ==MAX_SWAPPINESS ) +return LRU_GEN_ANON ;/* -* Compare the first tier of anon with that of file to determine which -* type to scan. Also need to compare other tiers of the selected type -* with the first tier of the other type to determine the last tier (of -* the selected type) to evict. +* Compare the sum of all tiers of anon with that of file to determine +* which type to scan. */ -read_ctrl_pos (lruvec,LRU_GEN_ANON ,0 , gain[LRU_GEN_ANON ], &sp); -read_ctrl_pos (lruvec,LRU_GEN_FILE ,0 , gain[LRU_GEN_FILE ], &pv); - type =positive_ctrl_err (&sp, &pv); +read_ctrl_pos (lruvec,LRU_GEN_ANON ,MAX_NR_TIERS , swappiness, &sp); +read_ctrl_pos (lruvec,LRU_GEN_FILE ,MAX_NR_TIERS ,MAX_SWAPPINESS - swappiness, &pv); -read_ctrl_pos (lruvec, !type,0 , gain[!type], &sp); -for (tier =1 ; tier <MAX_NR_TIERS ; tier++) { -read_ctrl_pos (lruvec, type, tier, gain[type], &pv); -if (!positive_ctrl_err (&sp, &pv)) -break ; - } - - *tier_idx = tier -1 ; - -return type; +return positive_ctrl_err (&sp, &pv); }static int isolate_folios (struct lruvec *lruvec,struct scan_control *sc,int swappiness,int *type_scanned,struct list_head *list) {int i; -int type; -int tier =-1 ; -DEFINE_MIN_SEQ (lruvec); - -/* -* Try to make the obvious choice first, and if anon and file are both -* available from the same generation, -* 1. Interpret swappiness 1 as file first and MAX_SWAPPINESS as anon -* first. -* 2. If !__GFP_IO, file first since clean pagecache is more likely to -* exist than clean swapcache. -*/ -if (!swappiness) - type =LRU_GEN_FILE ; -else if (min_seq[LRU_GEN_ANON ] < min_seq[LRU_GEN_FILE ]) - type =LRU_GEN_ANON ; -else if (swappiness ==1 ) - type =LRU_GEN_FILE ; -else if (swappiness ==MAX_SWAPPINESS ) - type =LRU_GEN_ANON ; -else if (!(sc->gfp_mask & __GFP_IO)) - type =LRU_GEN_FILE ; -else - type =get_type_to_scan (lruvec, swappiness, &tier); +int type =get_type_to_scan (lruvec, swappiness);for_each_evictable_type (i, swappiness) {int scanned; - -if (tier <0 ) - tier =get_tier_idx (lruvec, type); +int tier =get_tier_idx (lruvec, type); *type_scanned = type;
@@ -4564,7 +4535,6 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swreturn scanned; type = !type; - tier =-1 ; }return 0 ;
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 5/6 mm/mglru: rework refault detection
In reply to Yu ZhaoWith anon and file min_seq being able to move independently, rework workingset protection as well so that the comparison of refaults between anon and file is always on an equal footing.
Specifically, make lru_gen_test_recent() return true for refaults happening within the distance of MAX_NR_GENS. For example, if min_seq of a type is max_seq-MIN_NR_GENS, refaults from min_seq-1, i.e., max_seq-MIN_NR_GENS-1, are also considered recent, since the distance max_seq-(max_seq-MIN_NR_GENS-1), i.e., MIN_NR_GENS+1 is less than MAX_NR_GENS.
As an intermediate step to the final optimization, this change by itself should not have userspace-visiable effects beyond performance.
--- mm/workingset.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-)
mm/workingset.c +9 -8
@@ -260,11 +260,11 @@ static void *lru_gen_eviction(struct folio *folio)* Tests if the shadow entry is for a folio that was recently evicted. * Fills in @lruvec, @token, @workingset with the values unpacked from shadow. */ -static bool lru_gen_test_recent (void *shadow,bool file,struct lruvec **lruvec, +static bool lru_gen_test_recent (void *shadow,struct lruvec **lruvec,unsigned long *token,bool *workingset) {int memcg_id; -unsigned long min_seq; +unsigned long max_seq;struct mem_cgroup *memcg;struct pglist_data *pgdat;
@@ -273,8 +273,10 @@ static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, memcg =mem_cgroup_from_id (memcg_id); *lruvec =mem_cgroup_lruvec (memcg, pgdat); - min_seq =READ_ONCE ((*lruvec)->lrugen.min_seq[file]); -return (*token >>LRU_REFS_WIDTH ) == (min_seq & (EVICTION_MASK >>LRU_REFS_WIDTH )); + max_seq =READ_ONCE ((*lruvec)->lrugen.max_seq); + max_seq &=EVICTION_MASK >>LRU_REFS_WIDTH ; + +return abs_diff (max_seq, *token >>LRU_REFS_WIDTH ) <MAX_NR_GENS ; }static void lru_gen_refault (struct folio *folio,void *shadow)
@@ -290,7 +292,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)rcu_read_lock (); - recent =lru_gen_test_recent (shadow, type, &lruvec, &token, &workingset); + recent =lru_gen_test_recent (shadow, &lruvec, &token, &workingset);if (lruvec !=folio_lruvec (folio)) goto unlock;
@@ -331,7 +333,7 @@ static void *lru_gen_eviction(struct folio *folio)return NULL ; } -static bool lru_gen_test_recent (void *shadow,bool file,struct lruvec **lruvec, +static bool lru_gen_test_recent (void *shadow,struct lruvec **lruvec,unsigned long *token,bool *workingset) {return false;
@@ -432,8 +434,7 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset,bool recent;rcu_read_lock (); - recent =lru_gen_test_recent (shadow, file, &eviction_lruvec, - &eviction, workingset); + recent =lru_gen_test_recent (shadow, &eviction_lruvec, &eviction, workingset);rcu_read_unlock ();return recent; }
Signature · 1 line
2.47.0.338.g60cca15819-goog
PATCH v3 6/6 mm/mglru: rework workingset protection
In reply to Yu ZhaoWith the aging feedback no longer considering the distribution of folios in each generation, rework workingset protection to better distribute folios across MAX_NR_GENS. This is achieved by reusing PG_workingset and PG_referenced/LRU_REFS_FLAGS in a slightly different way.
For folios accessed multiple times through file descriptors, make lru_gen_inc_refs() set additional bits of LRU_REFS_WIDTH in folio->flags after PG_referenced, then PG_workingset after LRU_REFS_WIDTH. After all its bits are set, i.e., LRU_REFS_FLAGS|BIT(PG_workingset), a folio is lazily promoted into the second oldest generation in the eviction path. And when folio_inc_gen() does that, it clears LRU_REFS_FLAGS so that lru_gen_inc_refs() can start over. For this case, LRU_REFS_MASK is only valid when PG_referenced is set.
For folios accessed multiple times through page tables, folio_update_gen() from a page table walk or lru_gen_set_refs() from a rmap walk sets PG_referenced after the accessed bit is cleared for the first time. Thereafter, those two paths set PG_workingset and promote folios to the youngest generation. Like folio_inc_gen(), when folio_update_gen() does that, it also clears PG_referenced. For this case, LRU_REFS_MASK is not used.
For both of the cases, after PG_workingset is set on a folio, it remains until this folio is either reclaimed, or "deactivated" by lru_gen_clear_refs(). It can be set again if lru_gen_test_recent() returns true upon a refault.
When adding folios to the LRU lists, lru_gen_distance() distributes them as follows:
+---------------------------------+---------------------------------+ | Accessed thru page tables | Accessed thru file descriptors | +---------------------------------+---------------------------------+ | PG_active (set while isolated) | | +----------------+----------------+----------------+----------------+ | PG_workingset | PG_referenced | PG_workingset | LRU_REFS_FLAGS | +---------------------------------+---------------------------------+ |<--------- MIN_NR_GENS --------->| | |<-------------------------- MAX_NR_GENS -------------------------->|
After this patch, some typical client and server workloads showed improvements under heavy memory pressure. For example, Python TPC-C, which was used to benchmark a different approach [1] to better detect refault distances, showed a significant decrease in total refaults:
Before After Change Time (seconds) 10801 10801 0% Executed (transactions) 41472 43663 +5% workingset_nodes 109070 120244 +10% workingset_refault_anon 5019627 7281831 +45% workingset_refault_file 1294678786 554855564 -57% workingset_refault_total 1299698413 562137395 -57%
[1] https://lore.kernel.org/[email protected]/
--- include/linux/mm_inline.h | 94 ++++++++++++------------ include/linux/mmzone.h | 82 +++++++++++++-------- mm/swap.c | 23 +++--- mm/vmscan.c | 145 ++++++++++++++++++++++---------------- mm/workingset.c | 29 ++++---- 5 files changed, 208 insertions(+), 165 deletions(-)
include/linux/mm_inline.h +50 -44
@@ -133,31 +133,25 @@ static inline int lru_hist_from_seq(unsigned long seq)return seq %NR_HIST_GENS ; } -static inline int lru_tier_from_refs (int refs) +static inline int lru_tier_from_refs (int refs,bool workingset) {VM_WARN_ON_ONCE (refs >BIT (LRU_REFS_WIDTH )); -/* see the comment in folio_lru_refs() */ -return order_base_2 (refs +1 ); +/* see the comment on MAX_NR_TIERS */ +return workingset ?MAX_NR_TIERS -1 :order_base_2 (refs); }static inline int folio_lru_refs (struct folio *folio) {unsigned long flags =READ_ONCE (folio->flags); -bool workingset = flags &BIT (PG_workingset); +if (!(flags &BIT (PG_referenced))) +return 0 ;/* -* Return the number of accesses beyond PG_referenced, i.e., N-1 if the -* total number of accesses is N>1, since N=0,1 both map to the first -* tier. lru_tier_from_refs() will account for this off-by-one. Also see -* the comment on MAX_NR_TIERS. +* Return the total number of accesses including PG_referenced. Also see +* the comment on LRU_REFS_FLAGS. */ -return ((flags &LRU_REFS_MASK ) >>LRU_REFS_PGOFF ) + workingset; -} - -static inline void folio_clear_lru_refs (struct folio *folio) -{ -set_mask_bits (&folio->flags,LRU_REFS_MASK |LRU_REFS_FLAGS ,0 ); +return ((flags &LRU_REFS_MASK ) >>LRU_REFS_PGOFF ) +1 ; }static inline int folio_lru_gen (struct folio *folio)
@@ -223,11 +217,46 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foliVM_WARN_ON_ONCE (lru_gen_is_active (lruvec, old_gen) && !lru_gen_is_active (lruvec, new_gen)); } +static inline int lru_gen_distance (struct folio *folio,bool reclaiming) +{ +/* +* Distance until eviction (larger values provide stronger protection): +* +-------------------------------------+-------------------------------------+ +* | Accessed through page tables and | Accessed through file descriptors | +* | promoted by folio_update_gen() | and protected by folio_inc_gen() | +* +-------------------------------------+-------------------------------------+ +* | PG_active (only set while isolated) | | +* +------------------+------------------+------------------+------------------+ +* | PG_workingset | PG_referenced | PG_workingset | LRU_REFS_FLAGS | +* +-------------------------------------+-------------------------------------+ +* | 3 | 2 | 1 | 0 | +* +-------------------------------------+-------------------------------------+ +* |<----------- MIN_NR_GENS ----------->| | +* |<------------------------------ MAX_NR_GENS ------------------------------>| +*/ +if (reclaiming) +return 0 ; + +if (folio_test_active (folio)) +return MIN_NR_GENS +folio_test_workingset (folio); + +if (folio_test_workingset (folio)) +return MIN_NR_GENS -1 ; + +if (!folio_is_file_lru (folio) && !folio_test_swapcache (folio)) +return MIN_NR_GENS -1 ; + +if (folio_test_reclaim (folio) && (folio_test_dirty (folio) ||folio_test_writeback (folio))) +return MIN_NR_GENS -1 ; + +return 0 ; +} +static inline bool lru_gen_add_folio (struct lruvec *lruvec,struct folio *folio,bool reclaiming) { +int dist;unsigned long seq;unsigned long flags; -unsigned long mask;int gen =folio_lru_gen (folio);int type =folio_is_file_lru (folio);int zone =folio_zonenum (folio);
@@ -237,40 +266,17 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,if (folio_test_unevictable (folio) || !lrugen->enabled)return false; -/* -* There are four common cases for this page: -* 1. If it's hot, i.e., freshly faulted in, add it to the youngest -* generation, and it's protected over the rest below. -* 2. If it can't be evicted immediately, i.e., a dirty page pending -* writeback, add it to the second youngest generation. -* 3. If it should be evicted first, e.g., cold and clean from -* folio_rotate_reclaimable(), add it to the oldest generation. -* 4. Everything else falls between 2 & 3 above and is added to the -* second oldest generation if it's considered inactive, or the -* oldest generation otherwise. See lru_gen_is_active(). -*/ -if (folio_test_active (folio)) - seq = lrugen->max_seq; -else if ((type ==LRU_GEN_ANON && !folio_test_swapcache (folio)) || - (folio_test_reclaim (folio) && - (folio_test_dirty (folio) ||folio_test_writeback (folio)))) - seq = lrugen->max_seq -1 ; -else if (reclaiming || lrugen->min_seq[type] +MIN_NR_GENS >= lrugen->max_seq) - seq = lrugen->min_seq[type]; + + dist =lru_gen_distance (folio, reclaiming); +if (dist <MIN_NR_GENS ) + seq = lrugen->min_seq[type] + dist;else - seq = lrugen->min_seq[type] +1 ; + seq = lrugen->max_seq + dist -MIN_NR_GENS -1 ; gen =lru_gen_from_seq (seq); flags = (gen +1UL ) <<LRU_GEN_PGOFF ;/* see the comment on MIN_NR_GENS about PG_active */ - mask =LRU_GEN_MASK ; -/* -* Don't clear PG_workingset here because it can affect PSI accounting -* if the activation is due to workingset refault. -*/ -if (folio_test_active (folio)) - mask |=LRU_REFS_MASK |BIT (PG_referenced) |BIT (PG_active); -set_mask_bits (&folio->flags, mask, flags); +set_mask_bits (&folio->flags,LRU_GEN_MASK |BIT (PG_active), flags);lru_gen_update_size (lruvec, folio,-1 , gen);/* for folio_rotate_reclaimable() */
include/linux/mmzone.h +51 -31
125 lines not on this page · raw message
mm/swap.c +9 -14
@@ -387,24 +387,19 @@ static void lru_gen_inc_refs(struct folio *folio)if (!folio_test_lru (folio) ||folio_test_unevictable (folio))return ; +/* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced (folio)) { -folio_set_referenced (folio); +set_mask_bits (&folio->flags,LRU_REFS_MASK ,BIT (PG_referenced));return ; } -if (!folio_test_workingset (folio)) { -folio_set_workingset (folio); -return ; - } - -/* see the comment on MAX_NR_TIERS */ do { - new_flags = old_flags &LRU_REFS_MASK ; -if (new_flags ==LRU_REFS_MASK ) -break ; +if ((old_flags &LRU_REFS_MASK ) ==LRU_REFS_MASK ) { +folio_set_workingset (folio); +return ; + } - new_flags +=BIT (LRU_REFS_PGOFF ); - new_flags |= old_flags & ~LRU_REFS_MASK ; + new_flags = old_flags +BIT (LRU_REFS_PGOFF ); }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); }
@@ -416,7 +411,7 @@ static bool lru_gen_clear_refs(struct folio *folio)if (!folio_test_lru (folio) ||folio_test_unevictable (folio))return true; -set_mask_bits (&folio->flags,LRU_REFS_MASK |LRU_REFS_FLAGS ,0 ); +set_mask_bits (&folio->flags,LRU_REFS_FLAGS |BIT (PG_workingset),0 ); lrugen = &folio_lruvec (folio)->lrugen;/* whether can do without shuffling under the LRU lock */
@@ -498,7 +493,7 @@ void folio_add_lru(struct folio *folio) folio_test_unevictable(folio), folio);VM_BUG_ON_FOLIO (folio_test_lru (folio), folio); -/* see the comment in lru_gen_add_folio() */ +/* see the comment in lru_gen_distance() */ if (lru_gen_enabled () && !folio_test_unevictable (folio) &&lru_gen_in_fault () && !(current->flags &PF_MEMALLOC ))folio_set_active (folio);
mm/vmscan.c +85 -60
246 lines not on this page · raw message
mm/workingset.c +13 -16
@@ -239,7 +239,8 @@ static void *lru_gen_eviction(struct folio *folio)int type =folio_is_file_lru (folio);int delta =folio_nr_pages (folio);int refs =folio_lru_refs (folio); -int tier =lru_tier_from_refs (refs); +bool workingset =folio_test_workingset (folio); +int tier =lru_tier_from_refs (refs, workingset);struct mem_cgroup *memcg =folio_memcg (folio);struct pglist_data *pgdat =folio_pgdat (folio);
@@ -253,7 +254,7 @@ static void *lru_gen_eviction(struct folio *folio) hist =lru_hist_from_seq (min_seq);atomic_long_add (delta, &lrugen->evicted[hist][type][tier]); -return pack_shadow (mem_cgroup_id (memcg), pgdat, token, refs); +return pack_shadow (mem_cgroup_id (memcg), pgdat, token, workingset); }/*
@@ -304,24 +305,20 @@ static void lru_gen_refault(struct folio *folio, void *shadow) lrugen = &lruvec->lrugen; hist =lru_hist_from_seq (READ_ONCE (lrugen->min_seq[type])); -/* see the comment in folio_lru_refs() */ - refs = (token & (BIT (LRU_REFS_WIDTH ) -1 )) + workingset; - tier =lru_tier_from_refs (refs); + refs = (token & (BIT (LRU_REFS_WIDTH ) -1 )) +1 ; + tier =lru_tier_from_refs (refs, workingset);atomic_long_add (delta, &lrugen->refaulted[hist][type][tier]); -mod_lruvec_state (lruvec,WORKINGSET_ACTIVATE_BASE + type, delta); -/* -* Count the following two cases as stalls: -* 1. For pages accessed through page tables, hotter pages pushed out -* hot pages which refaulted immediately. -* 2. For pages accessed multiple times through file descriptors, -* they would have been protected by sort_folio(). -*/ -if (lru_gen_in_fault () || refs >=BIT (LRU_REFS_WIDTH ) -1 ) { -set_mask_bits (&folio->flags,0 ,LRU_REFS_MASK |BIT (PG_workingset)); +/* see folio_add_lru() where folio_set_active() happens */ +if (lru_gen_in_fault ()) +mod_lruvec_state (lruvec,WORKINGSET_ACTIVATE_BASE + type, delta); + +if (workingset) { +folio_set_workingset (folio);mod_lruvec_state (lruvec,WORKINGSET_RESTORE_BASE + type, delta); - } + }else +set_mask_bits (&folio->flags,LRU_REFS_MASK , (refs -1UL ) <<LRU_REFS_PGOFF ); unlock:rcu_read_unlock (); }
Signature · 1 line
2.47.0.338.g60cca15819-goog
kernel test robot
build botRe: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection
kernel test robot noticed a 5.7% regression of will-it-scale.per_process_ops on:
Hello,
kernel test robot noticed a 5.7% regression of will-it-scale.per_process_ops on:
commit: 3b7734aa8458b62ecbfd785ca7918e831565006e ("[PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection")
url: https://github.com/intel-lab-lkp/linux/commits/Yu-Zhao/mm-mglru-clean-up-workingset/20241208-061714
base: v6.13-rc1
patch link: https://lore.kernel.org/all/[email protected]/
patch subject: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection
testcase: will-it-scale
config: x86_64-rhel-9.4
compiler: gcc-12
test machine: 104 threads 2 sockets (Skylake) with 192G memory parameters:
nr_task: 100% mode: process test: pread2 cpufreq_governor: performance
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-lkp/[email protected]
Details are as below:
-------------------------------------------------------------------------------------------------->
The kernel config and materials to reproduce are available at:
https://download.01.org/0day-ci/archive/20241223/[email protected]
=========================================================================================
compiler/cpufreq_governor/kconfig/mode/nr_task/rootfs/tbox_group/test/testcase:
gcc-12/performance/x86_64-rhel-9.4/process/100%/debian-12-x86_64-20240206.cgz/lkp-skl-fpga01/pread2/will-it-scale
commit:
4a202aca7c ("mm/mglru: rework refault detection") 3b7734aa84 ("mm/mglru: rework workingset protection")
4a202aca7c7d9f99 3b7734aa8458b62ecbfd785ca79
---------------- ---------------------------
%stddev %change %stddev
\ | \
1.03 ± 3% -0.1 0.92 ± 5% mpstat.cpu.all.usr%
0.29 ± 14% +20.8% 0.35 ± 7% perf-sched.sch_delay.avg.ms.schedule_timeout.__wait_for_common.wait_for_completion_state.kernel_clone
1.02 ± 21% +50.7% 1.54 ± 23% perf-sched.sch_delay.max.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
0.01 ± 50% -66.9% 0.00 ± 82% perf-stat.i.major-faults
0.01 ± 50% -73.6% 0.00 ±112% perf-stat.ps.major-faults
335982 -60.7% 132060 ± 15% proc-vmstat.nr_active_anon
335982 -60.7% 132060 ± 15% proc-vmstat.nr_zone_active_anon
1343709 -60.7% 528460 ± 15% meminfo.Active
1343709 -60.7% 528460 ± 15% meminfo.Active(anon)
259.96 +3.2e+05% 821511 ± 11% meminfo.Inactive
1401961 -5.7% 1321692 ± 2% will-it-scale.104.processes
13479 -5.7% 12708 ± 2% will-it-scale.per_process_ops
1401961 -5.7% 1321692 ± 2% will-it-scale.workload
138691 ± 43% -75.8% 33574 ± 55% numa-vmstat.node0.nr_active_anon
138691 ± 43% -75.8% 33574 ± 55% numa-vmstat.node0.nr_zone_active_anon
197311 ± 30% -50.1% 98494 ± 18% numa-vmstat.node1.nr_active_anon
197311 ± 30% -50.1% 98494 ± 18% numa-vmstat.node1.nr_zone_active_anon
554600 ± 43% -75.8% 134360 ± 55% numa-meminfo.node0.Active
554600 ± 43% -75.8% 134360 ± 55% numa-meminfo.node0.Active(anon)
173.31 ± 70% +1.4e+05% 247821 ± 50% numa-meminfo.node0.Inactive
789291 ± 30% -50.1% 394029 ± 18% numa-meminfo.node1.Active
789291 ± 30% -50.1% 394029 ± 18% numa-meminfo.node1.Active(anon)
86.66 ±141% +6.6e+05% 573998 ± 27% numa-meminfo.node1.Inactive
38.95 -0.9 38.09 perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read
38.83 -0.9 37.97 perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock_irq.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter
39.70 -0.8 38.86 perf-profile.calltrace.cycles-pp.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64
41.03 -0.8 40.26 perf-profile.calltrace.cycles-pp.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64.do_syscall_64
0.91 +0.0 0.95 perf-profile.calltrace.cycles-pp.filemap_get_entry.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64
53.14 +0.5 53.66 perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock_irqsave.folio_wake_bit.shmem_file_read_iter.vfs_read
53.24 +0.5 53.76 perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.folio_wake_bit.shmem_file_read_iter.vfs_read.__x64_sys_pread64
53.84 +0.5 54.38 perf-profile.calltrace.cycles-pp.folio_wake_bit.shmem_file_read_iter.vfs_read.__x64_sys_pread64.do_syscall_64
38.96 -0.9 38.09 perf-profile.children.cycles-pp._raw_spin_lock_irq
39.71 -0.8 38.87 perf-profile.children.cycles-pp.folio_wait_bit_common
41.04 -0.8 40.26 perf-profile.children.cycles-pp.shmem_get_folio_gfp
92.00 -0.3 91.67 perf-profile.children.cycles-pp.native_queued_spin_lock_slowpath
0.22 -0.0 0.18 ± 3% perf-profile.children.cycles-pp._copy_to_iter
0.22 ± 2% -0.0 0.19 ± 2% perf-profile.children.cycles-pp.copy_page_to_iter
0.20 ± 2% -0.0 0.16 ± 4% perf-profile.children.cycles-pp.rep_movs_alternative
0.91 +0.0 0.96 perf-profile.children.cycles-pp.filemap_get_entry
0.00 +0.3 0.35 perf-profile.children.cycles-pp.folio_mark_accessed
53.27 +0.5 53.80 perf-profile.children.cycles-pp._raw_spin_lock_irqsave
53.86 +0.5 54.40 perf-profile.children.cycles-pp.folio_wake_bit
92.00 -0.3 91.67 perf-profile.self.cycles-pp.native_queued_spin_lock_slowpath
0.19 -0.0 0.16 ± 3% perf-profile.self.cycles-pp.rep_movs_alternative
0.41 +0.0 0.44 perf-profile.self.cycles-pp.shmem_get_folio_gfp 0.37 ± 2% +0.0 0.40 perf-profile.self.cycles-pp.folio_wait_bit_common
0.90 +0.0 0.94 perf-profile.self.cycles-pp.filemap_get_entry
0.61 +0.1 0.68 perf-profile.self.cycles-pp.shmem_file_read_iter
0.00 +0.3 0.34 ± 2% perf-profile.self.cycles-pp.folio_mark_accessed
Disclaimer:
Results have been estimated based on internal Intel analysis and are provided for informational purposes only. Any difference in system hardware or software design or configuration may affect actual performance.
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection
Quoted 5 lines · kernel test robot
Hello,
kernel test robot noticed a 5.7% regression of will-it-scale.per_process_ops on:
Thanks, Oliver!
Quoted 12 lines
commit:
3b7734aa8458b62ecbfd785ca7918e831565006e("[PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection")
url: https://github.com/intel-lab-lkp/linux/commits/Yu-Zhao/mm-mglru-clean-up-workingset/20241208-061714
base: v6.13-rc1
patch link: https://lore.kernel.org/all/[email protected]/
patch subject: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protectiontestcase: will-it-scale
config: x86_64-rhel-9.4
compiler: gcc-12
test machine: 104 threads 2 sockets (Skylake) with 192G memory parameters:
Quoted 4 lines
nr_task: 100% mode: process test: pread2 cpufreq_governor: performance
I think this is very likely caused by my change to folio_mark_accessed() that unncessarily dirties cache lines shared between different cores.
Could you try the following fix please?
mm/swap.c +2 -1
@@ -395,7 +395,8 @@ static void lru_gen_inc_refs(struct folio *folio) do { if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) { - folio_set_workingset(folio); + if (!folio_test_workingset(folio)) + folio_set_workingset(folio); return; }
Oliver Sang
Tested-byRe: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection
hi, Yu Zhao,
Quoted 26 lines · Yu Zhao
Quoted 5 lines · kernel test robot
Hello,
kernel test robot noticed a 5.7% regression of will-it-scale.per_process_ops on:
Thanks, Oliver!
Quoted 12 lines
commit:
3b7734aa8458b62ecbfd785ca7918e831565006e("[PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protection")
url: https://github.com/intel-lab-lkp/linux/commits/Yu-Zhao/mm-mglru-clean-up-workingset/20241208-061714
base: v6.13-rc1
patch link: https://lore.kernel.org/all/[email protected]/
patch subject: [PATCH mm-unstable v3 6/6] mm/mglru: rework workingset protectiontestcase: will-it-scale
config: x86_64-rhel-9.4
compiler: gcc-12
test machine: 104 threads 2 sockets (Skylake) with 192G memory parameters:Quoted 4 lines
nr_task: 100% mode: process test: pread2 cpufreq_governor: performance
Quoted 4 lines
I think this is very likely caused by my change to
folio_mark_accessed()that unncessarily dirties cache lines shared between different cores.Could you try the following fix please?
yes, this patch can recover the performance fully (as below (1)). thanks!
=========================================================================================
compiler/cpufreq_governor/kconfig/mode/nr_task/rootfs/tbox_group/test/testcase:
gcc-12/performance/x86_64-rhel-9.4/process/100%/debian-12-x86_64-20240206.cgz/lkp-skl-fpga01/pread2/will-it-scale
commit:
4a202aca7c ("mm/mglru: rework refault detection") 3b7734aa84 ("mm/mglru: rework workingset protection") c5346da9fe <-- fix patch from you
4a202aca7c7d9f99 3b7734aa8458b62ecbfd785ca79 c5346da9fe00d3b303057d93fd9 ---------------- --------------------------- ---------------------------
%stddev %change %stddev %change %stddev
\ | \ | \
1.03 ± 3% -0.1 0.92 ± 5% -0.0 0.98 ± 6% mpstat.cpu.all.usr%
144371 -0.5% 143667 ± 2% -2.0% 141486 vmstat.system.in
335982 -60.7% 132060 ± 15% -61.7% 128640 ± 14% proc-vmstat.nr_active_anon
335982 -60.7% 132060 ± 15% -61.7% 128640 ± 14% proc-vmstat.nr_zone_active_anon
1343709 -60.7% 528460 ± 15% -61.7% 514494 ± 14% meminfo.Active
1343709 -60.7% 528460 ± 15% -61.7% 514494 ± 14% meminfo.Active(anon)
259.96 +3.2e+05% 821511 ± 11% +3.2e+05% 829732 ± 9% meminfo.Inactive
1401961 -5.7% 1321692 ± 2% -0.1% 1399905 will-it-scale.104.processes
13479 -5.7% 12708 ± 2% -0.1% 13460 will-it-scale.per_process_ops <----- (1)
1401961 -5.7% 1321692 ± 2% -0.1% 1399905 will-it-scale.workload
138691 ± 43% -75.8% 33574 ± 55% -54.9% 62588 ± 61% numa-vmstat.node0.nr_active_anon
138691 ± 43% -75.8% 33574 ± 55% -54.9% 62588 ± 61% numa-vmstat.node0.nr_zone_active_anon
197311 ± 30% -50.1% 98494 ± 18% -66.5% 66034 ± 50% numa-vmstat.node1.nr_active_anon
197311 ± 30% -50.1% 98494 ± 18% -66.5% 66034 ± 50% numa-vmstat.node1.nr_zone_active_anon
0.29 ± 14% +20.8% 0.35 ± 7% -14.6% 0.25 ± 31% perf-sched.sch_delay.avg.ms.schedule_timeout.__wait_for_common.wait_for_completion_state.kernel_clone
1.02 ± 21% +50.7% 1.54 ± 23% -10.2% 0.92 ± 19% perf-sched.sch_delay.max.ms.schedule_timeout.rcu_gp_fqs_loop.rcu_gp_kthread.kthread
476.63 ± 10% -12.7% 415.87 ± 28% -31.2% 327.79 ± 35% perf-sched.wait_and_delay.avg.ms.schedule_hrtimeout_range.ep_poll.do_epoll_wait.__x64_sys_epoll_wait
476.50 ± 10% -12.7% 415.80 ± 28% -31.2% 327.69 ± 35% perf-sched.wait_time.avg.ms.schedule_hrtimeout_range.ep_poll.do_epoll_wait.__x64_sys_epoll_wait
554600 ± 43% -75.8% 134360 ± 55% -54.8% 250416 ± 61% numa-meminfo.node0.Active
554600 ± 43% -75.8% 134360 ± 55% -54.8% 250416 ± 61% numa-meminfo.node0.Active(anon)
173.31 ± 70% +1.4e+05% 247821 ± 50% +1.9e+05% 338038 ± 45% numa-meminfo.node0.Inactive
789291 ± 30% -50.1% 394029 ± 18% -66.5% 264180 ± 50% numa-meminfo.node1.Active
789291 ± 30% -50.1% 394029 ± 18% -66.5% 264180 ± 50% numa-meminfo.node1.Active(anon)
86.66 ±141% +6.6e+05% 573998 ± 27% +5.7e+05% 491639 ± 33% numa-meminfo.node1.Inactive
2.657e+09 -2.2% 2.598e+09 ± 2% -2.4% 2.592e+09 ± 2% perf-stat.i.branch-instructions
1.156e+10 -2.3% 1.13e+10 ± 2% -2.5% 1.127e+10 ± 2% perf-stat.i.instructions
0.01 ± 50% -66.9% 0.00 ± 82% -72.9% 0.00 ±110% perf-stat.i.major-faults
2.648e+09 -18.7% 2.152e+09 ± 44% -2.4% 2.584e+09 ± 2% perf-stat.ps.branch-instructions
1.152e+10 -18.8% 9.358e+09 ± 44% -2.5% 1.123e+10 ± 2% perf-stat.ps.instructions
0.01 ± 50% -73.6% 0.00 ±112% -72.8% 0.00 ±110% perf-stat.ps.major-faults
38.95 -0.9 38.09 +0.0 38.96 perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read
38.83 -0.9 37.97 +0.0 38.84 perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock_irq.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter
39.70 -0.8 38.86 +0.0 39.71 perf-profile.calltrace.cycles-pp.folio_wait_bit_common.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64
41.03 -0.8 40.26 +0.0 41.04 perf-profile.calltrace.cycles-pp.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64.do_syscall_64
0.91 +0.0 0.95 -0.0 0.91 ± 2% perf-profile.calltrace.cycles-pp.filemap_get_entry.shmem_get_folio_gfp.shmem_file_read_iter.vfs_read.__x64_sys_pread64
53.14 +0.5 53.66 -0.0 53.13 perf-profile.calltrace.cycles-pp.native_queued_spin_lock_slowpath._raw_spin_lock_irqsave.folio_wake_bit.shmem_file_read_iter.vfs_read
53.24 +0.5 53.76 -0.0 53.23 perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.folio_wake_bit.shmem_file_read_iter.vfs_read.__x64_sys_pread64
53.84 +0.5 54.38 -0.0 53.82 perf-profile.calltrace.cycles-pp.folio_wake_bit.shmem_file_read_iter.vfs_read.__x64_sys_pread64.do_syscall_64
38.96 -0.9 38.09 +0.0 38.96 perf-profile.children.cycles-pp._raw_spin_lock_irq
39.71 -0.8 38.87 +0.0 39.72 perf-profile.children.cycles-pp.folio_wait_bit_common
41.04 -0.8 40.26 +0.0 41.05 perf-profile.children.cycles-pp.shmem_get_folio_gfp
92.00 -0.3 91.67 -0.0 92.00 perf-profile.children.cycles-pp.native_queued_spin_lock_slowpath
0.22 -0.0 0.18 ± 3% -0.0 0.22 ± 3% perf-profile.children.cycles-pp._copy_to_iter
0.22 ± 2% -0.0 0.19 ± 2% -0.0 0.22 ± 2% perf-profile.children.cycles-pp.copy_page_to_iter
0.20 ± 2% -0.0 0.16 ± 4% -0.0 0.19 ± 2% perf-profile.children.cycles-pp.rep_movs_alternative
0.91 +0.0 0.96 -0.0 0.91 ± 2% perf-profile.children.cycles-pp.filemap_get_entry
0.00 +0.3 0.35 +0.0 0.01 ±299% perf-profile.children.cycles-pp.folio_mark_accessed
53.27 +0.5 53.80 -0.0 53.26 perf-profile.children.cycles-pp._raw_spin_lock_irqsave
53.86 +0.5 54.40 -0.0 53.84 perf-profile.children.cycles-pp.folio_wake_bit
92.00 -0.3 91.67 -0.0 92.00 perf-profile.self.cycles-pp.native_queued_spin_lock_slowpath
0.19 -0.0 0.16 ± 3% +0.0 0.19 ± 2% perf-profile.self.cycles-pp.rep_movs_alternative
0.41 +0.0 0.44 +0.0 0.41 ± 3% perf-profile.self.cycles-pp.shmem_get_folio_gfp
0.37 ± 2% +0.0 0.40 +0.0 0.38 ± 2% perf-profile.self.cycles-pp.folio_wait_bit_common
0.90 +0.0 0.94 -0.0 0.90 ± 2% perf-profile.self.cycles-pp.filemap_get_entry
0.61 +0.1 0.68 +0.0 0.61 ± 2% perf-profile.self.cycles-pp.shmem_file_read_iter
0.00 +0.3 0.34 ± 2% +0.0 0.00 perf-profile.self.cycles-pp.folio_mark_accessedQuoted 15 lines
mm/swap.c +2
-1@@ -395,7 +395,8 @@ static void lru_gen_inc_refs(struct folio *folio) do { if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) { - folio_set_workingset(folio); + if (!folio_test_workingset(folio)) + folio_set_workingset(folio); return; }