mm/mglru: code cleanup and
Subsystems (4)
cover mm/mglru: code cleanup and
This series is a follow up to the comment:
https://lore.kernel.org/[email protected]/
This provides MGLRU code cleanup and refactoring for better readability.
Kinsey Ho (5):
mm/mglru: drop unused parameter
mm/mglru: improve should_run_aging()
mm/mglru: improve reset_mm_stats()
mm/mglru: improve struct lru_gen_mm_walk
mm/mglru: improve swappiness handling
include/linux/mmzone.h | 6 +- mm/vmscan.c | 142 ++++++++++++++++++++--------------------- 2 files changed, 74 insertions(+), 74 deletions(-)
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
PATCH 1/5 mm/mglru: drop unused parameter
struct scan_control *sc is currently passed into try_to_inc_max_seq() and run_aging(). This parameter is not used.
Drop the unused parameter struct scan_control *sc. No functional change.
--- mm/vmscan.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
mm/vmscan.c +5 -5
@@ -3834,7 +3834,7 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, }static bool try_to_inc_max_seq (struct lruvec *lruvec,unsigned long max_seq, -struct scan_control *sc,bool can_swap,bool force_scan) +bool can_swap,bool force_scan) {bool success;struct lru_gen_mm_walk *walk;
@@ -4690,7 +4690,7 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, boolreturn nr_to_scan;/* skip this lruvec as it's low on cold folios */ -return try_to_inc_max_seq (lruvec, max_seq, sc, can_swap, false) ?-1 :0 ; +return try_to_inc_max_seq (lruvec, max_seq, can_swap, false) ?-1 :0 ; }static bool should_abort_scan (struct lruvec *lruvec,struct scan_control *sc)
@@ -5350,7 +5350,7 @@ static const struct seq_operations lru_gen_seq_ops = { .show = lru_gen_seq_show, }; -static int run_aging (struct lruvec *lruvec,unsigned long seq,struct scan_control *sc, +static int run_aging (struct lruvec *lruvec,unsigned long seq,bool can_swap,bool force_scan) {DEFINE_MAX_SEQ (lruvec);
@@ -5365,7 +5365,7 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_contrif (!force_scan && min_seq[!can_swap] +MAX_NR_GENS -1 <= max_seq)return -ERANGE ; -try_to_inc_max_seq (lruvec, max_seq, sc, can_swap, force_scan); +try_to_inc_max_seq (lruvec, max_seq, can_swap, force_scan);return 0 ; }
@@ -5433,7 +5433,7 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,switch (cmd) {case '+' : - err =run_aging (lruvec, seq, sc, swappiness, opt); + err =run_aging (lruvec, seq, swappiness, opt);break ;case '-' : err =run_eviction (lruvec, seq, sc, swappiness, opt);
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
PATCH 2/5 mm/mglru: improve should_run_aging()
In reply to Kinsey Hoscan_control *sc does not need to be passed into should_run_aging(), as it provides only the reclaim priority. This can be moved to get_nr_to_scan().
Refactor should_run_aging() and get_nr_to_scan() to improve code readability. No functional changes.
--- mm/vmscan.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
mm/vmscan.c +11 -14
@@ -4602,14 +4602,13 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap }static bool should_run_aging (struct lruvec *lruvec,unsigned long max_seq, -struct scan_control *sc,bool can_swap,unsigned long *nr_to_scan) +bool can_swap,unsigned long *nr_to_scan) {int gen, type, zone;unsigned long old =0 ;unsigned long young =0 ;unsigned long total =0 ;struct lru_gen_folio *lrugen = &lruvec->lrugen; -struct mem_cgroup *memcg =lruvec_memcg (lruvec);DEFINE_MIN_SEQ (lruvec);/* whether this lruvec is completely out of cold folios */
@@ -4637,13 +4636,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, } } -/* try to scrape all its memory if this memcg was deleted */ -if (!mem_cgroup_online (memcg)) { - *nr_to_scan = total; -return false; - } - - *nr_to_scan = total >> sc->priority; + *nr_to_scan = total;/* * The aging tries to be lazy to reduce the overhead, while the eviction
@@ -4675,6 +4668,7 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,*/ static long get_nr_to_scan (struct lruvec *lruvec,struct scan_control *sc,bool can_swap) { +bool success;unsigned long nr_to_scan;struct mem_cgroup *memcg =lruvec_memcg (lruvec);DEFINE_MAX_SEQ (lruvec);
@@ -4682,14 +4676,17 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, boolif (mem_cgroup_below_min (sc->target_mem_cgroup, memcg))return -1 ; -if (!should_run_aging (lruvec, max_seq, sc, can_swap, &nr_to_scan)) -return nr_to_scan; + success =should_run_aging (lruvec, max_seq, can_swap, &nr_to_scan); -/* skip the aging path at the default priority */ -if (sc->priority ==DEF_PRIORITY ) +/* try to scrape all its memory if this memcg was deleted */ +if (nr_to_scan && !mem_cgroup_online (memcg))return nr_to_scan; -/* skip this lruvec as it's low on cold folios */ +/* try to get away with not aging at the default priority */ +if (!success || sc->priority ==DEF_PRIORITY ) +return nr_to_scan >> sc->priority; + +/* stop scanning this lruvec as it's low on cold folios */ return try_to_inc_max_seq (lruvec, max_seq, can_swap, false) ?-1 :0 ; }
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
PATCH 3/5 mm/mglru: improve reset_mm_stats()
In reply to Kinsey Hostruct lruvec* is already a field of struct lru_gen_mm_walk. Remove the parameter struct lruvec* into functions that already have access to struct lru_gen_mm_walk*.
Also, we do not need to handle reset histogram stats when !should_walk_mmu(). Remove the call to reset_mm_stats() in iterate_mm_list_nowalk().
--- mm/vmscan.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
mm/vmscan.c +22 -20
@@ -2897,38 +2897,37 @@ static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)#endif -static void reset_mm_stats (struct lruvec *lruvec,struct lru_gen_mm_walk *walk,bool last) +static void reset_mm_stats (struct lru_gen_mm_walk *walk,bool last) {int i;int hist; +struct lruvec *lruvec = walk->lruvec;struct lru_gen_mm_state *mm_state =get_mm_state (lruvec);lockdep_assert_held (&get_mm_list (lruvec_memcg (lruvec))->lock); -if (walk) { - hist =lru_hist_from_seq (walk->max_seq); + hist =lru_hist_from_seq (walk->max_seq); -for (i =0 ; i <NR_MM_STATS ; i++) { -WRITE_ONCE (mm_state->stats[hist][i], - mm_state->stats[hist][i] + walk->mm_stats[i]); - walk->mm_stats[i] =0 ; - } +for (i =0 ; i <NR_MM_STATS ; i++) { +WRITE_ONCE (mm_state->stats[hist][i], + mm_state->stats[hist][i] + walk->mm_stats[i]); + walk->mm_stats[i] =0 ; }if (NR_HIST_GENS >1 && last) { - hist =lru_hist_from_seq (mm_state->seq +1 ); + hist =lru_hist_from_seq (walk->max_seq +1 );for (i =0 ; i <NR_MM_STATS ; i++)WRITE_ONCE (mm_state->stats[hist][i],0 ); } } -static bool iterate_mm_list (struct lruvec *lruvec,struct lru_gen_mm_walk *walk, -struct mm_struct **iter) +static bool iterate_mm_list (struct lru_gen_mm_walk *walk,struct mm_struct **iter) {bool first = false;bool last = false;struct mm_struct *mm =NULL ; +struct lruvec *lruvec = walk->lruvec;struct mem_cgroup *memcg =lruvec_memcg (lruvec);struct lru_gen_mm_list *mm_list =get_mm_list (memcg);struct lru_gen_mm_state *mm_state =get_mm_state (lruvec);
@@ -2972,7 +2971,7 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, }while (!(mm =get_next_mm (walk))); done:if (*iter || last) -reset_mm_stats (lruvec, walk, last); +reset_mm_stats (walk, last);spin_unlock (&mm_list->lock);
@@ -3002,7 +3001,6 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) mm_state->head =NULL ; mm_state->tail =NULL ;WRITE_ONCE (mm_state->seq, mm_state->seq +1 ); -reset_mm_stats (lruvec,NULL , true); success = true; }
@@ -3177,9 +3175,10 @@ static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, walk->nr_pages[new_gen][type][zone] += delta; } -static void reset_batch_size (struct lruvec *lruvec,struct lru_gen_mm_walk *walk) +static void reset_batch_size (struct lru_gen_mm_walk *walk) {int gen, type, zone; +struct lruvec *lruvec = walk->lruvec;struct lru_gen_folio *lrugen = &lruvec->lrugen; walk->batched =0 ;
@@ -3609,7 +3608,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,return -EAGAIN ; } -static void walk_mm (struct lruvec *lruvec,struct mm_struct *mm,struct lru_gen_mm_walk *walk) +static void walk_mm (struct mm_struct *mm,struct lru_gen_mm_walk *walk) {static const struct mm_walk_ops mm_walk_ops = { .test_walk = should_skip_vma,
@@ -3618,6 +3617,7 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_ };int err; +struct lruvec *lruvec = walk->lruvec;struct mem_cgroup *memcg =lruvec_memcg (lruvec); walk->next_addr =FIRST_USER_ADDRESS ;
@@ -3646,7 +3646,7 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_if (walk->batched) {spin_lock_irq (&lruvec->lru_lock); -reset_batch_size (lruvec, walk); +reset_batch_size (walk);spin_unlock_irq (&lruvec->lru_lock); }
@@ -3874,9 +3874,9 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, walk->force_scan = force_scan;do { - success =iterate_mm_list (lruvec, walk, &mm); + success =iterate_mm_list (walk, &mm);if (mm) -walk_mm (lruvec, mm, walk); +walk_mm (mm, walk); }while (mm); done:if (success) {
@@ -4576,8 +4576,10 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swapmove_folios_to_lru (lruvec, &list); walk = current->reclaim_state->mm_walk; -if (walk && walk->batched) -reset_batch_size (lruvec, walk); +if (walk && walk->batched) { + walk->lruvec = lruvec; +reset_batch_size (walk); + } item =PGSTEAL_KSWAPD +reclaimer_offset ();if (!cgroup_reclaim (sc))
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
PATCH 4/5 mm/mglru: improve struct lru_gen_mm_walk
In reply to Kinsey HoRename max_seq to seq in struct lru_gen_mm_walk to keep consistent with struct lru_gen_mm_state. Note that seq is not always up to date with max_seq from lru_gen_folio.
No functional changes.
--- include/linux/mmzone.h | 6 ++--- mm/vmscan.c | 50 ++++++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 27 deletions(-)
include/linux/mmzone.h +3 -3
@@ -468,7 +468,7 @@ enum {#define NR_BLOOM_FILTERS 2struct lru_gen_mm_state { -/* set to max_seq after each iteration */ +/* synced with max_seq after each iteration */ unsigned long seq;/* where the current iteration continues after */ struct list_head *head;
@@ -483,8 +483,8 @@ struct lru_gen_mm_state {struct lru_gen_mm_walk {/* the lruvec under reclaim */ struct lruvec *lruvec; -/* unstable max_seq from lru_gen_folio */ -unsigned long max_seq; +/* max_seq from lru_gen_folio: can be out of date */ +unsigned long seq;/* the next address within an mm to scan */ unsigned long next_addr;/* to batch promoted pages */
mm/vmscan.c +26 -24
173 lines not on this page · raw message
Signature · 1 line
2.43.0.687.g38aa6559b0-goog
PATCH 5/5 mm/mglru: improve swappiness handling
In reply to Kinsey HoThe reclaimable number of anon pages used to set initial reclaim priority is only based on get_swappiness(). Use can_reclaim_anon_pages() to include NUMA node demotion.
Also move the swappiness handling of when !__GFP_IO in try_to_shrink_lruvec() into isolate_folios().
--- mm/vmscan.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
mm/vmscan.c +9 -10
@@ -4307,7 +4307,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca {bool success; -/* swapping inhibited */ +/* swap constrained */ if (!(sc->gfp_mask & __GFP_IO) && (folio_test_dirty (folio) || (folio_test_anon (folio) && !folio_test_swapcache (folio))))
@@ -4476,9 +4476,11 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swDEFINE_MIN_SEQ (lruvec);/* -* Try to make the obvious choice first. When anon and file are both -* available from the same generation, interpret swappiness 1 as file -* first and MAX_SWAPPINESS as anon first. +* Try to make the obvious choice first, and if anon and file are both +* available from the same generation, +* 1. Interpret swappiness 1 as file first and MAX_SWAPPINESS as anon first. +* 2. If !__GFP_IO, file first since clean pagecache is more likely to +* exist than clean swapcache. */ if (!swappiness) type =LRU_GEN_FILE ;
@@ -4488,6 +4490,8 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw type =LRU_GEN_FILE ; else if (swappiness == MAX_SWAPPINESS) type =LRU_GEN_ANON ; + else if (!(sc->gfp_mask & __GFP_IO)) + type =LRU_GEN_FILE ; else type =get_type_to_scan (lruvec, swappiness, &tier);
@@ -4731,10 +4735,6 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)unsigned long scanned =0 ;int swappiness =get_swappiness (lruvec, sc); -/* clean file folios are more likely to exist */ -if (swappiness && !(sc->gfp_mask & __GFP_IO)) - swappiness =1 ; -while (true) {int delta;
@@ -4897,7 +4897,6 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control {int priority;unsigned long reclaimable; -struct lruvec *lruvec =mem_cgroup_lruvec (NULL , pgdat);if (sc->priority !=DEF_PRIORITY || sc->nr_to_reclaim <MIN_LRU_BATCH )return ;
@@ -4907,7 +4906,7 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control* where reclaimed_to_scanned_ratio = inactive / total. */ reclaimable =node_page_state (pgdat,NR_INACTIVE_FILE ); -if (get_swappiness (lruvec, sc)) +if (can_reclaim_anon_pages (NULL , pgdat->node_id, sc)) reclaimable +=node_page_state (pgdat,NR_INACTIVE_ANON );/* round down reclaimable and round up sc->nr_to_reclaim */
Signature · 1 line
2.43.0.687.g38aa6559b0-goog