Avoid building lrugen page table walk code
Subsystems (9)
v2 cover Avoid building lrugen page table walk code
This patchset avoids building changes added by commit bd74fdaea146 ("mm:
multi-gen LRU: support page table walks") on platforms that don't support hardware atomic updates of access bits.
Aneesh Kumar K.V (5):
mm/mglru: Create a new helper iterate_mm_list_walk mm/mglru: Move Bloom filter code around
mm/mglru: Move code around to make future patch easy mm/mglru: move iterate_mm_list_walk Helper mm/mglru: Don't build multi-gen LRU page table walk code on
architecture not supported
arch/Kconfig | 3 + arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 10 +- include/linux/mmzone.h | 12 +- kernel/fork.c | 2 +- mm/memcontrol.c | 2 +- mm/vmscan.c | 955 +++++++++++++++++++------------------ 9 files changed, 528 insertions(+), 460 deletions(-)
Signature · 1 line
2.41.0
PATCH v2 1/5 mm/mglru: Create a new helper iterate_mm_list_walk
In a later patch we will not build this on ppc64 architecture. No functional change in this patch.
Signed-off-by Aneesh Kumar K.V <[email protected]>
--- mm/vmscan.c | 52 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 22 deletions(-)
mm/vmscan.c +30 -22
@@ -4491,12 +4491,37 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)spin_unlock_irq (&lruvec->lru_lock); } +static bool iterate_mm_list_walk (struct lruvec *lruvec,unsigned long max_seq, +bool can_swap,bool force_scan) +{ +bool success; +struct mm_struct *mm =NULL ; +struct lru_gen_mm_walk *walk; + + walk =set_mm_walk (NULL , true); +if (!walk) { + success =iterate_mm_list_nowalk (lruvec, max_seq); +return success; + } + + walk->lruvec = lruvec; + walk->max_seq = max_seq; + walk->can_swap = can_swap; + walk->force_scan = force_scan; + +do { + success =iterate_mm_list (lruvec, walk, &mm); +if (mm) +walk_mm (lruvec, mm, walk); + }while (mm); + +return success; +} +static bool try_to_inc_max_seq (struct lruvec *lruvec,unsigned long max_seq,struct scan_control *sc,bool can_swap,bool force_scan) {bool success; -struct lru_gen_mm_walk *walk; -struct mm_struct *mm =NULL ;struct lru_gen_folio *lrugen = &lruvec->lrugen;VM_WARN_ON_ONCE (max_seq >READ_ONCE (lrugen->max_seq));
@@ -4506,34 +4531,17 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, success = false; goto done; } -/* * If the hardware doesn't automatically set the accessed bit, fallback * to lru_gen_look_around(), which only clears the accessed bit in a * handful of PTEs. Spreading the work out over a period of time usually * is less efficient, but it avoids bursty page faults. */ -if (!should_walk_mmu ()) { - success =iterate_mm_list_nowalk (lruvec, max_seq); - goto done; - } - - walk =set_mm_walk (NULL , true); -if (!walk) { +if (!should_walk_mmu ()) success =iterate_mm_list_nowalk (lruvec, max_seq); - goto done; - } - - walk->lruvec = lruvec; - walk->max_seq = max_seq; - walk->can_swap = can_swap; - walk->force_scan = force_scan; +else + success =iterate_mm_list_walk (lruvec, max_seq, can_swap, force_scan); -do { - success =iterate_mm_list (lruvec, walk, &mm); -if (mm) -walk_mm (lruvec, mm, walk); - }while (mm); done:if (success)inc_max_seq (lruvec, can_swap, force_scan);
Signature · 1 line
2.41.0
PATCH v2 2/5 mm/mglru: Move Bloom filter code around
In reply to Aneesh Kumar K.VThis allows to avoid building this code on powerpc architecture. No functional change in this patch.
Signed-off-by Aneesh Kumar K.V <[email protected]>
--- mm/vmscan.c | 462 ++++++++++++++++++++++++++-------------------------- 1 file changed, 231 insertions(+), 231 deletions(-)
mm/vmscan.c +231 -231
@@ -3325,6 +3325,237 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)get_nr_gens (lruvec,LRU_GEN_ANON ) <=MAX_NR_GENS ; } +/****************************************************************************** +* PID controller +******************************************************************************/ + +/* +* A feedback loop based on Proportional-Integral-Derivative (PID) controller. +* +* The P term is refaulted/(evicted+protected) from a tier in the generation +* currently being evicted; the I term is the exponential moving average of the +* P term over the generations previously evicted, using the smoothing factor +* 1/2; the D term isn't supported. +* +* The setpoint (SP) is always the first tier of one type; the process variable +* (PV) is either any tier of the other type or any other tier of the same +* type. +* +* The error is the difference between the SP and the PV; the correction is to +* turn off protection when SP>PV or turn on protection when SP<PV. +* +* For future optimizations: +* 1. The D term may discount the other two terms over time so that long-lived +* generations can resist stale information. +*/ +struct ctrl_pos { +unsigned long refaulted; +unsigned long total; +int gain; +}; + +static void read_ctrl_pos (struct lruvec *lruvec,int type,int tier,int gain, +struct ctrl_pos *pos) +{ +struct lru_gen_folio *lrugen = &lruvec->lrugen; +int hist =lru_hist_from_seq (lrugen->min_seq[type]); + + pos->refaulted = lrugen->avg_refaulted[type][tier] + +atomic_long_read (&lrugen->refaulted[hist][type][tier]); + pos->total = lrugen->avg_total[type][tier] + +atomic_long_read (&lrugen->evicted[hist][type][tier]); +if (tier) + pos->total += lrugen->protected[hist][type][tier -1 ]; + pos->gain = gain; +} + +static void reset_ctrl_pos (struct lruvec *lruvec,int type,bool carryover) +{ +int hist, tier; +struct lru_gen_folio *lrugen = &lruvec->lrugen; +bool clear = carryover ?NR_HIST_GENS ==1 :NR_HIST_GENS >1 ; +unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq +1 ; + +lockdep_assert_held (&lruvec->lru_lock); + +if (!carryover && !clear) +return ; + + hist =lru_hist_from_seq (seq); + +for (tier =0 ; tier <MAX_NR_TIERS ; tier++) { +if (carryover) { +unsigned long sum; + + sum = lrugen->avg_refaulted[type][tier] + +atomic_long_read (&lrugen->refaulted[hist][type][tier]); +WRITE_ONCE (lrugen->avg_refaulted[type][tier], sum /2 ); + + sum = lrugen->avg_total[type][tier] + +atomic_long_read (&lrugen->evicted[hist][type][tier]); +if (tier) + sum += lrugen->protected[hist][type][tier -1 ]; +WRITE_ONCE (lrugen->avg_total[type][tier], sum /2 ); + } + +if (clear) { +atomic_long_set (&lrugen->refaulted[hist][type][tier],0 ); +atomic_long_set (&lrugen->evicted[hist][type][tier],0 ); +if (tier) +WRITE_ONCE (lrugen->protected[hist][type][tier -1 ],0 ); + } + } +} + +static bool positive_ctrl_err (struct ctrl_pos *sp,struct ctrl_pos *pv) +{ +/* +* Return true if the PV has a limited number of refaults or a lower +* refaulted/total than the SP. +*/ +return pv->refaulted <MIN_LRU_BATCH || + pv->refaulted * (sp->total +MIN_LRU_BATCH ) * sp->gain <= + (sp->refaulted +1 ) * pv->total * pv->gain; +} + +/****************************************************************************** +* the aging +******************************************************************************/ + +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen (struct lruvec *lruvec,struct folio *folio,bool reclaiming) +{ +int type =folio_is_file_lru (folio); +struct lru_gen_folio *lrugen = &lruvec->lrugen; +int new_gen, old_gen =lru_gen_from_seq (lrugen->min_seq[type]); +unsigned long new_flags, old_flags =READ_ONCE (folio->flags); + +VM_WARN_ON_ONCE_FOLIO (!(old_flags &LRU_GEN_MASK ), folio); + +do { + new_gen = ((old_flags &LRU_GEN_MASK ) >>LRU_GEN_PGOFF ) -1 ; +/* folio_update_gen() has promoted this page? */ +if (new_gen >=0 && new_gen != old_gen) +return new_gen; + + new_gen = (old_gen +1 ) %MAX_NR_GENS ; + + new_flags = old_flags & ~(LRU_GEN_MASK |LRU_REFS_MASK |LRU_REFS_FLAGS ); + new_flags |= (new_gen +1UL ) <<LRU_GEN_PGOFF ; +/* for folio_end_writeback() */ +if (reclaiming) + new_flags |=BIT (PG_reclaim); + }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); + +lru_gen_update_size (lruvec, folio, old_gen, new_gen); + +return new_gen; +} + +static unsigned long get_pte_pfn (pte_t pte,struct vm_area_struct *vma,unsigned long addr) +{ +unsigned long pfn =pte_pfn (pte); + +VM_WARN_ON_ONCE (addr < vma->vm_start || addr >= vma->vm_end); + +if (!pte_present (pte) ||is_zero_pfn (pfn)) +return -1 ; + +if (WARN_ON_ONCE (pte_devmap (pte) ||pte_special (pte))) +return -1 ; + +if (WARN_ON_ONCE (!pfn_valid (pfn))) +return -1 ; + +return pfn; +} + +static struct folio *get_pfn_folio (unsigned long pfn,struct mem_cgroup *memcg, +struct pglist_data *pgdat,bool can_swap) +{ +struct folio *folio; + +/* try to avoid unnecessary memory loads */ +if (pfn < pgdat->node_start_pfn || pfn >=pgdat_end_pfn (pgdat)) +return NULL ; + + folio =pfn_folio (pfn); +if (folio_nid (folio) != pgdat->node_id) +return NULL ; + +if (folio_memcg_rcu (folio) != memcg) +return NULL ; + +/* file VMAs can contain anon pages from COW */ +if (!folio_is_file_lru (folio) && !can_swap) +return NULL ; + +return folio; +} + +/* promote pages accessed through page tables */ +static int folio_update_gen (struct folio *folio,int gen) +{ +unsigned long new_flags, old_flags =READ_ONCE (folio->flags); + +VM_WARN_ON_ONCE (gen >=MAX_NR_GENS ); +VM_WARN_ON_ONCE (!rcu_read_lock_held ()); + +do { +/* lru_gen_del_folio() has isolated this page? */ +if (!(old_flags &LRU_GEN_MASK )) { +/* for shrink_folio_list() */ + new_flags = old_flags |BIT (PG_referenced); +continue ; + } + + new_flags = old_flags & ~(LRU_GEN_MASK |LRU_REFS_MASK |LRU_REFS_FLAGS ); + new_flags |= (gen +1UL ) <<LRU_GEN_PGOFF ; + }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); + +return ((old_flags &LRU_GEN_MASK ) >>LRU_GEN_PGOFF ) -1 ; +} + +static void update_batch_size (struct lru_gen_mm_walk *walk,struct folio *folio, +int old_gen,int new_gen) +{ +int type =folio_is_file_lru (folio); +int zone =folio_zonenum (folio); +int delta =folio_nr_pages (folio); + +VM_WARN_ON_ONCE (old_gen >=MAX_NR_GENS ); +VM_WARN_ON_ONCE (new_gen >=MAX_NR_GENS ); + + walk->batched++; + + walk->nr_pages[old_gen][type][zone] -= delta; + walk->nr_pages[new_gen][type][zone] += delta; +} + +static void reset_batch_size (struct lruvec *lruvec,struct lru_gen_mm_walk *walk) +{ +int gen, type, zone; +struct lru_gen_folio *lrugen = &lruvec->lrugen; + + walk->batched =0 ; + +for_each_gen_type_zone (gen, type, zone) { +enum lru_list lru = type *LRU_INACTIVE_FILE ; +int delta = walk->nr_pages[gen][type][zone]; + +if (!delta) +continue ; + + walk->nr_pages[gen][type][zone] =0 ; +WRITE_ONCE (lrugen->nr_pages[gen][type][zone], + lrugen->nr_pages[gen][type][zone] + delta); + +if (lru_gen_is_active (lruvec, gen)) + lru +=LRU_ACTIVE ; +__update_lru_size (lruvec, lru, zone, delta); + } +} +/****************************************************************************** * Bloom filters ******************************************************************************/
@@ -3672,237 +3903,6 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)return success; } -/****************************************************************************** -* PID controller -******************************************************************************/ - -/* -* A feedback loop based on Proportional-Integral-Derivative (PID) controller. -* -* The P term is refaulted/(evicted+protected) from a tier in the generation -* currently being evicted; the I term is the exponential moving average of the -* P term over the generations previously evicted, using the smoothing factor -* 1/2; the D term isn't supported. -* -* The setpoint (SP) is always the first tier of one type; the process variable -* (PV) is either any tier of the other type or any other tier of the same -* type. -* -* The error is the difference between the SP and the PV; the correction is to -* turn off protection when SP>PV or turn on protection when SP<PV. -* -* For future optimizations: -* 1. The D term may discount the other two terms over time so that long-lived -* generations can resist stale information. -*/ -struct ctrl_pos { -unsigned long refaulted; -unsigned long total; -int gain; -}; - -static void read_ctrl_pos (struct lruvec *lruvec,int type,int tier,int gain, -struct ctrl_pos *pos) -{ -struct lru_gen_folio *lrugen = &lruvec->lrugen; -int hist =lru_hist_from_seq (lrugen->min_seq[type]); - - pos->refaulted = lrugen->avg_refaulted[type][tier] + -atomic_long_read (&lrugen->refaulted[hist][type][tier]); - pos->total = lrugen->avg_total[type][tier] + -atomic_long_read (&lrugen->evicted[hist][type][tier]); -if (tier) - pos->total += lrugen->protected[hist][type][tier -1 ]; - pos->gain = gain; -} - -static void reset_ctrl_pos (struct lruvec *lruvec,int type,bool carryover) -{ -int hist, tier; -struct lru_gen_folio *lrugen = &lruvec->lrugen; -bool clear = carryover ?NR_HIST_GENS ==1 :NR_HIST_GENS >1 ; -unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq +1 ; - -lockdep_assert_held (&lruvec->lru_lock); - -if (!carryover && !clear) -return ; - - hist =lru_hist_from_seq (seq); - -for (tier =0 ; tier <MAX_NR_TIERS ; tier++) { -if (carryover) { -unsigned long sum; - - sum = lrugen->avg_refaulted[type][tier] + -atomic_long_read (&lrugen->refaulted[hist][type][tier]); -WRITE_ONCE (lrugen->avg_refaulted[type][tier], sum /2 ); - - sum = lrugen->avg_total[type][tier] + -atomic_long_read (&lrugen->evicted[hist][type][tier]); -if (tier) - sum += lrugen->protected[hist][type][tier -1 ]; -WRITE_ONCE (lrugen->avg_total[type][tier], sum /2 ); - } - -if (clear) { -atomic_long_set (&lrugen->refaulted[hist][type][tier],0 ); -atomic_long_set (&lrugen->evicted[hist][type][tier],0 ); -if (tier) -WRITE_ONCE (lrugen->protected[hist][type][tier -1 ],0 ); - } - } -} - -static bool positive_ctrl_err (struct ctrl_pos *sp,struct ctrl_pos *pv) -{ -/* -* Return true if the PV has a limited number of refaults or a lower -* refaulted/total than the SP. -*/ -return pv->refaulted <MIN_LRU_BATCH || - pv->refaulted * (sp->total +MIN_LRU_BATCH ) * sp->gain <= - (sp->refaulted +1 ) * pv->total * pv->gain; -} - -/****************************************************************************** -* the aging -******************************************************************************/ - -/* protect pages accessed multiple times through file descriptors */ -static int folio_inc_gen (struct lruvec *lruvec,struct folio *folio,bool reclaiming) -{ -int type =folio_is_file_lru (folio); -struct lru_gen_folio *lrugen = &lruvec->lrugen; -int new_gen, old_gen =lru_gen_from_seq (lrugen->min_seq[type]); -unsigned long new_flags, old_flags =READ_ONCE (folio->flags); - -VM_WARN_ON_ONCE_FOLIO (!(old_flags &LRU_GEN_MASK ), folio); - -do { - new_gen = ((old_flags &LRU_GEN_MASK ) >>LRU_GEN_PGOFF ) -1 ; -/* folio_update_gen() has promoted this page? */ -if (new_gen >=0 && new_gen != old_gen) -return new_gen; - - new_gen = (old_gen +1 ) %MAX_NR_GENS ; - - new_flags = old_flags & ~(LRU_GEN_MASK |LRU_REFS_MASK |LRU_REFS_FLAGS ); - new_flags |= (new_gen +1UL ) <<LRU_GEN_PGOFF ; -/* for folio_end_writeback() */ -if (reclaiming) - new_flags |=BIT (PG_reclaim); - }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); - -lru_gen_update_size (lruvec, folio, old_gen, new_gen); - -return new_gen; -} - -static unsigned long get_pte_pfn (pte_t pte,struct vm_area_struct *vma,unsigned long addr) -{ -unsigned long pfn =pte_pfn (pte); - -VM_WARN_ON_ONCE (addr < vma->vm_start || addr >= vma->vm_end); - -if (!pte_present (pte) ||is_zero_pfn (pfn)) -return -1 ; - -if (WARN_ON_ONCE (pte_devmap (pte) ||pte_special (pte))) -return -1 ; - -if (WARN_ON_ONCE (!pfn_valid (pfn))) -return -1 ; - -return pfn; -} - -static struct folio *get_pfn_folio (unsigned long pfn,struct mem_cgroup *memcg, -struct pglist_data *pgdat,bool can_swap) -{ -struct folio *folio; - -/* try to avoid unnecessary memory loads */ -if (pfn < pgdat->node_start_pfn || pfn >=pgdat_end_pfn (pgdat)) -return NULL ; - - folio =pfn_folio (pfn); -if (folio_nid (folio) != pgdat->node_id) -return NULL ; - -if (folio_memcg_rcu (folio) != memcg) -return NULL ; - -/* file VMAs can contain anon pages from COW */ -if (!folio_is_file_lru (folio) && !can_swap) -return NULL ; - -return folio; -} - -/* promote pages accessed through page tables */ -static int folio_update_gen (struct folio *folio,int gen) -{ -unsigned long new_flags, old_flags =READ_ONCE (folio->flags); - -VM_WARN_ON_ONCE (gen >=MAX_NR_GENS ); -VM_WARN_ON_ONCE (!rcu_read_lock_held ()); - -do { -/* lru_gen_del_folio() has isolated this page? */ -if (!(old_flags &LRU_GEN_MASK )) { -/* for shrink_folio_list() */ - new_flags = old_flags |BIT (PG_referenced); -continue ; - } - - new_flags = old_flags & ~(LRU_GEN_MASK |LRU_REFS_MASK |LRU_REFS_FLAGS ); - new_flags |= (gen +1UL ) <<LRU_GEN_PGOFF ; - }while (!try_cmpxchg (&folio->flags, &old_flags, new_flags)); - -return ((old_flags &LRU_GEN_MASK ) >>LRU_GEN_PGOFF ) -1 ; -} - -static void update_batch_size (struct lru_gen_mm_walk *walk,struct folio *folio, -int old_gen,int new_gen) -{ -int type =folio_is_file_lru (folio); -int zone =folio_zonenum (folio); -int delta =folio_nr_pages (folio); - -VM_WARN_ON_ONCE (old_gen >=MAX_NR_GENS ); -VM_WARN_ON_ONCE (new_gen >=MAX_NR_GENS ); - - walk->batched++; - - walk->nr_pages[old_gen][type][zone] -= delta; - walk->nr_pages[new_gen][type][zone] += delta; -} - -static void reset_batch_size (struct lruvec *lruvec,struct lru_gen_mm_walk *walk) -{ -int gen, type, zone; -struct lru_gen_folio *lrugen = &lruvec->lrugen; - - walk->batched =0 ; - -for_each_gen_type_zone (gen, type, zone) { -enum lru_list lru = type *LRU_INACTIVE_FILE ; -int delta = walk->nr_pages[gen][type][zone]; - -if (!delta) -continue ; - - walk->nr_pages[gen][type][zone] =0 ; -WRITE_ONCE (lrugen->nr_pages[gen][type][zone], - lrugen->nr_pages[gen][type][zone] + delta); - -if (lru_gen_is_active (lruvec, gen)) - lru +=LRU_ACTIVE ; -__update_lru_size (lruvec, lru, zone, delta); - } -} -static int should_skip_vma (unsigned long start,unsigned long end,struct mm_walk *args) {struct address_space *mapping;
Signature · 1 line
2.41.0
PATCH v2 3/5 mm/mglru: Move code around to make future patch easy
In reply to Aneesh Kumar K.VNo functional change in this patch.
Signed-off-by Aneesh Kumar K.V <[email protected]>
--- mm/vmscan.c | 64 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-)
mm/vmscan.c +32 -32
@@ -3556,6 +3556,38 @@ static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk } } +static struct lru_gen_mm_walk *set_mm_walk (struct pglist_data *pgdat,bool force_alloc) +{ +struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + +if (pgdat &¤t_is_kswapd ()) { +VM_WARN_ON_ONCE (walk); + + walk = &pgdat->mm_walk; + }else if (!walk && force_alloc) { +VM_WARN_ON_ONCE (current_is_kswapd ()); + + walk =kzalloc (sizeof (*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + } + + current->reclaim_state->mm_walk = walk; + +return walk; +} + +static void clear_mm_walk (void ) +{ +struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + +VM_WARN_ON_ONCE (walk &&memchr_inv (walk->nr_pages,0 ,sizeof (walk->nr_pages))); +VM_WARN_ON_ONCE (walk &&memchr_inv (walk->mm_stats,0 ,sizeof (walk->mm_stats))); + + current->reclaim_state->mm_walk =NULL ; + +if (!current_is_kswapd ()) +kfree (walk); +} +/****************************************************************************** * Bloom filters ******************************************************************************/
@@ -4324,38 +4356,6 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_ }while (err == -EAGAIN ); } -static struct lru_gen_mm_walk *set_mm_walk (struct pglist_data *pgdat,bool force_alloc) -{ -struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; - -if (pgdat &¤t_is_kswapd ()) { -VM_WARN_ON_ONCE (walk); - - walk = &pgdat->mm_walk; - }else if (!walk && force_alloc) { -VM_WARN_ON_ONCE (current_is_kswapd ()); - - walk =kzalloc (sizeof (*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); - } - - current->reclaim_state->mm_walk = walk; - -return walk; -} - -static void clear_mm_walk (void ) -{ -struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; - -VM_WARN_ON_ONCE (walk &&memchr_inv (walk->nr_pages,0 ,sizeof (walk->nr_pages))); -VM_WARN_ON_ONCE (walk &&memchr_inv (walk->mm_stats,0 ,sizeof (walk->mm_stats))); - - current->reclaim_state->mm_walk =NULL ; - -if (!current_is_kswapd ()) -kfree (walk); -} -static bool inc_min_seq (struct lruvec *lruvec,int type,bool can_swap) {int zone;
Signature · 1 line
2.41.0
PATCH v2 4/5 mm/mglru: move iterate_mm_list_walk Helper
In reply to Aneesh Kumar K.VThis helps to avoid building this code on powerpc. No functional change in this patch.
Signed-off-by Aneesh Kumar K.V <[email protected]>
--- mm/vmscan.c | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-)
mm/vmscan.c +27 -27
@@ -4356,6 +4356,33 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_ }while (err == -EAGAIN ); } +static bool iterate_mm_list_walk (struct lruvec *lruvec,unsigned long max_seq, +bool can_swap,bool force_scan) +{ +bool success; +struct mm_struct *mm =NULL ; +struct lru_gen_mm_walk *walk; + + walk =set_mm_walk (NULL , true); +if (!walk) { + success =iterate_mm_list_nowalk (lruvec, max_seq); +return success; + } + + walk->lruvec = lruvec; + walk->max_seq = max_seq; + walk->can_swap = can_swap; + walk->force_scan = force_scan; + +do { + success =iterate_mm_list (lruvec, walk, &mm); +if (mm) +walk_mm (lruvec, mm, walk); + }while (mm); + +return success; +} +static bool inc_min_seq (struct lruvec *lruvec,int type,bool can_swap) {int zone;
@@ -4491,33 +4518,6 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)spin_unlock_irq (&lruvec->lru_lock); } -static bool iterate_mm_list_walk (struct lruvec *lruvec,unsigned long max_seq, -bool can_swap,bool force_scan) -{ -bool success; -struct mm_struct *mm =NULL ; -struct lru_gen_mm_walk *walk; - - walk =set_mm_walk (NULL , true); -if (!walk) { - success =iterate_mm_list_nowalk (lruvec, max_seq); -return success; - } - - walk->lruvec = lruvec; - walk->max_seq = max_seq; - walk->can_swap = can_swap; - walk->force_scan = force_scan; - -do { - success =iterate_mm_list (lruvec, walk, &mm); -if (mm) -walk_mm (lruvec, mm, walk); - }while (mm); - -return success; -} -static bool try_to_inc_max_seq (struct lruvec *lruvec,unsigned long max_seq,struct scan_control *sc,bool can_swap,bool force_scan) {
Signature · 1 line
2.41.0
PATCH v2 5/5 mm/mglru: Don't build multi-gen LRU page table walk code on architecture not supported
In reply to Aneesh Kumar K.VNot all architecture supports hardware atomic updates of access bits. On such an arch, we don't use a page table walk to classify pages into generations. Add a kernel config option and remove adding all the page table walk code on such architecture.
This avoid calling lru_gen related code (lru_gen_add/remove/migrate_mm) in fork/exit/context switch Also we don't build different components like Bloom filter and all the page table walk code (walk_mm and related code) on not supported architecture with this change.
No preformance change observed with mongodb ycsb test:
Patch details Throughput(Ops/sec)
without patch 91252
With patch 91488
Without patch:
$ size mm/vmscan.o text data bss dec hex filename 116016 36857 40 152913 25551 mm/vmscan.o
With patch
$ size mm/vmscan.o text data bss dec hex filename 112864 36437 40 149341 2475d mm/vmscan.o
Signed-off-by Aneesh Kumar K.V <[email protected]>
--- arch/Kconfig | 3 +++ arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 10 ++++----- include/linux/mmzone.h | 12 +++++++++- kernel/fork.c | 2 +- mm/memcontrol.c | 2 +- mm/vmscan.c | 45 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 69 insertions(+), 9 deletions(-)
arch/Kconfig +3 -0
@@ -1471,6 +1471,9 @@ config DYNAMIC_SIGFRAME config HAVE_ARCH_NODE_DEV_GROUP bool +config LRU_TASK_PAGE_AGING + bool + config ARCH_HAS_NONLEAF_PMD_YOUNG bool help
arch/arm64/Kconfig +1 -0
@@ -233,6 +233,7 @@ config ARM64 select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN select LOCK_MM_AND_FIND_VMA + select LRU_TASK_PAGE_AGING if LRU_GEN select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE select NEED_SG_DMA_LENGTH
arch/x86/Kconfig +1 -0
@@ -281,6 +281,7 @@ config X86 select HOTPLUG_SPLIT_STARTUP if SMP && X86_32 select IRQ_FORCED_THREADING select LOCK_MM_AND_FIND_VMA + select LRU_TASK_PAGE_AGING if LRU_GEN select NEED_PER_CPU_EMBED_FIRST_CHUNK select NEED_PER_CPU_PAGE_FIRST_CHUNK select NEED_SG_DMA_LENGTH
include/linux/memcontrol.h +1 -1
@@ -321,7 +321,7 @@ struct mem_cgroup {struct deferred_split deferred_split_queue;#endif -#ifdef CONFIG_LRU_GEN +#ifdef CONFIG_LRU_TASK_PAGE_AGING /* per-memcg mm_struct list */ struct lru_gen_mm_list mm_list;#endif
include/linux/mm_types.h +5 -5
@@ -793,7 +793,7 @@ struct mm_struct {*/ unsigned long ksm_rmap_items;#endif -#ifdef CONFIG_LRU_GEN +#ifdef CONFIG_LRU_TASK_PAGE_AGING struct { /* this mm_struct is on lru_gen_mm_list */ struct list_head list;
@@ -808,7 +808,7 @@ struct mm_struct {struct mem_cgroup *memcg; #endif } lru_gen; -#endif /* CONFIG_LRU_GEN */ +#endif /* CONFIG_LRU_TASK_PAGE_AGING */ } __randomize_layout;/*
@@ -837,7 +837,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)return (struct cpumask *)&mm->cpu_bitmap; } -#ifdef CONFIG_LRU_GEN +#ifdef CONFIG_LRU_TASK_PAGE_AGING struct lru_gen_mm_list {/* mm_struct list for page table walkers */
@@ -871,7 +871,7 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) WRITE_ONCE(mm->lru_gen.bitmap, -1); } -#else /* !CONFIG_LRU_GEN */ +#else /* !CONFIG_LRU_TASK_PAGE_AGING */ static inline void lru_gen_add_mm (struct mm_struct *mm) {
@@ -895,7 +895,7 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) { } -#endif /* CONFIG_LRU_GEN */ +#endif /* CONFIG_LRU_TASK_PAGE_AGING */ struct vma_iterator {struct ma_state mas;
include/linux/mmzone.h +11 -1
@@ -461,6 +461,7 @@ enum {struct lru_gen_mm_state {/* set to max_seq after each iteration */ unsigned long seq; +#ifdef CONFIG_LRU_TASK_PAGE_AGING /* where the current iteration continues after */ struct list_head *head;/* where the last iteration ended before */
@@ -469,6 +470,11 @@ struct lru_gen_mm_state {unsigned long *filters[NR_BLOOM_FILTERS ];/* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS ][NR_MM_STATS ]; +#else +/* protect the seq update above */ +/* May be we can use lruvec->lock? */ +spinlock_t lock; +#endif };struct lru_gen_mm_walk {
@@ -546,9 +552,13 @@ struct lru_gen_memcg { };void lru_gen_init_pgdat (struct pglist_data *pgdat); - +#ifdef CONFIG_LRU_TASK_PAGE_AGING void lru_gen_init_memcg (struct mem_cgroup *memcg);void lru_gen_exit_memcg (struct mem_cgroup *memcg); +#else +static inline void lru_gen_init_memcg (struct mem_cgroup *memcg) {} +static inline void lru_gen_exit_memcg (struct mem_cgroup *memcg) {} +#endif void lru_gen_online_memcg (struct mem_cgroup *memcg);void lru_gen_offline_memcg (struct mem_cgroup *memcg);void lru_gen_release_memcg (struct mem_cgroup *memcg);
kernel/fork.c +1 -1
@@ -2932,7 +2932,7 @@ pid_t kernel_clone(struct kernel_clone_args *args)get_task_struct (p); } -if (IS_ENABLED (CONFIG_LRU_GEN ) && !(clone_flags &CLONE_VM )) { +if (IS_ENABLED (CONFIG_LRU_TASK_PAGE_AGING ) && !(clone_flags &CLONE_VM )) {/* lock the task to synchronize with memcg migration */ task_lock (p);lru_gen_add_mm (p->mm);
mm/memcontrol.c +1 -1
@@ -6357,7 +6357,7 @@ static void mem_cgroup_move_task(void) }#endif -#ifdef CONFIG_LRU_GEN +#ifdef CONFIG_LRU_TASK_PAGE_AGING static void mem_cgroup_attach (struct cgroup_taskset *tset) {struct task_struct *task;
mm/vmscan.c +45 -0
106 lines not on this page · raw message
Signature · 1 line
2.41.0
Quoted 13 lines · Aneesh Kumar K.V
This patchset avoids building changes added by commit
bd74fdaea146("mm:
multi-gen LRU: support page table walks") on platforms that don't support hardware atomic updates of access bits.Aneesh Kumar K.V (5):
mm/mglru: Create a new helperiterate_mm_list_walkmm/mglru: Move Bloom filter code around
mm/mglru: Move code around to make future patch easy mm/mglru: moveiterate_mm_list_walkHelper mm/mglru: Don't build multi-gen LRU page table walk code on
architecture not supported
Quoted 10 lines
arch/Kconfig | 3 + arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 10 +- include/linux/mmzone.h | 12 +- kernel/fork.c | 2 +- mm/memcontrol.c | 2 +- mm/vmscan.c | 955 +++++++++++++++++++------------------ 9 files changed, 528 insertions(+), 460 deletions(-)
1. There is no need for a new Kconfig -- the condition is simply defined(CONFIG_LRU_GEN) && !defined(arch_has_hw_pte_young)
2. The best practice to disable static functions is not by macros but:
static int const_cond(void)
{
return 1;
}
int main(void)
{
int a = const_cond();
if (a)
return 0;
/* the compiler doesn't generate code for static funcs below */
static_func_1();
...
static_func_N();LTO also optimizes external functions. But not everyone uses it. So we still need macros for them, and of course data structures.
3. In 4/5, you have:
@@ -461,6 +461,7 @@ enum {
struct lru_gen_mm_state {
/* set to max_seq after each iteration */
unsigned long seq;
+#ifdef CONFIG_LRU_TASK_PAGE_AGING
/* where the current iteration continues after */
struct list_head *head;
/* where the last iteration ended before */
@@ -469,6 +470,11 @@ struct lru_gen_mm_state {
unsigned long *filters[NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
+#else
+ /* protect the seq update above */
+ /* May be we can use lruvec->lock? */
+ spinlock_t lock;
+#endif
};The answer is yes, and not only that, we don't need lru_gen_mm_state at all.
I'm attaching a patch that fixes all above. If you want to post it, please feel free -- fully test it please, since I didn't. Otherwise I can ask TJ to help make this work for you.
$ git diff --stat include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 12 +- include/linux/mmzone.h | 2 + kernel/bounds.c | 6 +- kernel/fork.c | 2 +- mm/vmscan.c | 169 +++++++++++++++++++-------- 6 files changed, 137 insertions(+), 56 deletions(-)
On x86:
$ ./scripts/bloat-o-meter mm/vmscan.o.old mm/vmscan.o add/remove: 24/34 grow/shrink: 2/7 up/down: 966/-8716 (-7750) Function old new delta ... should_skip_vma 206 - -206 get_pte_pfn 261 - -261 lru_gen_add_mm 323 - -323 lru_gen_seq_show 1710 1370 -340 lru_gen_del_mm 432 - -432 reset_batch_size 572 - -572 try_to_inc_max_seq 2947 1635 -1312 walk_pmd_range_locked 1508 - -1508 walk_pud_range 3238 - -3238 Total: Before=99449, After=91699, chg -7.79%
$ objdump -S mm/vmscan.o | grep -A 20 "<try_to_inc_max_seq>:"
000000000000a350 <try_to_inc_max_seq>:
{
a350: e8 00 00 00 00 call a355 <try_to_inc_max_seq+0x5>
a355: 55 push %rbp
a356: 48 89 e5 mov %rsp,%rbp
a359: 41 57 push %r15
a35b: 41 56 push %r14
a35d: 41 55 push %r13
a35f: 41 54 push %r12
a361: 53 push %rbx
a362: 48 83 ec 70 sub $0x70,%rsp
a366: 41 89 d4 mov %edx,%r12d
a369: 49 89 f6 mov %rsi,%r14
a36c: 49 89 ff mov %rdi,%r15
spin_lock_irq(&lruvec->lru_lock);
a36f: 48 8d 5f 50 lea 0x50(%rdi),%rbx
a373: 48 89 df mov %rbx,%rdi
a376: e8 00 00 00 00 call a37b <try_to_inc_max_seq+0x2b>
success = max_seq == lrugen->max_seq;
a37b: 49 8b 87 88 00 00 00 mov 0x88(%r15),%rax
a382: 4c 39 f0 cmp %r14,%raxinclude/linux/memcontrol.h +1 -1
@@ -321,7 +321,7 @@ struct mem_cgroup {struct deferred_split deferred_split_queue;#endif -#ifdef CONFIG_LRU_GEN +#ifdef LRU_GEN_WALKS_PGTABLES /* per-memcg mm_struct list */ struct lru_gen_mm_list mm_list;#endif
include/linux/mm_types.h +8 -4
@@ -798,7 +798,7 @@ struct mm_struct {*/ unsigned long ksm_zero_pages;#endif /* CONFIG_KSM */ -#ifdef CONFIG_LRU_GEN +#ifdef LRU_GEN_WALKS_PGTABLES struct { /* this mm_struct is on lru_gen_mm_list */ struct list_head list;
@@ -813,7 +813,7 @@ struct mm_struct {struct mem_cgroup *memcg; #endif } lru_gen; -#endif /* CONFIG_LRU_GEN */ +#endif /* LRU_GEN_WALKS_PGTABLES */ } __randomize_layout;/*
@@ -851,6 +851,10 @@ struct lru_gen_mm_list { spinlock_t lock; }; +#endif /* CONFIG_LRU_GEN */ + +#ifdef LRU_GEN_WALKS_PGTABLES +void lru_gen_add_mm (struct mm_struct *mm);void lru_gen_del_mm (struct mm_struct *mm);#ifdef CONFIG_MEMCG
@@ -876,7 +880,7 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) WRITE_ONCE(mm->lru_gen.bitmap, -1); } -#else /* !CONFIG_LRU_GEN */ +#else /* !LRU_GEN_WALKS_PGTABLES */ static inline void lru_gen_add_mm (struct mm_struct *mm) {
@@ -900,7 +904,7 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) { } -#endif /* CONFIG_LRU_GEN */ +#endif /* LRU_GEN_WALKS_PGTABLES */ struct vma_iterator {struct ma_state mas;
include/linux/mmzone.h +2 -0
@@ -631,9 +631,11 @@ struct lruvec {#ifdef CONFIG_LRU_GEN /* evictable pages divided into generations */ struct lru_gen_folio lrugen; +#ifdef LRU_GEN_WALKS_PGTABLES /* to concurrently iterate lru_gen_mm_list */ struct lru_gen_mm_state mm_state;#endif +#endif /* CONFIG_LRU_GEN */ #ifdef CONFIG_MEMCG struct pglist_data *pgdat;#endif
kernel/bounds.c +5 -1
@@ -12,6 +12,7 @@#include <linux/kbuild.h> #include <linux/log2.h> #include <linux/spinlock_types.h> +#include <linux/pgtable.h> int main (void ) {
@@ -25,7 +26,10 @@ int main(void)#ifdef CONFIG_LRU_GEN DEFINE (LRU_GEN_WIDTH ,order_base_2 (MAX_NR_GENS +1 ));DEFINE (__LRU_REFS_WIDTH,MAX_NR_TIERS -2 ); -#else +#ifdef arch_has_hw_pte_young +DEFINE (LRU_GEN_WALKS_PGTABLES ,1 ); +#endif +#else /* !CONFIG_LRU_GEN */ DEFINE (LRU_GEN_WIDTH ,0 );DEFINE (__LRU_REFS_WIDTH,0 );#endif
kernel/fork.c +1 -1
@@ -2938,7 +2938,7 @@ pid_t kernel_clone(struct kernel_clone_args *args)get_task_struct (p); } -if (IS_ENABLED (CONFIG_LRU_GEN ) && !(clone_flags &CLONE_VM )) { +if (IS_ENABLED (LRU_GEN_WALKS_PGTABLES ) && !(clone_flags &CLONE_VM )) {/* lock the task to synchronize with memcg migration */ task_lock (p);lru_gen_add_mm (p->mm);
mm/vmscan.c +120 -49
411 lines not on this page · raw message
Quoted 118 lines · Yu Zhao
Quoted 13 lines · Aneesh Kumar K.V
This patchset avoids building changes added by commit
bd74fdaea146("mm:
multi-gen LRU: support page table walks") on platforms that don't support hardware atomic updates of access bits.Aneesh Kumar K.V (5):
mm/mglru: Create a new helperiterate_mm_list_walkmm/mglru: Move Bloom filter code around
mm/mglru: Move code around to make future patch easy mm/mglru: moveiterate_mm_list_walkHelper mm/mglru: Don't build multi-gen LRU page table walk code on
architecture not supportedQuoted 10 lines
arch/Kconfig | 3 + arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 10 +- include/linux/mmzone.h | 12 +- kernel/fork.c | 2 +- mm/memcontrol.c | 2 +- mm/vmscan.c | 955 +++++++++++++++++++------------------ 9 files changed, 528 insertions(+), 460 deletions(-)1. There is no need for a new
Kconfig-- the condition is simplydefined(CONFIG_LRU_GEN)&& !defined(arch_has_hw_pte_young)2. The best practice to disable static functions is not by macros but:
static int const_cond(void) { return 1; } int main(void) { int a = const_cond(); if (a) return 0; /* the compiler doesn't generate code for static funcs below */ static_func_1(); ... static_func_N();LTO also optimizes external functions. But not everyone uses it. So we still need macros for them, and of course data structures.
3. In 4/5, you have:
@@ -461,6 +461,7 @@ enum { struct lru_gen_mm_state { /* set to max_seq after each iteration */ unsigned long seq; +#ifdef CONFIG_LRU_TASK_PAGE_AGING /* where the current iteration continues after */ struct list_head *head; /* where the last iteration ended before */ @@ -469,6 +470,11 @@ struct lru_gen_mm_state { unsigned long *filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; +#else + /* protect the seq update above */ + /* May be we can use lruvec->lock? */ + spinlock_t lock; +#endif };The answer is yes, and not only that, we don't need
lru_gen_mm_stateat all.I'm attaching a patch that fixes all above. If you want to post it, please feel free -- fully test it please, since I didn't. Otherwise I can ask TJ to help make this work for you.
$ git diff --stat include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 12 +- include/linux/mmzone.h | 2 + kernel/bounds.c | 6 +- kernel/fork.c | 2 +- mm/vmscan.c | 169 +++++++++++++++++++-------- 6 files changed, 137 insertions(+), 56 deletions(-)On x86:
$ ./scripts/bloat-o-meter mm/vmscan.o.old mm/vmscan.o add/remove: 24/34 grow/shrink: 2/7 up/down: 966/-8716 (-7750) Function old new delta ... should_skip_vma 206 - -206 get_pte_pfn 261 - -261 lru_gen_add_mm 323 - -323 lru_gen_seq_show 1710 1370 -340 lru_gen_del_mm 432 - -432 reset_batch_size 572 - -572 try_to_inc_max_seq 2947 1635 -1312 walk_pmd_range_locked 1508 - -1508 walk_pud_range 3238 - -3238 Total: Before=99449, After=91699, chg -7.79%$ objdump -S mm/vmscan.o | grep -A 20 "<try_to_inc_max_seq>:" 000000000000a350 <try_to_inc_max_seq>: { a350: e8 00 00 00 00 call a355 <try_to_inc_max_seq+0x5> a355: 55 push %rbp a356: 48 89 e5 mov %rsp,%rbp a359: 41 57 push %r15 a35b: 41 56 push %r14 a35d: 41 55 push %r13 a35f: 41 54 push %r12 a361: 53 push %rbx a362: 48 83 ec 70 sub $0x70,%rsp a366: 41 89 d4 mov %edx,%r12d a369: 49 89 f6 mov %rsi,%r14
Quoted 8 lines
a36c: 49 89 ff mov %rdi,%r15
spin_lock_irq(&lruvec->lru_lock);
a36f: 48 8d 5f 50 lea 0x50(%rdi),%rbx
a373: 48 89 df mov %rbx,%rdi
a376: e8 00 00 00 00 call a37b <try_to_inc_max_seq+0x2b>
success = max_seq == lrugen->max_seq;
a37b: 49 8b 87 88 00 00 00 mov 0x88(%r15),%rax
a382: 4c 39 f0 cmp %r14,%raxFor the below diff:
@@ -4497,14 +4547,16 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
struct lru_gen_mm_walk *walk;
struct mm_struct *mm = NULL;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
+ struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
+ if (!mm_state)
+ return inc_max_seq(lruvec, max_seq, can_swap, force_scan);
+
/* see the comment in iterate_mm_list() */
- if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) {
- success = false;
- goto done;
- }
+ if (max_seq <= READ_ONCE(mm_state->seq))
+ return false;
/*
* If the hardware doesn't automatically set the accessed bit, fallback
@@ -4534,8 +4586,10 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
walk_mm(lruvec, mm, walk);
} while (mm);
done:
- if (success)
- inc_max_seq(lruvec, can_swap, force_scan);
+ if (success) {
+ success = inc_max_seq(lruvec, max_seq, can_swap, force_scan);
+ WARN_ON_ONCE(!success);
+ }
return success;
}
@
We did discuss a possible race that can happen if we allow multiple callers hit inc_max_seq at the same time. inc_max_seq drop the lru_lock and restart the loop at the previous value of type. ie. if we want to do the above we might also need the below?
modified mm/vmscan.c
@@ -4368,6 +4368,7 @@ void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
int type, zone;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
+retry:
spin_lock_irq(&lruvec->lru_lock);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
@@ -4381,7 +4382,7 @@ void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
while (!inc_min_seq(lruvec, type, can_swap)) {
spin_unlock_irq(&lruvec->lru_lock);
cond_resched();
- spin_lock_irq(&lruvec->lru_lock);
+ goto retry;
}
}
I also found that allowing only one cpu to increment max seq value and making other request with the same max_seq return false is also useful in performance runs. ie, we need an equivalent of this?
+ if (max_seq <= READ_ONCE(mm_state->seq))
+ return false;
Quoted 167 lines · Aneesh Kumar K V
Quoted 118 lines · Yu Zhao
Quoted 13 lines · Aneesh Kumar K.V
This patchset avoids building changes added by commit
bd74fdaea146("mm:
multi-gen LRU: support page table walks") on platforms that don't support hardware atomic updates of access bits.Aneesh Kumar K.V (5):
mm/mglru: Create a new helperiterate_mm_list_walkmm/mglru: Move Bloom filter code around
mm/mglru: Move code around to make future patch easy mm/mglru: moveiterate_mm_list_walkHelper mm/mglru: Don't build multi-gen LRU page table walk code on
architecture not supportedQuoted 10 lines
arch/Kconfig | 3 + arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 10 +- include/linux/mmzone.h | 12 +- kernel/fork.c | 2 +- mm/memcontrol.c | 2 +- mm/vmscan.c | 955 +++++++++++++++++++------------------ 9 files changed, 528 insertions(+), 460 deletions(-)1. There is no need for a new
Kconfig-- the condition is simplydefined(CONFIG_LRU_GEN)&& !defined(arch_has_hw_pte_young)2. The best practice to disable static functions is not by macros but:
static int const_cond(void) { return 1; } int main(void) { int a = const_cond(); if (a) return 0; /* the compiler doesn't generate code for static funcs below */ static_func_1(); ... static_func_N();LTO also optimizes external functions. But not everyone uses it. So we still need macros for them, and of course data structures.
3. In 4/5, you have:
@@ -461,6 +461,7 @@ enum { struct lru_gen_mm_state { /* set to max_seq after each iteration */ unsigned long seq; +#ifdef CONFIG_LRU_TASK_PAGE_AGING /* where the current iteration continues after */ struct list_head *head; /* where the last iteration ended before */ @@ -469,6 +470,11 @@ struct lru_gen_mm_state { unsigned long *filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; +#else + /* protect the seq update above */ + /* May be we can use lruvec->lock? */ + spinlock_t lock; +#endif };The answer is yes, and not only that, we don't need
lru_gen_mm_stateat all.I'm attaching a patch that fixes all above. If you want to post it, please feel free -- fully test it please, since I didn't. Otherwise I can ask TJ to help make this work for you.
$ git diff --stat include/linux/memcontrol.h | 2 +- include/linux/mm_types.h | 12 +- include/linux/mmzone.h | 2 + kernel/bounds.c | 6 +- kernel/fork.c | 2 +- mm/vmscan.c | 169 +++++++++++++++++++-------- 6 files changed, 137 insertions(+), 56 deletions(-)On x86:
$ ./scripts/bloat-o-meter mm/vmscan.o.old mm/vmscan.o add/remove: 24/34 grow/shrink: 2/7 up/down: 966/-8716 (-7750) Function old new delta ... should_skip_vma 206 - -206 get_pte_pfn 261 - -261 lru_gen_add_mm 323 - -323 lru_gen_seq_show 1710 1370 -340 lru_gen_del_mm 432 - -432 reset_batch_size 572 - -572 try_to_inc_max_seq 2947 1635 -1312 walk_pmd_range_locked 1508 - -1508 walk_pud_range 3238 - -3238 Total: Before=99449, After=91699, chg -7.79%$ objdump -S mm/vmscan.o | grep -A 20 "<try_to_inc_max_seq>:" 000000000000a350 <try_to_inc_max_seq>: { a350: e8 00 00 00 00 call a355 <try_to_inc_max_seq+0x5> a355: 55 push %rbp a356: 48 89 e5 mov %rsp,%rbp a359: 41 57 push %r15 a35b: 41 56 push %r14 a35d: 41 55 push %r13 a35f: 41 54 push %r12 a361: 53 push %rbx a362: 48 83 ec 70 sub $0x70,%rsp a366: 41 89 d4 mov %edx,%r12d a369: 49 89 f6 mov %rsi,%r14Quoted 8 lines
a36c: 49 89 ff mov %rdi,%r15 spin_lock_irq(&lruvec->lru_lock); a36f: 48 8d 5f 50 lea 0x50(%rdi),%rbx a373: 48 89 df mov %rbx,%rdi a376: e8 00 00 00 00 call a37b <try_to_inc_max_seq+0x2b> success = max_seq == lrugen->max_seq; a37b: 49 8b 87 88 00 00 00 mov 0x88(%r15),%rax a382: 4c 39 f0 cmp %r14,%raxFor the below diff:
mm/vmscan.c +10
-6@@ -4497,14 +4547,16 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,struct lru_gen_mm_walk *walk;struct mm_struct *mm =NULL ;struct lru_gen_folio *lrugen = &lruvec->lrugen; +struct lru_gen_mm_state *mm_state =get_mm_state (lruvec);VM_WARN_ON_ONCE (max_seq >READ_ONCE (lrugen->max_seq)); +if (!mm_state) +return inc_max_seq (lruvec, max_seq, can_swap, force_scan); +/* see the comment in iterate_mm_list() */ -if (max_seq <=READ_ONCE (lruvec->mm_state.seq)) { - success = false; - goto done; - } +if (max_seq <=READ_ONCE (mm_state->seq)) +return false;/* * If the hardware doesn't automatically set the accessed bit, fallback @@ -4534,8 +4586,10 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,walk_mm (lruvec, mm, walk); }while (mm); done: -if (success) -inc_max_seq (lruvec, can_swap, force_scan); +if (success) { + success =inc_max_seq (lruvec, max_seq, can_swap, force_scan); +WARN_ON_ONCE (!success); + }return success; }@
Quoted 3 lines
We did discuss a possible race that can happen if we allow multiple callers hit
inc_max_seqat the same time.inc_max_seqdrop thelru_lockand restart the loop at the previous value of type. ie. if we want to do the above we might also need the below?
Yes, you are right.
In fact, there is an existing bug here: even though max_seq can't change without the patch, min_seq still can, and if it does, the initial condition for inc_min_seq() to keep looping, i.e., nr_gens at max, doesn't hold anymore.
for (type = ANON_AND_FILE - 1; type >= 0; type--) {
if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
continue;This only affects the debugfs interface (force_scan=true), which is probably why it wasn't never reported.
Quoted 22 lines
modified
mm/vmscan.c(quoted diff) +2
-1@@ -4368,6 +4368,7 @@ void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) int type, zone; struct lru_gen_struct *lrugen = &lruvec->lrugen; +retry: spin_lock_irq(&lruvec->lru_lock); VM_WARN_ON_ONCE(!seq_is_valid(lruvec));@@ -4381,7 +4382,7 @@ void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) while (!inc_min_seq(lruvec, type, can_swap)) { spin_unlock_irq(&lruvec->lru_lock); cond_resched(); - spin_lock_irq(&lruvec->lru_lock); + goto retry; } }I also found that allowing only one cpu to increment max seq value and making other request with the same
max_seqreturn false is also useful in performance runs. ie, we need an equivalent of this?
mm/vmscan.c static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, · line 4559
Quoted 2 lines
mm/vmscan.c +2
-0+if (max_seq <=READ_ONCE (mm_state->seq)) +return false;
Yes, but the condition should be "<" -- ">" is a bug, which was asserted in try_to_in_max_seq().