incoming
Subsystems (62)
Andrew Morton
maintainerPATCH 85/131 mm: memcontrol: drop @compound parameter from memcg charging API
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: drop @compound parameter from memcg charging API
The memcg charging API carries a boolean @compound parameter that tells whether the page we're dealing with is a hugepage. mem_cgroup_commit_charge() has another boolean @lrucare that indicates whether the page needs LRU locking or not while charging. The majority of callsites know those parameters at compile time, which results in a lot of naked "false, false" argument lists. This makes for cryptic code and is a breeding ground for subtle mistakes.
Thankfully, the huge page state can be inferred from the page itself and doesn't need to be passed along. This is safe because charging completes before the page is published and somebody may split it.
Simplify the callsites by removing @compound, and let memcg infer the state by using hpage_nr_pages() unconditionally. That function does PageTransHuge() to identify huge pages, which also helpfully asserts that nobody passes in tail pages by accident.
The following patches will introduce a new charging API, best not to carry over unnecessary weight.
---
include/linux/memcontrol.h | 22 +++++++------------- kernel/events/uprobes.c | 6 ++--- mm/filemap.c | 6 ++--- mm/huge_memory.c | 8 +++---- mm/khugepaged.c | 20 +++++++++--------- mm/memcontrol.c | 38 +++++++++++++---------------------- mm/memory.c | 32 +++++++++++++---------------- mm/migrate.c | 6 ++--- mm/shmem.c | 22 ++++++++------------ mm/swapfile.c | 9 +++----- mm/userfaultfd.c | 6 ++--- 11 files changed, 77 insertions(+), 98 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +8 -14
@@ -359,15 +359,12 @@ enum mem_cgroup_protection mem_cgroup_pr struct mem_cgroup *memcg);int mem_cgroup_try_charge (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp, -bool compound); +gfp_t gfp_mask,struct mem_cgroup **memcgp);int mem_cgroup_try_charge_delay (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp, -bool compound); +gfp_t gfp_mask,struct mem_cgroup **memcgp);void mem_cgroup_commit_charge (struct page *page,struct mem_cgroup *memcg, -bool lrucare,bool compound); -void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg, -bool compound); +bool lrucare); +void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg);void mem_cgroup_uncharge (struct page *page);void mem_cgroup_uncharge_list (struct list_head *page_list);
@@ -849,8 +846,7 @@ static inline enum mem_cgroup_protectionstatic inline int mem_cgroup_try_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask, -struct mem_cgroup **memcgp, -bool compound) +struct mem_cgroup **memcgp) { *memcgp =NULL ;return 0 ;
@@ -859,8 +855,7 @@ static inline int mem_cgroup_try_charge(static inline int mem_cgroup_try_charge_delay (struct page *page,struct mm_struct *mm,gfp_t gfp_mask, -struct mem_cgroup **memcgp, -bool compound) +struct mem_cgroup **memcgp) { *memcgp =NULL ;return 0 ;
@@ -868,13 +863,12 @@ static inline int mem_cgroup_try_charge_static inline void mem_cgroup_commit_charge (struct page *page,struct mem_cgroup *memcg, -bool lrucare,bool compound) +bool lrucare) { }static inline void mem_cgroup_cancel_charge (struct page *page, -struct mem_cgroup *memcg, -bool compound) +struct mem_cgroup *memcg) { }
kernel/events/uprobes.c renamed from kernel/events/uprobes.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +3 -3
@@ -169,7 +169,7 @@ static int __replace_page(struct vm_areaif (new_page) { err =mem_cgroup_try_charge (new_page, vma->vm_mm,GFP_KERNEL , - &memcg, false); + &memcg);if (err)return err; }
@@ -181,7 +181,7 @@ static int __replace_page(struct vm_area err = -EAGAIN ;if (!page_vma_mapped_walk (&pvmw)) {if (new_page) -mem_cgroup_cancel_charge (new_page, memcg, false); +mem_cgroup_cancel_charge (new_page, memcg); goto unlock; }VM_BUG_ON_PAGE (addr != pvmw.address, old_page);
@@ -189,7 +189,7 @@ static int __replace_page(struct vm_areaif (new_page) {get_page (new_page);page_add_new_anon_rmap (new_page, vma, addr, false); -mem_cgroup_commit_charge (new_page, memcg, false, false); +mem_cgroup_commit_charge (new_page, memcg, false);lru_cache_add_active_or_unevictable (new_page, vma); }else /* no new page, just dec_mm_counter for old_page */
mm/filemap.c renamed from mm/filemap.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +3 -3
@@ -842,7 +842,7 @@ static int __add_to_page_cache_locked(stif (!huge) { error =mem_cgroup_try_charge (page, current->mm, - gfp_mask, &memcg, false); + gfp_mask, &memcg);if (error)return error; }
@@ -878,14 +878,14 @@ unlock: goto error;if (!huge) -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);trace_mm_filemap_add_to_page_cache (page);return 0 ; error: page->mapping =NULL ;/* Leave page->index set: truncation relies upon it */ if (!huge) -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg);put_page (page);return xas_error (&xas); }
mm/huge_memory.c renamed from mm/huge_memory.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +4 -4
@@ -594,7 +594,7 @@ static vm_fault_t __do_huge_pmd_anonymouVM_BUG_ON_PAGE (!PageCompound (page), page); -if (mem_cgroup_try_charge_delay (page, vma->vm_mm, gfp, &memcg, true)) { +if (mem_cgroup_try_charge_delay (page, vma->vm_mm, gfp, &memcg)) {put_page (page);count_vm_event (THP_FAULT_FALLBACK );count_vm_event (THP_FAULT_FALLBACK_CHARGE );
@@ -630,7 +630,7 @@ static vm_fault_t __do_huge_pmd_anonymouvm_fault_t ret2;spin_unlock (vmf->ptl); -mem_cgroup_cancel_charge (page, memcg, true); +mem_cgroup_cancel_charge (page, memcg);put_page (page);pte_free (vma->vm_mm, pgtable); ret2 =handle_userfault (vmf,VM_UFFD_MISSING );
@@ -641,7 +641,7 @@ static vm_fault_t __do_huge_pmd_anonymou entry =mk_huge_pmd (page, vma->vm_page_prot); entry =maybe_pmd_mkwrite (pmd_mkdirty (entry), vma);page_add_new_anon_rmap (page, vma, haddr, true); -mem_cgroup_commit_charge (page, memcg, false, true); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, vma);pgtable_trans_huge_deposit (vma->vm_mm, vmf->pmd, pgtable);set_pmd_at (vma->vm_mm, haddr, vmf->pmd, entry);
@@ -658,7 +658,7 @@ unlock_release: release:if (pgtable)pte_free (vma->vm_mm, pgtable); -mem_cgroup_cancel_charge (page, memcg, true); +mem_cgroup_cancel_charge (page, memcg);put_page (page);return ret;
mm/khugepaged.c renamed from mm/khugepaged.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +10 -10
@@ -1060,7 +1060,7 @@ static void collapse_huge_page(struct mm goto out_nolock; } -if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg, true))) { +if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out_nolock; }
@@ -1068,7 +1068,7 @@ static void collapse_huge_page(struct mmdown_read (&mm->mmap_sem); result =hugepage_vma_revalidate (mm, address, &vma);if (result) { -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1076,7 +1076,7 @@ static void collapse_huge_page(struct mm pmd =mm_find_pmd (mm, address);if (!pmd) { result =SCAN_PMD_NULL ; -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1088,7 +1088,7 @@ static void collapse_huge_page(struct mm*/ if (unmapped && !__collapse_huge_page_swapin (mm, vma, address, pmd, referenced)) { -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1176,7 +1176,7 @@ static void collapse_huge_page(struct mmspin_lock (pmd_ptl);BUG_ON (!pmd_none (*pmd));page_add_new_anon_rmap (new_page, vma, address, true); -mem_cgroup_commit_charge (new_page, memcg, false, true); +mem_cgroup_commit_charge (new_page, memcg, false);count_memcg_events (memcg,THP_COLLAPSE_ALLOC ,1 );lru_cache_add_active_or_unevictable (new_page, vma);pgtable_trans_huge_deposit (mm, pmd, pgtable);
@@ -1194,7 +1194,7 @@ out_nolock:trace_mm_collapse_huge_page (mm, isolated, result);return ; out: -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg); goto out_up_write; }
@@ -1637,7 +1637,7 @@ static void collapse_file(struct mm_stru goto out; } -if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg, true))) { +if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out; }
@@ -1650,7 +1650,7 @@ static void collapse_file(struct mm_strubreak ;xas_unlock_irq (&xas);if (!xas_nomem (&xas,GFP_KERNEL )) { -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg); result =SCAN_FAIL ; goto out; }
@@ -1887,7 +1887,7 @@ xa_unlocked:SetPageUptodate (new_page);page_ref_add (new_page,HPAGE_PMD_NR -1 ); -mem_cgroup_commit_charge (new_page, memcg, false, true); +mem_cgroup_commit_charge (new_page, memcg, false);if (is_shmem) {set_page_dirty (new_page);
@@ -1942,7 +1942,7 @@ xa_unlocked:VM_BUG_ON (nr_none);xas_unlock_irq (&xas); -mem_cgroup_cancel_charge (new_page, memcg, true); +mem_cgroup_cancel_charge (new_page, memcg); new_page->mapping =NULL ; }
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +15 -23
@@ -834,7 +834,7 @@ static unsigned long memcg_events_local(static void mem_cgroup_charge_statistics (struct mem_cgroup *memcg,struct page *page, -bool compound,int nr_pages) +int nr_pages) {/* * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
@@ -848,7 +848,7 @@ static void mem_cgroup_charge_statistics__mod_memcg_state (memcg,NR_SHMEM , nr_pages); } -if (compound) { +if (abs (nr_pages) >1 ) {VM_BUG_ON_PAGE (!PageTransHuge (page), page);__mod_memcg_state (memcg,MEMCG_RSS_HUGE , nr_pages); }
@@ -5501,9 +5501,9 @@ static int mem_cgroup_move_account(struc ret =0 ;local_irq_disable (); -mem_cgroup_charge_statistics (to, page, compound, nr_pages); +mem_cgroup_charge_statistics (to, page, nr_pages);memcg_check_events (to, page); -mem_cgroup_charge_statistics (from, page, compound, -nr_pages); +mem_cgroup_charge_statistics (from, page, -nr_pages);memcg_check_events (from, page);local_irq_enable (); out_unlock:
@@ -6494,7 +6494,6 @@ out:* @mm: mm context of the victim * @gfp_mask: reclaim mode * @memcgp: charged memcg return -* @compound: charge the page as compound or small page * * Try to charge @page to the memcg that @mm belongs to, reclaiming * pages according to @gfp_mask if necessary.
@@ -6507,11 +6506,10 @@ out:* with mem_cgroup_cancel_charge() in case page instantiation fails. */ int mem_cgroup_try_charge (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp, -bool compound) +gfp_t gfp_mask,struct mem_cgroup **memcgp) { +unsigned int nr_pages =hpage_nr_pages (page);struct mem_cgroup *memcg =NULL ; -unsigned int nr_pages = compound ?hpage_nr_pages (page) :1 ;int ret =0 ;if (mem_cgroup_disabled ())
@@ -6553,13 +6551,12 @@ out: }int mem_cgroup_try_charge_delay (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp, -bool compound) +gfp_t gfp_mask,struct mem_cgroup **memcgp) {struct mem_cgroup *memcg;int ret; - ret =mem_cgroup_try_charge (page, mm, gfp_mask, memcgp, compound); + ret =mem_cgroup_try_charge (page, mm, gfp_mask, memcgp); memcg = *memcgp;mem_cgroup_throttle_swaprate (memcg,page_to_nid (page), gfp_mask);return ret;
@@ -6570,7 +6567,6 @@ int mem_cgroup_try_charge_delay(struct p* @page: page to charge * @memcg: memcg to charge the page to * @lrucare: page might be on LRU already -* @compound: charge the page as compound or small page * * Finalize a charge transaction started by mem_cgroup_try_charge(), * after page->mapping has been set up. This must happen atomically
@@ -6583,9 +6579,9 @@ int mem_cgroup_try_charge_delay(struct p* Use mem_cgroup_cancel_charge() to cancel the transaction instead. */ void mem_cgroup_commit_charge (struct page *page,struct mem_cgroup *memcg, -bool lrucare,bool compound) +bool lrucare) { -unsigned int nr_pages = compound ?hpage_nr_pages (page) :1 ; +unsigned int nr_pages =hpage_nr_pages (page);VM_BUG_ON_PAGE (!page->mapping, page);VM_BUG_ON_PAGE (PageLRU (page) && !lrucare, page);
@@ -6603,7 +6599,7 @@ void mem_cgroup_commit_charge(struct pagcommit_charge (page, memcg, lrucare);local_irq_disable (); -mem_cgroup_charge_statistics (memcg, page, compound, nr_pages); +mem_cgroup_charge_statistics (memcg, page, nr_pages);memcg_check_events (memcg, page);local_irq_enable ();
@@ -6622,14 +6618,12 @@ void mem_cgroup_commit_charge(struct pag* mem_cgroup_cancel_charge - cancel a page charge * @page: page to charge * @memcg: memcg to charge the page to -* @compound: charge the page as compound or small page * * Cancel a charge transaction started by mem_cgroup_try_charge(). */ -void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg, -bool compound) +void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg) { -unsigned int nr_pages = compound ?hpage_nr_pages (page) :1 ; +unsigned int nr_pages =hpage_nr_pages (page);if (mem_cgroup_disabled ())return ;
@@ -6844,8 +6838,7 @@ void mem_cgroup_migrate(struct page *oldcommit_charge (newpage, memcg, false);local_irq_save (flags); -mem_cgroup_charge_statistics (memcg, newpage,PageTransHuge (newpage), - nr_pages); +mem_cgroup_charge_statistics (memcg, newpage, nr_pages);memcg_check_events (memcg, newpage);local_irq_restore (flags); }
@@ -7075,8 +7068,7 @@ void mem_cgroup_swapout(struct page *pag* only synchronisation we have for updating the per-CPU variables. */ VM_BUG_ON (!irqs_disabled ()); -mem_cgroup_charge_statistics (memcg, page,PageTransHuge (page), - -nr_entries); +mem_cgroup_charge_statistics (memcg, page, -nr_entries);memcg_check_events (memcg, page);if (!mem_cgroup_is_root (memcg))
mm/memory.c renamed from mm/memory.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +15 -17
@@ -2676,7 +2676,7 @@ static vm_fault_t wp_page_copy(struct vm } } -if (mem_cgroup_try_charge_delay (new_page, mm,GFP_KERNEL , &memcg, false)) +if (mem_cgroup_try_charge_delay (new_page, mm,GFP_KERNEL , &memcg)) goto oom_free_new;__SetPageUptodate (new_page);
@@ -2711,7 +2711,7 @@ static vm_fault_t wp_page_copy(struct vm*/ ptep_clear_flush_notify (vma, vmf->address, vmf->pte);page_add_new_anon_rmap (new_page, vma, vmf->address, false); -mem_cgroup_commit_charge (new_page, memcg, false, false); +mem_cgroup_commit_charge (new_page, memcg, false);lru_cache_add_active_or_unevictable (new_page, vma);/* * We call the notify macro here because, when using secondary
@@ -2750,7 +2750,7 @@ static vm_fault_t wp_page_copy(struct vm new_page = old_page; page_copied =1 ; }else { -mem_cgroup_cancel_charge (new_page, memcg, false); +mem_cgroup_cancel_charge (new_page, memcg); }if (new_page)
@@ -3193,8 +3193,7 @@ vm_fault_t do_swap_page(struct vm_fault goto out_page; } -if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , - &memcg, false)) { +if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , &memcg)) { ret =VM_FAULT_OOM ; goto out_page; }
@@ -3245,11 +3244,11 @@ vm_fault_t do_swap_page(struct vm_fault/* ksm created a completely new copy */ if (unlikely (page != swapcache && swapcache)) {page_add_new_anon_rmap (page, vma, vmf->address, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, vma); }else {do_page_add_anon_rmap (page, vma, vmf->address, exclusive); -mem_cgroup_commit_charge (page, memcg, true, false); +mem_cgroup_commit_charge (page, memcg, true);activate_page (page); }
@@ -3285,7 +3284,7 @@ unlock: out:return ret; out_nomap: -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg);pte_unmap_unlock (vmf->pte, vmf->ptl); out_page:unlock_page (page);
@@ -3359,8 +3358,7 @@ static vm_fault_t do_anonymous_page(struif (!page) goto oom; -if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , &memcg, - false)) +if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , &memcg)) goto oom_free_page;/*
@@ -3386,14 +3384,14 @@ static vm_fault_t do_anonymous_page(stru/* Deliver the page fault to userland, check inside PT lock */ if (userfaultfd_missing (vma)) {pte_unmap_unlock (vmf->pte, vmf->ptl); -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg);put_page (page);return handle_userfault (vmf,VM_UFFD_MISSING ); }inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES );page_add_new_anon_rmap (page, vma, vmf->address, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, vma); setpte:set_pte_at (vma->vm_mm, vmf->address, vmf->pte, entry);
@@ -3404,7 +3402,7 @@ unlock:pte_unmap_unlock (vmf->pte, vmf->ptl);return ret; release: -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg);put_page (page); goto unlock; oom_free_page:
@@ -3655,7 +3653,7 @@ vm_fault_t alloc_set_pte(struct vm_faultif (write && !(vma->vm_flags &VM_SHARED )) {inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES );page_add_new_anon_rmap (page, vma, vmf->address, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, vma); }else {inc_mm_counter_fast (vma->vm_mm,mm_counter_file (page));
@@ -3864,8 +3862,8 @@ static vm_fault_t do_cow_fault(struct vmif (!vmf->cow_page)return VM_FAULT_OOM ; -if (mem_cgroup_try_charge_delay (vmf->cow_page, vma->vm_mm,GFP_KERNEL , - &vmf->memcg, false)) { +if (mem_cgroup_try_charge_delay (vmf->cow_page, vma->vm_mm, +GFP_KERNEL , &vmf->memcg)) {put_page (vmf->cow_page);return VM_FAULT_OOM ; }
@@ -3886,7 +3884,7 @@ static vm_fault_t do_cow_fault(struct vm goto uncharge_out;return ret; uncharge_out: -mem_cgroup_cancel_charge (vmf->cow_page, vmf->memcg, false); +mem_cgroup_cancel_charge (vmf->cow_page, vmf->memcg);put_page (vmf->cow_page);return ret; }
mm/migrate.c renamed from mm/migrate.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +3 -3
@@ -2780,7 +2780,7 @@ static void migrate_vma_insert_page(struif (unlikely (anon_vma_prepare (vma))) goto abort; -if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , &memcg, false)) +if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , &memcg)) goto abort;/*
@@ -2826,7 +2826,7 @@ static void migrate_vma_insert_page(struinc_mm_counter (mm,MM_ANONPAGES );page_add_new_anon_rmap (page, vma, addr, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);if (!is_zone_device_page (page))lru_cache_add_active_or_unevictable (page, vma);get_page (page);
@@ -2848,7 +2848,7 @@ static void migrate_vma_insert_page(stru unlock_abort:pte_unmap_unlock (ptep, ptl); -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg); abort: *src &= ~MIGRATE_PFN_MIGRATE ; }
mm/shmem.c renamed from mm/shmem.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +9 -13
@@ -1664,8 +1664,7 @@ static int shmem_swapin_page(struct inod goto failed; } - error =mem_cgroup_try_charge_delay (page, charge_mm, gfp, &memcg, - false); + error =mem_cgroup_try_charge_delay (page, charge_mm, gfp, &memcg);if (!error) { error =shmem_add_to_page_cache (page, mapping, index,swp_to_radix_entry (swap), gfp);
@@ -1680,14 +1679,14 @@ static int shmem_swapin_page(struct inod* the rest. */ if (error) { -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg);delete_from_swap_cache (page); } }if (error) goto failed; -mem_cgroup_commit_charge (page, memcg, true, false); +mem_cgroup_commit_charge (page, memcg, true);spin_lock_irq (&info->lock); info->swapped--;
@@ -1859,8 +1858,7 @@ alloc_nohuge:if (sgp ==SGP_WRITE )__SetPageReferenced (page); - error =mem_cgroup_try_charge_delay (page, charge_mm, gfp, &memcg, -PageTransHuge (page)); + error =mem_cgroup_try_charge_delay (page, charge_mm, gfp, &memcg);if (error) {if (PageTransHuge (page)) {count_vm_event (THP_FILE_FALLBACK );
@@ -1871,12 +1869,10 @@ alloc_nohuge: error =shmem_add_to_page_cache (page, mapping, hindex,NULL , gfp &GFP_RECLAIM_MASK );if (error) { -mem_cgroup_cancel_charge (page, memcg, -PageTransHuge (page)); +mem_cgroup_cancel_charge (page, memcg); goto unacct; } -mem_cgroup_commit_charge (page, memcg, false, -PageTransHuge (page)); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_anon (page);spin_lock_irq (&info->lock);
@@ -2364,7 +2360,7 @@ static int shmem_mfill_atomic_pte(structif (unlikely (offset >= max_off)) goto out_release; - ret =mem_cgroup_try_charge_delay (page, dst_mm, gfp, &memcg, false); + ret =mem_cgroup_try_charge_delay (page, dst_mm, gfp, &memcg);if (ret) goto out_release;
@@ -2373,7 +2369,7 @@ static int shmem_mfill_atomic_pte(structif (ret) goto out_release_uncharge; -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false); _dst_pte =mk_pte (page, dst_vma->vm_page_prot);if (dst_vma->vm_flags &VM_WRITE )
@@ -2424,7 +2420,7 @@ out_release_uncharge_unlock:ClearPageDirty (page);delete_from_page_cache (page); out_release_uncharge: -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg); out_release:unlock_page (page);put_page (page);
mm/swapfile.c renamed from mm/swapfile.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +4 -5
@@ -1902,15 +1902,14 @@ static int unuse_pte(struct vm_area_struif (unlikely (!page))return -ENOMEM ; -if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , - &memcg, false)) { +if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , &memcg)) { ret = -ENOMEM ; goto out_nolock; } pte =pte_offset_map_lock (vma->vm_mm, pmd, addr, &ptl);if (unlikely (!pte_same_as_swp (*pte,swp_entry_to_pte (entry)))) { -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg); ret =0 ; goto out; }
@@ -1922,10 +1921,10 @@ static int unuse_pte(struct vm_area_stru pte_mkold(mk_pte(page, vma->vm_page_prot)));if (page == swapcache) {page_add_anon_rmap (page, vma, addr, false); -mem_cgroup_commit_charge (page, memcg, true, false); +mem_cgroup_commit_charge (page, memcg, true); }else {/* ksm created a completely new copy */ page_add_new_anon_rmap (page, vma, addr, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, vma); }swap_free (entry);
mm/userfaultfd.c renamed from mm/userfaultfd.c~mm-memcontrol-drop-compound-parameter-from-memcg-charging-api +3 -3
@@ -97,7 +97,7 @@ static int mcopy_atomic_pte(struct mm_st__SetPageUptodate (page); ret = -ENOMEM ; -if (mem_cgroup_try_charge (page, dst_mm,GFP_KERNEL , &memcg, false)) +if (mem_cgroup_try_charge (page, dst_mm,GFP_KERNEL , &memcg)) goto out_release; _dst_pte =pte_mkdirty (mk_pte (page, dst_vma->vm_page_prot));
@@ -124,7 +124,7 @@ static int mcopy_atomic_pte(struct mm_stinc_mm_counter (dst_mm,MM_ANONPAGES );page_add_new_anon_rmap (page, dst_vma, dst_addr, false); -mem_cgroup_commit_charge (page, memcg, false, false); +mem_cgroup_commit_charge (page, memcg, false);lru_cache_add_active_or_unevictable (page, dst_vma);set_pte_at (dst_mm, dst_addr, dst_pte, _dst_pte);
@@ -138,7 +138,7 @@ out:return ret; out_release_uncharge_unlock:pte_unmap_unlock (dst_pte, ptl); -mem_cgroup_cancel_charge (page, memcg, false); +mem_cgroup_cancel_charge (page, memcg); out_release:put_page (page); goto out;
_
Andrew Morton
maintainerPATCH 86/131 mm: shmem: remove rare optimization when swapin races with hole punching
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: shmem: remove rare optimization when swapin races with hole punching
Commit 215c02bc33bb ("tmpfs: fix shmem_getpage_gfp() VM_BUG_ON") recognized that hole punching can race with swapin and removed the BUG_ON() for a truncated entry from the swapin path.
The patch also added a swapcache deletion to optimize this rare case:
Since swapin has the page locked, and free_swap_and_cache() merely trylocks, this situation can leave the page stranded in swapcache. Usually, page reclaim picks up stale swapcache pages, and the race can happen at any other time when the page is locked. (The same happens for non-shmem swapin racing with page table zapping.) The thinking here was:
we already observed the race and we have the page locked, we may as well do the cleanup instead of waiting for reclaim.
However, this optimization complicates the next patch which moves the cgroup charging code around. As this is just a minor speedup for a race condition that is so rare that it required a fuzzer to trigger the original BUG_ON(), it's no longer worth the complications.
---
mm/shmem.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-)
mm/shmem.c renamed from mm/shmem.c~mm-shmem-remove-rare-optimization-when-swapin-races-with-hole-punching +7 -18
@@ -1665,27 +1665,16 @@ static int shmem_swapin_page(struct inod } error =mem_cgroup_try_charge_delay (page, charge_mm, gfp, &memcg); -if (!error) { - error =shmem_add_to_page_cache (page, mapping, index, -swp_to_radix_entry (swap), gfp); -/* -* We already confirmed swap under page lock, and make -* no memory allocation here, so usually no possibility -* of error; but free_swap_and_cache() only trylocks a -* page, so it is just possible that the entry has been -* truncated or holepunched since swap was confirmed. -* shmem_undo_range() will have done some of the -* unaccounting, now delete_from_swap_cache() will do -* the rest. -*/ -if (error) { -mem_cgroup_cancel_charge (page, memcg); -delete_from_swap_cache (page); - } - }if (error) goto failed; + error =shmem_add_to_page_cache (page, mapping, index, +swp_to_radix_entry (swap), gfp); +if (error) { +mem_cgroup_cancel_charge (page, memcg); + goto failed; + } +mem_cgroup_commit_charge (page, memcg, true);spin_lock_irq (&info->lock);
_
Andrew Morton
maintainerPATCH 87/131 mm: memcontrol: move out cgroup swaprate throttling
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: move out cgroup swaprate throttling
The cgroup swaprate throttling is about matching new anon allocations to the rate of available IO when that is being throttled. It's the io controller hooking into the VM, rather than a memory controller thing.
Rename mem_cgroup_throttle_swaprate() to cgroup_throttle_swaprate(), and drop the @memcg argument which is only used to check whether the preceding page charge has succeeded and the fault is proceeding.
We could decouple the call from mem_cgroup_try_charge() here as well, but that would cause unnecessary churn: the following patches convert all callsites to a new charge API and we'll decouple as we go along.
---
include/linux/swap.h | 6 ++---- mm/memcontrol.c | 5 ++--- mm/swapfile.c | 14 +++++++------- 3 files changed, 11 insertions(+), 14 deletions(-)
include/linux/swap.h renamed from include/linux/swap.h~mm-memcontrol-move-out-cgroup-swaprate-throttling +2 -4
@@ -651,11 +651,9 @@ static inline int mem_cgroup_swappiness(#endif #if defined(CONFIG_SWAP ) && defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -extern void mem_cgroup_throttle_swaprate (struct mem_cgroup *memcg,int node, -gfp_t gfp_mask); +extern void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask);#else -static inline void mem_cgroup_throttle_swaprate (struct mem_cgroup *memcg, -int node,gfp_t gfp_mask) +static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) { }#endif
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-move-out-cgroup-swaprate-throttling +2 -3
@@ -6553,12 +6553,11 @@ out:int mem_cgroup_try_charge_delay (struct page *page,struct mm_struct *mm,gfp_t gfp_mask,struct mem_cgroup **memcgp) { -struct mem_cgroup *memcg;int ret; ret =mem_cgroup_try_charge (page, mm, gfp_mask, memcgp); - memcg = *memcgp; -mem_cgroup_throttle_swaprate (memcg,page_to_nid (page), gfp_mask); +if (*memcgp) +cgroup_throttle_swaprate (page, gfp_mask);return ret; }
mm/swapfile.c renamed from mm/swapfile.c~mm-memcontrol-move-out-cgroup-swaprate-throttling +7 -7
@@ -3798,11 +3798,12 @@ static void free_swap_count_continuation }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -void mem_cgroup_throttle_swaprate (struct mem_cgroup *memcg,int node, -gfp_t gfp_mask) +void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {struct swap_info_struct *si, *next; -if (!(gfp_mask & __GFP_IO) || !memcg) +int nid =page_to_nid (page); + +if (!(gfp_mask & __GFP_IO))return ;if (!blk_cgroup_congested ())
@@ -3816,11 +3817,10 @@ void mem_cgroup_throttle_swaprate(structreturn ;spin_lock (&swap_avail_lock); -plist_for_each_entry_safe (si, next, &swap_avail_heads[node], - avail_lists[node]) { +plist_for_each_entry_safe (si, next, &swap_avail_heads[nid], + avail_lists[nid]) {if (si->bdev) { -blkcg_schedule_throttle (bdev_get_queue (si->bdev), - true); +blkcg_schedule_throttle (bdev_get_queue (si->bdev), true);break ; } }
_
Andrew Morton
maintainerPATCH 88/131 mm: memcontrol: convert page cache to a new mem_cgroup_charge() API
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: convert page cache to a new mem_cgroup_charge() API
The try/commit/cancel protocol that memcg uses dates back to when pages used to be uncharged upon removal from the page cache, and thus couldn't be committed before the insertion had succeeded. Nowadays, pages are uncharged when they are physically freed; it doesn't matter whether the insertion was successful or not. For the page cache, the transaction dance has become unnecessary.
Introduce a mem_cgroup_charge() function that simply charges a newly allocated page to a cgroup and sets up page->mem_cgroup in one single step. If the insertion fails, the caller doesn't have to do anything but free/put the page.
Then switch the page cache over to this new API.
Subsequent patches will also convert anon pages, but it needs a bit more prep work. Right now, memcg depends on page->mapping being already set up at the time of charging, so that it can maintain its own MEMCG_CACHE and MEMCG_RSS counters. For anon, page->mapping is set under the same pte lock under which the page is publishd, so a single charge point that can block doesn't work there just yet.
The following prep patches will replace the private memcg counters with the generic vmstat counters, thus removing the page->mapping dependency, then complete the transition to the new single-point charge API and delete the old transactional scheme.
v2: leave shmem swapcache when charging fails to avoid double IO (Joonsoo)
v3: rebase on preceeding shmem simplification patch
---
include/linux/memcontrol.h | 10 ++++ mm/filemap.c | 24 ++++------- mm/memcontrol.c | 29 ++++++++++++- mm/shmem.c | 73 ++++++++++++++--------------------- 4 files changed, 77 insertions(+), 59 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-convert-page-cache-to-a-new-mem_cgroup_charge-api +10 -0
@@ -365,6 +365,10 @@ int mem_cgroup_try_charge_delay(struct pvoid mem_cgroup_commit_charge (struct page *page,struct mem_cgroup *memcg,bool lrucare);void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg); + +int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask, +bool lrucare); +void mem_cgroup_uncharge (struct page *page);void mem_cgroup_uncharge_list (struct list_head *page_list);
@@ -872,6 +876,12 @@ static inline void mem_cgroup_cancel_cha { } +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask,bool lrucare) +{ +return 0 ; +} +static inline void mem_cgroup_uncharge (struct page *page) { }
mm/filemap.c renamed from mm/filemap.c~mm-memcontrol-convert-page-cache-to-a-new-mem_cgroup_charge-api +10 -14
@@ -832,7 +832,6 @@ static int __add_to_page_cache_locked(st {XA_STATE (xas, &mapping->i_pages, offset);int huge =PageHuge (page); -struct mem_cgroup *memcg;int error;void *old;
@@ -840,17 +839,16 @@ static int __add_to_page_cache_locked(stVM_BUG_ON_PAGE (PageSwapBacked (page), page);mapping_set_update (&xas, mapping); -if (!huge) { - error =mem_cgroup_try_charge (page, current->mm, - gfp_mask, &memcg); -if (error) -return error; - } -get_page (page); page->mapping = mapping; page->index = offset; +if (!huge) { + error =mem_cgroup_charge (page, current->mm, gfp_mask, false); +if (error) + goto error; + } + do { xas_lock_irq(&xas); old = xas_load(&xas);
@@ -874,20 +872,18 @@ unlock:xas_unlock_irq (&xas); }while (xas_nomem (&xas, gfp_mask &GFP_RECLAIM_MASK )); -if (xas_error (&xas)) +if (xas_error (&xas)) { + error =xas_error (&xas); goto error; + } -if (!huge) -mem_cgroup_commit_charge (page, memcg, false);trace_mm_filemap_add_to_page_cache (page);return 0 ; error: page->mapping =NULL ;/* Leave page->index set: truncation relies upon it */ -if (!huge) -mem_cgroup_cancel_charge (page, memcg);put_page (page); -return xas_error (&xas); +return error; }ALLOW_ERROR_INJECTION (__add_to_page_cache_locked,ERRNO );
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-convert-page-cache-to-a-new-mem_cgroup_charge-api +27 -2
@@ -6637,6 +6637,33 @@ void mem_cgroup_cancel_charge(struct pagcancel_charge (memcg, nr_pages); } +/** +* mem_cgroup_charge - charge a newly allocated page to a cgroup +* @page: page to charge +* @mm: mm context of the victim +* @gfp_mask: reclaim mode +* @lrucare: page might be on the LRU already +* +* Try to charge @page to the memcg that @mm belongs to, reclaiming +* pages according to @gfp_mask if necessary. +* +* Returns 0 on success. Otherwise, an error code is returned. +*/ +int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask, +bool lrucare) +{ +struct mem_cgroup *memcg; +int ret; + +VM_BUG_ON_PAGE (!page->mapping, page); + + ret =mem_cgroup_try_charge (page, mm, gfp_mask, &memcg); +if (ret) +return ret; +mem_cgroup_commit_charge (page, memcg, lrucare); +return 0 ; +} +struct uncharge_gather {struct mem_cgroup *memcg;unsigned long pgpgout;
@@ -6684,8 +6711,6 @@ static void uncharge_batch(const structstatic void uncharge_page (struct page *page,struct uncharge_gather *ug) {VM_BUG_ON_PAGE (PageLRU (page), page); -VM_BUG_ON_PAGE (page_count (page) && !is_zone_device_page (page) && - !PageHWPoison (page) , page);if (!page->mem_cgroup)return ;
mm/shmem.c renamed from mm/shmem.c~mm-memcontrol-convert-page-cache-to-a-new-mem_cgroup_charge-api +30 -43
158 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 89/131 mm: memcontrol: prepare uncharging for removal of private page type counters
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: prepare uncharging for removal of private page type counters
The uncharge batching code adds up the anon, file, kmem counts to determine the total number of pages to uncharge and references to drop. But the next patches will remove the anon and file counters.
Maintain an aggregate nr_pages in the uncharge_gather struct.
---
mm/memcontrol.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-prepare-uncharging-for-removal-of-private-page-type-counters +12 -11
@@ -6666,6 +6666,7 @@ int mem_cgroup_charge(struct page *page,struct uncharge_gather {struct mem_cgroup *memcg; +unsigned long nr_pages;unsigned long pgpgout;unsigned long nr_anon;unsigned long nr_file;
@@ -6682,13 +6683,12 @@ static inline void uncharge_gather_clearstatic void uncharge_batch (const struct uncharge_gather *ug) { -unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;unsigned long flags;if (!mem_cgroup_is_root (ug->memcg)) { -page_counter_uncharge (&ug->memcg->memory, nr_pages); +page_counter_uncharge (&ug->memcg->memory, ug->nr_pages);if (do_memsw_account ()) -page_counter_uncharge (&ug->memcg->memsw, nr_pages); +page_counter_uncharge (&ug->memcg->memsw, ug->nr_pages);if (!cgroup_subsys_on_dfl (memory_cgrp_subsys) && ug->nr_kmem)page_counter_uncharge (&ug->memcg->kmem, ug->nr_kmem);memcg_oom_recover (ug->memcg);
@@ -6700,16 +6700,18 @@ static void uncharge_batch(const struct__mod_memcg_state (ug->memcg,MEMCG_RSS_HUGE , -ug->nr_huge);__mod_memcg_state (ug->memcg,NR_SHMEM , -ug->nr_shmem);__count_memcg_events (ug->memcg,PGPGOUT , ug->pgpgout); -__this_cpu_add (ug->memcg->vmstats_percpu->nr_page_events, nr_pages); +__this_cpu_add (ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);memcg_check_events (ug->memcg, ug->dummy_page);local_irq_restore (flags);if (!mem_cgroup_is_root (ug->memcg)) -css_put_many (&ug->memcg->css, nr_pages); +css_put_many (&ug->memcg->css, ug->nr_pages); }static void uncharge_page (struct page *page,struct uncharge_gather *ug) { +unsigned long nr_pages; +VM_BUG_ON_PAGE (PageLRU (page), page);if (!page->mem_cgroup)
@@ -6729,13 +6731,12 @@ static void uncharge_page(struct page *p ug->memcg = page->mem_cgroup; } -if (!PageKmemcg (page)) { -unsigned int nr_pages =1 ; + nr_pages =compound_nr (page); + ug->nr_pages += nr_pages; -if (PageTransHuge (page)) { - nr_pages =compound_nr (page); +if (!PageKmemcg (page)) { +if (PageTransHuge (page)) ug->nr_huge += nr_pages; - }if (PageAnon (page)) ug->nr_anon += nr_pages;else {
@@ -6745,7 +6746,7 @@ static void uncharge_page(struct page *p } ug->pgpgout++; }else { - ug->nr_kmem +=compound_nr (page); + ug->nr_kmem += nr_pages;__ClearPageKmemcg (page); }
_
Andrew Morton
maintainerPATCH 90/131 mm: memcontrol: prepare move_account for removal of private page type counters
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: prepare move_account for removal of private page type counters
When memcg uses the generic vmstat counters, it doesn't need to do anything at charging and uncharging time. It does, however, need to migrate counts when pages move to a different cgroup in move_account.
Prepare the move_account function for the arrival of NR_FILE_PAGES, NR_ANON_MAPPED, NR_ANON_THPS etc. by having a branch for files and a branch for anon, which can then divided into sub-branches.
---
mm/memcontrol.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-)
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-prepare-move_account-for-removal-of-private-page-type-counters +13 -12
@@ -5434,7 +5434,6 @@ static int mem_cgroup_move_account(strucstruct pglist_data *pgdat;unsigned int nr_pages = compound ?hpage_nr_pages (page) :1 ;int ret; -bool anon;VM_BUG_ON (from == to);VM_BUG_ON_PAGE (PageLRU (page), page);
@@ -5452,25 +5451,27 @@ static int mem_cgroup_move_account(strucif (page->mem_cgroup != from) goto out_unlock; - anon =PageAnon (page); - pgdat =page_pgdat (page); from_vec =mem_cgroup_lruvec (from, pgdat); to_vec =mem_cgroup_lruvec (to, pgdat);lock_page_memcg (page); -if (!anon &&page_mapped (page)) { -__mod_lruvec_state (from_vec,NR_FILE_MAPPED , -nr_pages); -__mod_lruvec_state (to_vec,NR_FILE_MAPPED , nr_pages); - } +if (!PageAnon (page)) { +if (page_mapped (page)) { +__mod_lruvec_state (from_vec,NR_FILE_MAPPED , -nr_pages); +__mod_lruvec_state (to_vec,NR_FILE_MAPPED , nr_pages); + } -if (!anon &&PageDirty (page)) { -struct address_space *mapping =page_mapping (page); +if (PageDirty (page)) { +struct address_space *mapping =page_mapping (page); -if (mapping_cap_account_dirty (mapping)) { -__mod_lruvec_state (from_vec,NR_FILE_DIRTY , -nr_pages); -__mod_lruvec_state (to_vec,NR_FILE_DIRTY , nr_pages); +if (mapping_cap_account_dirty (mapping)) { +__mod_lruvec_state (from_vec,NR_FILE_DIRTY , + -nr_pages); +__mod_lruvec_state (to_vec,NR_FILE_DIRTY , + nr_pages); + } } }
_
Andrew Morton
maintainerPATCH 91/131 mm: memcontrol: prepare cgroup vmstat infrastructure for native anon counters
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: prepare cgroup vmstat infrastructure for native anon counters
Anonymous compound pages can be mapped by ptes, which means that if we want to track NR_MAPPED_ANON, NR_ANON_THPS on a per-cgroup basis, we have to be prepared to see tail pages in our accounting functions.
Make mod_lruvec_page_state() and lock_page_memcg() deal with tail pages correctly, namely by redirecting to the head page which has the page->mem_cgroup set up.
---
include/linux/memcontrol.h | 5 +++-- mm/memcontrol.c | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-prepare-cgroup-vmstat-infrastructure-for-native-anon-counters +3 -2
@@ -709,16 +709,17 @@ static inline void mod_lruvec_state(strustatic inline void __mod_lruvec_page_state (struct page *page,enum node_stat_item idx,int val) { +struct page *head =compound_head (page);/* rmap on tail pages */ pg_data_t *pgdat =page_pgdat (page);struct lruvec *lruvec;/* Untracked pages have no memcg, no lruvec. Update only the node */ -if (!page->mem_cgroup) { +if (!head->mem_cgroup) {__mod_node_page_state (pgdat, idx, val);return ; } - lruvec =mem_cgroup_lruvec (page->mem_cgroup, pgdat); + lruvec =mem_cgroup_lruvec (head->mem_cgroup, pgdat);__mod_lruvec_state (lruvec, idx, val); }
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-prepare-cgroup-vmstat-infrastructure-for-native-anon-counters +6 -3
@@ -1981,6 +1981,7 @@ void mem_cgroup_print_oom_group(struct m*/ struct mem_cgroup *lock_page_memcg (struct page *page) { +struct page *head =compound_head (page);/* rmap on tail pages */ struct mem_cgroup *memcg;unsigned long flags;
@@ -2000,7 +2001,7 @@ struct mem_cgroup *lock_page_memcg(strucif (mem_cgroup_disabled ())return NULL ; again: - memcg = page->mem_cgroup; + memcg = head->mem_cgroup;if (unlikely (!memcg))return NULL ;
@@ -2008,7 +2009,7 @@ again:return memcg;spin_lock_irqsave (&memcg->move_lock, flags); -if (memcg != page->mem_cgroup) { +if (memcg != head->mem_cgroup) {spin_unlock_irqrestore (&memcg->move_lock, flags); goto again; }
@@ -2051,7 +2052,9 @@ void __unlock_page_memcg(struct mem_cgro*/ void unlock_page_memcg (struct page *page) { -__unlock_page_memcg (page->mem_cgroup); +struct page *head =compound_head (page); + +__unlock_page_memcg (head->mem_cgroup); }EXPORT_SYMBOL (unlock_page_memcg);
_
Andrew Morton
maintainerPATCH 92/131 mm: memcontrol: switch to native NR_FILE_PAGES and NR_SHMEM counters
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: switch to native NR_FILE_PAGES and NR_SHMEM counters
Memcg maintains private MEMCG_CACHE and NR_SHMEM counters. This divergence from the generic VM accounting means unnecessary code overhead, and creates a dependency for memcg that page->mapping is set up at the time of charging, so that page types can be told apart.
Convert the generic accounting sites to mod_lruvec_page_state and friends to maintain the per-cgroup vmstat counters of NR_FILE_PAGES and NR_SHMEM. The page is already locked in these places, so page->mem_cgroup is stable; we only need minimal tweaks of two mem_cgroup_migrate() calls to ensure it's set up in time.
Then replace MEMCG_CACHE with NR_FILE_PAGES and delete the private NR_SHMEM accounting sites.
---
include/linux/memcontrol.h | 3 +-- mm/filemap.c | 17 +++++++++-------- mm/khugepaged.c | 16 +++++++++++----- mm/memcontrol.c | 28 +++++++++++----------------- mm/migrate.c | 15 +++++++++++---- mm/shmem.c | 14 +++++++------- 6 files changed, 50 insertions(+), 43 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +1 -2
@@ -29,8 +29,7 @@ struct kmem_cache;/* Cgroup-specific page state, on top of universal node page state */ enum memcg_stat_item { -MEMCG_CACHE =NR_VM_NODE_STAT_ITEMS , -MEMCG_RSS , +MEMCG_RSS =NR_VM_NODE_STAT_ITEMS ,MEMCG_RSS_HUGE ,MEMCG_SWAP ,MEMCG_SOCK ,
mm/filemap.c renamed from mm/filemap.c~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +9 -8
@@ -199,9 +199,9 @@ static void unaccount_page_cache_page(st nr =hpage_nr_pages (page); -__mod_node_page_state (page_pgdat (page),NR_FILE_PAGES , -nr); +__mod_lruvec_page_state (page,NR_FILE_PAGES , -nr);if (PageSwapBacked (page)) { -__mod_node_page_state (page_pgdat (page),NR_SHMEM , -nr); +__mod_lruvec_page_state (page,NR_SHMEM , -nr);if (PageTransHuge (page))__dec_node_page_state (page,NR_SHMEM_THPS ); }else if (PageTransHuge (page)) {
@@ -802,21 +802,22 @@ int replace_page_cache_page(struct page new->mapping = mapping; new->index = offset; +mem_cgroup_migrate (old, new); +xas_lock_irqsave (&xas, flags);xas_store (&xas, new); old->mapping =NULL ;/* hugetlb pages do not participate in page cache accounting. */ if (!PageHuge (old)) -__dec_node_page_state (old,NR_FILE_PAGES ); +__dec_lruvec_page_state (old,NR_FILE_PAGES );if (!PageHuge (new)) -__inc_node_page_state (new,NR_FILE_PAGES ); +__inc_lruvec_page_state (new,NR_FILE_PAGES );if (PageSwapBacked (old)) -__dec_node_page_state (old,NR_SHMEM ); +__dec_lruvec_page_state (old,NR_SHMEM );if (PageSwapBacked (new)) -__inc_node_page_state (new,NR_SHMEM ); +__inc_lruvec_page_state (new,NR_SHMEM );xas_unlock_irqrestore (&xas, flags); -mem_cgroup_migrate (old, new);if (freepage)freepage (old);put_page (old);
@@ -867,7 +868,7 @@ static int __add_to_page_cache_locked(st/* hugetlb pages do not participate in page cache accounting */ if (!huge) -__inc_node_page_state (page,NR_FILE_PAGES ); +__inc_lruvec_page_state (page,NR_FILE_PAGES ); unlock:xas_unlock_irq (&xas); }while (xas_nomem (&xas, gfp_mask &GFP_RECLAIM_MASK ));
mm/khugepaged.c renamed from mm/khugepaged.c~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +11 -5
@@ -1844,12 +1844,18 @@ out_unlock: }if (nr_none) { -struct zone *zone =page_zone (new_page); - -__mod_node_page_state (zone->zone_pgdat,NR_FILE_PAGES , nr_none); +struct lruvec *lruvec; +/* +* XXX: We have started try_charge and pinned the +* memcg, but the page isn't committed yet so we +* cannot use mod_lruvec_page_state(). This hackery +* will be cleaned up when remove the page->mapping +* dependency from memcg and fully charge above. +*/ + lruvec =mem_cgroup_lruvec (memcg,page_pgdat (new_page)); +__mod_lruvec_state (lruvec,NR_FILE_PAGES , nr_none);if (is_shmem) -__mod_node_page_state (zone->zone_pgdat, -NR_SHMEM , nr_none); +__mod_lruvec_state (lruvec,NR_SHMEM , nr_none); } xa_locked:
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +11 -17
@@ -842,11 +842,6 @@ static void mem_cgroup_charge_statistics*/ if (PageAnon (page))__mod_memcg_state (memcg,MEMCG_RSS , nr_pages); -else { -__mod_memcg_state (memcg,MEMCG_CACHE , nr_pages); -if (PageSwapBacked (page)) -__mod_memcg_state (memcg,NR_SHMEM , nr_pages); - }if (abs (nr_pages) >1 ) {VM_BUG_ON_PAGE (!PageTransHuge (page), page);
@@ -1392,7 +1387,7 @@ static char *memory_stat_format(struct m (u64 )memcg_page_state (memcg,MEMCG_RSS ) * PAGE_SIZE);seq_buf_printf (&s,"file %llu\n" , - (u64 )memcg_page_state (memcg,MEMCG_CACHE ) * + (u64 )memcg_page_state (memcg,NR_FILE_PAGES ) *PAGE_SIZE ); seq_buf_printf(&s, "kernel_stack %llu\n", (u64)memcg_page_state(memcg, MEMCG_KERNEL_STACK_KB) *
@@ -3357,7 +3352,7 @@ static unsigned long mem_cgroup_usage(stunsigned long val;if (mem_cgroup_is_root (memcg)) { - val =memcg_page_state (memcg,MEMCG_CACHE ) + + val =memcg_page_state (memcg,NR_FILE_PAGES ) +memcg_page_state (memcg,MEMCG_RSS );if (swap) val +=memcg_page_state (memcg,MEMCG_SWAP );
@@ -3828,7 +3823,7 @@ static int memcg_numa_stat_show(struct s#endif /* CONFIG_NUMA */ static const unsigned int memcg1_stats[] = { -MEMCG_CACHE , +NR_FILE_PAGES ,MEMCG_RSS ,MEMCG_RSS_HUGE ,NR_SHMEM ,
@@ -5461,6 +5456,14 @@ static int mem_cgroup_move_account(struclock_page_memcg (page);if (!PageAnon (page)) { +__mod_lruvec_state (from_vec,NR_FILE_PAGES , -nr_pages); +__mod_lruvec_state (to_vec,NR_FILE_PAGES , nr_pages); + +if (PageSwapBacked (page)) { +__mod_lruvec_state (from_vec,NR_SHMEM , -nr_pages); +__mod_lruvec_state (to_vec,NR_SHMEM , nr_pages); + } +if (page_mapped (page)) {__mod_lruvec_state (from_vec,NR_FILE_MAPPED , -nr_pages);__mod_lruvec_state (to_vec,NR_FILE_MAPPED , nr_pages);
@@ -6673,10 +6676,8 @@ struct uncharge_gather {unsigned long nr_pages;unsigned long pgpgout;unsigned long nr_anon; -unsigned long nr_file;unsigned long nr_kmem;unsigned long nr_huge; -unsigned long nr_shmem;struct page *dummy_page; };
@@ -6700,9 +6701,7 @@ static void uncharge_batch(const structlocal_irq_save (flags);__mod_memcg_state (ug->memcg,MEMCG_RSS , -ug->nr_anon); -__mod_memcg_state (ug->memcg,MEMCG_CACHE , -ug->nr_file);__mod_memcg_state (ug->memcg,MEMCG_RSS_HUGE , -ug->nr_huge); -__mod_memcg_state (ug->memcg,NR_SHMEM , -ug->nr_shmem);__count_memcg_events (ug->memcg,PGPGOUT , ug->pgpgout);__this_cpu_add (ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);memcg_check_events (ug->memcg, ug->dummy_page);
@@ -6743,11 +6742,6 @@ static void uncharge_page(struct page *p ug->nr_huge += nr_pages;if (PageAnon (page)) ug->nr_anon += nr_pages; -else { - ug->nr_file += nr_pages; -if (PageSwapBacked (page)) - ug->nr_shmem += nr_pages; - } ug->pgpgout++; }else { ug->nr_kmem += nr_pages;
mm/migrate.c renamed from mm/migrate.c~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +11 -4
@@ -490,11 +490,18 @@ int migrate_page_move_mapping(struct add* are mapped to swap space. */ if (newzone != oldzone) { -__dec_node_state (oldzone->zone_pgdat,NR_FILE_PAGES ); -__inc_node_state (newzone->zone_pgdat,NR_FILE_PAGES ); +struct lruvec *old_lruvec, *new_lruvec; +struct mem_cgroup *memcg; + + memcg =page_memcg (page); + old_lruvec =mem_cgroup_lruvec (memcg, oldzone->zone_pgdat); + new_lruvec =mem_cgroup_lruvec (memcg, newzone->zone_pgdat); + +__dec_lruvec_state (old_lruvec,NR_FILE_PAGES ); +__inc_lruvec_state (new_lruvec,NR_FILE_PAGES );if (PageSwapBacked (page) && !PageSwapCache (page)) { -__dec_node_state (oldzone->zone_pgdat,NR_SHMEM ); -__inc_node_state (newzone->zone_pgdat,NR_SHMEM ); +__dec_lruvec_state (old_lruvec,NR_SHMEM ); +__inc_lruvec_state (new_lruvec,NR_SHMEM ); }if (dirty &&mapping_cap_account_dirty (mapping)) {__dec_node_state (oldzone->zone_pgdat,NR_FILE_DIRTY );
mm/shmem.c renamed from mm/shmem.c~mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters +7 -7
@@ -653,8 +653,8 @@ next:__inc_node_page_state (page,NR_SHMEM_THPS ); } mapping->nrpages += nr; -__mod_node_page_state (page_pgdat (page),NR_FILE_PAGES , nr); -__mod_node_page_state (page_pgdat (page),NR_SHMEM , nr); +__mod_lruvec_page_state (page,NR_FILE_PAGES , nr); +__mod_lruvec_page_state (page,NR_SHMEM , nr); unlock:xas_unlock_irq (&xas); }while (xas_nomem (&xas, gfp));
@@ -685,8 +685,8 @@ static void shmem_delete_from_page_cache error =shmem_replace_entry (mapping, page->index, page, radswap); page->mapping =NULL ; mapping->nrpages--; -__dec_node_page_state (page,NR_FILE_PAGES ); -__dec_node_page_state (page,NR_SHMEM ); +__dec_lruvec_page_state (page,NR_FILE_PAGES ); +__dec_lruvec_page_state (page,NR_SHMEM );xa_unlock_irq (&mapping->i_pages);put_page (page);BUG_ON (error);
@@ -1593,8 +1593,9 @@ static int shmem_replace_page(struct pagxa_lock_irq (&swap_mapping->i_pages); error =shmem_replace_entry (swap_mapping, swap_index, oldpage, newpage);if (!error) { -__inc_node_page_state (newpage,NR_FILE_PAGES ); -__dec_node_page_state (oldpage,NR_FILE_PAGES ); +mem_cgroup_migrate (oldpage, newpage); +__inc_lruvec_page_state (newpage,NR_FILE_PAGES ); +__dec_lruvec_page_state (oldpage,NR_FILE_PAGES ); }xa_unlock_irq (&swap_mapping->i_pages);
@@ -1606,7 +1607,6 @@ static int shmem_replace_page(struct pag*/ oldpage = newpage; }else { -mem_cgroup_migrate (oldpage, newpage);lru_cache_add_anon (newpage); *pagep = newpage; }
_
Andrew Morton
maintainerPATCH 93/131 mm: memcontrol: switch to native NR_ANON_MAPPED counter
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: switch to native NR_ANON_MAPPED counter
Memcg maintains a private MEMCG_RSS counter. This divergence from the generic VM accounting means unnecessary code overhead, and creates a dependency for memcg that page->mapping is set up at the time of charging, so that page types can be told apart.
Convert the generic accounting sites to mod_lruvec_page_state and friends to maintain the per-cgroup vmstat counter of NR_ANON_MAPPED. We use lock_page_memcg() to stabilize page->mem_cgroup during rmap changes, the same way we do for NR_FILE_MAPPED.
With the previous patch removing MEMCG_CACHE and the private NR_SHMEM counter, this patch finally eliminates the need to have page->mapping set up at charge time. However, we need to have page->mem_cgroup set up by the time rmap runs and does the accounting, so switch the commit and the rmap callbacks around.
v2: fix temporary accounting bug by switching rmap<->commit (Joonsoo)
---
include/linux/memcontrol.h | 3 -- kernel/events/uprobes.c | 2 - mm/huge_memory.c | 2 - mm/khugepaged.c | 2 - mm/memcontrol.c | 27 ++++++-------------- mm/memory.c | 10 +++---- mm/migrate.c | 2 - mm/rmap.c | 47 +++++++++++++++++++++-------------- mm/swapfile.c | 4 +- mm/userfaultfd.c | 2 - 10 files changed, 51 insertions(+), 50 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -2
@@ -29,8 +29,7 @@ struct kmem_cache;/* Cgroup-specific page state, on top of universal node page state */ enum memcg_stat_item { -MEMCG_RSS =NR_VM_NODE_STAT_ITEMS , -MEMCG_RSS_HUGE , +MEMCG_RSS_HUGE =NR_VM_NODE_STAT_ITEMS ,MEMCG_SWAP ,MEMCG_SOCK ,/* XXX: why are these zone and not node counters? */
kernel/events/uprobes.c renamed from kernel/events/uprobes.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -1
@@ -188,8 +188,8 @@ static int __replace_page(struct vm_areaif (new_page) {get_page (new_page); -page_add_new_anon_rmap (new_page, vma, addr, false);mem_cgroup_commit_charge (new_page, memcg, false); +page_add_new_anon_rmap (new_page, vma, addr, false);lru_cache_add_active_or_unevictable (new_page, vma); }else /* no new page, just dec_mm_counter for old_page */
mm/huge_memory.c renamed from mm/huge_memory.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -1
@@ -640,8 +640,8 @@ static vm_fault_t __do_huge_pmd_anonymou entry =mk_huge_pmd (page, vma->vm_page_prot); entry =maybe_pmd_mkwrite (pmd_mkdirty (entry), vma); -page_add_new_anon_rmap (page, vma, haddr, true);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, haddr, true);lru_cache_add_active_or_unevictable (page, vma);pgtable_trans_huge_deposit (vma->vm_mm, vmf->pmd, pgtable);set_pmd_at (vma->vm_mm, haddr, vmf->pmd, entry);
mm/khugepaged.c renamed from mm/khugepaged.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -1
@@ -1175,8 +1175,8 @@ static void collapse_huge_page(struct mmspin_lock (pmd_ptl);BUG_ON (!pmd_none (*pmd)); -page_add_new_anon_rmap (new_page, vma, address, true);mem_cgroup_commit_charge (new_page, memcg, false); +page_add_new_anon_rmap (new_page, vma, address, true);count_memcg_events (memcg,THP_COLLAPSE_ALLOC ,1 );lru_cache_add_active_or_unevictable (new_page, vma);pgtable_trans_huge_deposit (mm, pmd, pgtable);
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +9 -18
@@ -836,13 +836,6 @@ static void mem_cgroup_charge_statistics struct page *page, int nr_pages) { -/* -* Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is -* counted as CACHE even if it's on ANON LRU. -*/ -if (PageAnon (page)) -__mod_memcg_state (memcg,MEMCG_RSS , nr_pages); - if (abs(nr_pages) > 1) {VM_BUG_ON_PAGE (!PageTransHuge (page), page);__mod_memcg_state (memcg,MEMCG_RSS_HUGE , nr_pages);
@@ -1384,7 +1377,7 @@ static char *memory_stat_format(struct m*/ seq_buf_printf (&s,"anon %llu\n" , - (u64 )memcg_page_state (memcg,MEMCG_RSS ) * + (u64 )memcg_page_state (memcg,NR_ANON_MAPPED ) *PAGE_SIZE ); seq_buf_printf(&s, "file %llu\n", (u64)memcg_page_state(memcg, NR_FILE_PAGES) *
@@ -3353,7 +3346,7 @@ static unsigned long mem_cgroup_usage(stif (mem_cgroup_is_root (memcg)) { val =memcg_page_state (memcg,NR_FILE_PAGES ) + -memcg_page_state (memcg,MEMCG_RSS ); +memcg_page_state (memcg,NR_ANON_MAPPED );if (swap) val +=memcg_page_state (memcg,MEMCG_SWAP ); }else {
@@ -3824,7 +3817,7 @@ static int memcg_numa_stat_show(struct sstatic const unsigned int memcg1_stats[] = {NR_FILE_PAGES , -MEMCG_RSS , +NR_ANON_MAPPED ,MEMCG_RSS_HUGE ,NR_SHMEM ,NR_FILE_MAPPED ,
@@ -5455,7 +5448,12 @@ static int mem_cgroup_move_account(struclock_page_memcg (page); -if (!PageAnon (page)) { +if (PageAnon (page)) { +if (page_mapped (page)) { +__mod_lruvec_state (from_vec,NR_ANON_MAPPED , -nr_pages); +__mod_lruvec_state (to_vec,NR_ANON_MAPPED , nr_pages); + } + }else {__mod_lruvec_state (from_vec,NR_FILE_PAGES , -nr_pages);__mod_lruvec_state (to_vec,NR_FILE_PAGES , nr_pages);
@@ -6589,7 +6587,6 @@ void mem_cgroup_commit_charge(struct pag {unsigned int nr_pages =hpage_nr_pages (page); -VM_BUG_ON_PAGE (!page->mapping, page);VM_BUG_ON_PAGE (PageLRU (page) && !lrucare, page);if (mem_cgroup_disabled ())
@@ -6662,8 +6659,6 @@ int mem_cgroup_charge(struct page *page,struct mem_cgroup *memcg;int ret; -VM_BUG_ON_PAGE (!page->mapping, page); - ret =mem_cgroup_try_charge (page, mm, gfp_mask, &memcg);if (ret)return ret;
@@ -6675,7 +6670,6 @@ struct uncharge_gather {struct mem_cgroup *memcg;unsigned long nr_pages;unsigned long pgpgout; -unsigned long nr_anon;unsigned long nr_kmem;unsigned long nr_huge;struct page *dummy_page;
@@ -6700,7 +6694,6 @@ static void uncharge_batch(const struct }local_irq_save (flags); -__mod_memcg_state (ug->memcg,MEMCG_RSS , -ug->nr_anon);__mod_memcg_state (ug->memcg,MEMCG_RSS_HUGE , -ug->nr_huge);__count_memcg_events (ug->memcg,PGPGOUT , ug->pgpgout);__this_cpu_add (ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
@@ -6740,8 +6733,6 @@ static void uncharge_page(struct page *pif (!PageKmemcg (page)) {if (PageTransHuge (page)) ug->nr_huge += nr_pages; -if (PageAnon (page)) - ug->nr_anon += nr_pages; ug->pgpgout++; }else { ug->nr_kmem += nr_pages;
mm/memory.c renamed from mm/memory.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +5 -5
@@ -2710,8 +2710,8 @@ static vm_fault_t wp_page_copy(struct vm* thread doing COW. */ ptep_clear_flush_notify (vma, vmf->address, vmf->pte); -page_add_new_anon_rmap (new_page, vma, vmf->address, false);mem_cgroup_commit_charge (new_page, memcg, false); +page_add_new_anon_rmap (new_page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (new_page, vma);/* * We call the notify macro here because, when using secondary
@@ -3243,12 +3243,12 @@ vm_fault_t do_swap_page(struct vm_fault/* ksm created a completely new copy */ if (unlikely (page != swapcache && swapcache)) { -page_add_new_anon_rmap (page, vma, vmf->address, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); }else { -do_page_add_anon_rmap (page, vma, vmf->address, exclusive);mem_cgroup_commit_charge (page, memcg, true); +do_page_add_anon_rmap (page, vma, vmf->address, exclusive);activate_page (page); }
@@ -3390,8 +3390,8 @@ static vm_fault_t do_anonymous_page(stru }inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES ); -page_add_new_anon_rmap (page, vma, vmf->address, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); setpte:set_pte_at (vma->vm_mm, vmf->address, vmf->pte, entry);
@@ -3652,8 +3652,8 @@ vm_fault_t alloc_set_pte(struct vm_fault/* copy-on-write page */ if (write && !(vma->vm_flags &VM_SHARED )) {inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES ); -page_add_new_anon_rmap (page, vma, vmf->address, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); }else {inc_mm_counter_fast (vma->vm_mm,mm_counter_file (page));
mm/migrate.c renamed from mm/migrate.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -1
@@ -2832,8 +2832,8 @@ static void migrate_vma_insert_page(stru goto unlock_abort;inc_mm_counter (mm,MM_ANONPAGES ); -page_add_new_anon_rmap (page, vma, addr, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, addr, false);if (!is_zone_device_page (page))lru_cache_add_active_or_unevictable (page, vma);get_page (page);
mm/rmap.c renamed from mm/rmap.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +29 -18
124 lines not on this page · raw message
mm/swapfile.c renamed from mm/swapfile.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +2 -2
@@ -1920,11 +1920,11 @@ static int unuse_pte(struct vm_area_struset_pte_at (vma->vm_mm, addr, pte,pte_mkold (mk_pte (page, vma->vm_page_prot)));if (page == swapcache) { -page_add_anon_rmap (page, vma, addr, false);mem_cgroup_commit_charge (page, memcg, true); +page_add_anon_rmap (page, vma, addr, false); }else {/* ksm created a completely new copy */ -page_add_new_anon_rmap (page, vma, addr, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, vma, addr, false);lru_cache_add_active_or_unevictable (page, vma); }swap_free (entry);
mm/userfaultfd.c renamed from mm/userfaultfd.c~mm-memcontrol-switch-to-native-nr_anon_mapped-counter +1 -1
@@ -123,8 +123,8 @@ static int mcopy_atomic_pte(struct mm_st goto out_release_uncharge_unlock;inc_mm_counter (dst_mm,MM_ANONPAGES ); -page_add_new_anon_rmap (page, dst_vma, dst_addr, false);mem_cgroup_commit_charge (page, memcg, false); +page_add_new_anon_rmap (page, dst_vma, dst_addr, false);lru_cache_add_active_or_unevictable (page, dst_vma);set_pte_at (dst_mm, dst_addr, dst_pte, _dst_pte);
_
Andrew Morton
maintainerPATCH 94/131 mm: memcontrol: switch to native NR_ANON_THPS counter
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: switch to native NR_ANON_THPS counter
With rmap memcg locking already in place for NR_ANON_MAPPED, it's just a small step to remove the MEMCG_RSS_HUGE wart and switch memcg to the native NR_ANON_THPS accounting sites.
---
include/linux/memcontrol.h | 3 -- mm/huge_memory.c | 4 ++ mm/memcontrol.c | 47 +++++++++++++++++------------------ mm/rmap.c | 6 ++-- 4 files changed, 31 insertions(+), 29 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-switch-to-native-nr_anon_thps-counter +1 -2
@@ -29,8 +29,7 @@ struct kmem_cache;/* Cgroup-specific page state, on top of universal node page state */ enum memcg_stat_item { -MEMCG_RSS_HUGE =NR_VM_NODE_STAT_ITEMS , -MEMCG_SWAP , +MEMCG_SWAP =NR_VM_NODE_STAT_ITEMS ,MEMCG_SOCK ,/* XXX: why are these zone and not node counters? */ MEMCG_KERNEL_STACK_KB ,
mm/huge_memory.c renamed from mm/huge_memory.c~mm-memcontrol-switch-to-native-nr_anon_thps-counter +3 -1
@@ -2159,15 +2159,17 @@ static void __split_huge_pmd_locked(struatomic_inc (&page[i]._mapcount); } +lock_page_memcg (page);if (atomic_add_negative (-1 ,compound_mapcount_ptr (page))) {/* Last compound_mapcount is gone. */ -__dec_node_page_state (page,NR_ANON_THPS ); +__dec_lruvec_page_state (page,NR_ANON_THPS );if (TestClearPageDoubleMap (page)) {/* No need in mapcount reference anymore */ for (i =0 ; i <HPAGE_PMD_NR ; i++)atomic_dec (&page[i]._mapcount); } } +unlock_page_memcg (page);smp_wmb ();/* make pte visible before pmd */ pmd_populate (mm, pmd, pgtable);
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-switch-to-native-nr_anon_thps-counter +24 -23
111 lines not on this page · raw message
mm/rmap.c renamed from mm/rmap.c~mm-memcontrol-switch-to-native-nr_anon_thps-counter +3 -3
@@ -1138,7 +1138,7 @@ void do_page_add_anon_rmap(struct page ** disabled. */ if (compound) -__inc_node_page_state (page,NR_ANON_THPS ); +__inc_lruvec_page_state (page,NR_ANON_THPS );__mod_lruvec_page_state (page,NR_ANON_MAPPED , nr); }
@@ -1180,7 +1180,7 @@ void page_add_new_anon_rmap(struct pageif (hpage_pincount_available (page))atomic_set (compound_pincount_ptr (page),0 ); -__inc_node_page_state (page,NR_ANON_THPS ); +__inc_lruvec_page_state (page,NR_ANON_THPS ); }else {/* Anon THP always mapped first with PMD */ VM_BUG_ON_PAGE (PageTransCompound (page), page);
@@ -1286,7 +1286,7 @@ static void page_remove_anon_compound_rmif (!IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE ))return ; -__dec_node_page_state (page,NR_ANON_THPS ); +__dec_lruvec_page_state (page,NR_ANON_THPS );if (TestClearPageDoubleMap (page)) {/*
_
Andrew Morton
maintainerPATCH 95/131 mm: memcontrol: convert anon and file-thp to new mem_cgroup_charge() API
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: convert anon and file-thp to new mem_cgroup_charge() API
With the page->mapping requirement gone from memcg, we can charge anon and file-thp pages in one single step, right after they're allocated.
This removes two out of three API calls - especially the tricky commit step that needed to happen at just the right time between when the page is "set up" and when it's "published" - somewhat vague and fluid concepts that varied by page type. All we need is a freshly allocated page and a memcg context to charge.
v2: prevent double charges on pre-allocated hugepages in khugepaged
---
include/linux/mm.h | 4 +--- kernel/events/uprobes.c | 11 +++-------- mm/filemap.c | 2 +- mm/huge_memory.c | 9 +++------ mm/khugepaged.c | 35 ++++++++++------------------------- mm/memory.c | 36 ++++++++++-------------------------- mm/migrate.c | 5 +---- mm/swapfile.c | 6 +----- mm/userfaultfd.c | 5 +---- 9 files changed, 31 insertions(+), 82 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +1 -3
@@ -501,7 +501,6 @@ struct vm_fault {pte_t orig_pte;/* Value of PTE at the time of fault */ struct page *cow_page;/* Page handler may use for COW fault */ -struct mem_cgroup *memcg;/* Cgroup cow_page belongs to */ struct page *page;/* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by
@@ -946,8 +945,7 @@ static inline pte_t maybe_mkwrite(pte_t return pte; } -vm_fault_t alloc_set_pte (struct vm_fault *vmf,struct mem_cgroup *memcg, -struct page *page); +vm_fault_t alloc_set_pte (struct vm_fault *vmf,struct page *page);vm_fault_t finish_fault (struct vm_fault *vmf);vm_fault_t finish_mkwrite_fault (struct vm_fault *vmf);#endif
kernel/events/uprobes.c renamed from kernel/events/uprobes.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +3 -8
@@ -162,14 +162,13 @@ static int __replace_page(struct vm_area };int err;struct mmu_notifier_range range; -struct mem_cgroup *memcg;mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, mm, addr, addr +PAGE_SIZE );if (new_page) { - err =mem_cgroup_try_charge (new_page, vma->vm_mm,GFP_KERNEL , - &memcg); + err =mem_cgroup_charge (new_page, vma->vm_mm,GFP_KERNEL , + false);if (err)return err; }
@@ -179,16 +178,12 @@ static int __replace_page(struct vm_areammu_notifier_invalidate_range_start (&range); err = -EAGAIN ; -if (!page_vma_mapped_walk (&pvmw)) { -if (new_page) -mem_cgroup_cancel_charge (new_page, memcg); +if (!page_vma_mapped_walk (&pvmw)) goto unlock; - }VM_BUG_ON_PAGE (addr != pvmw.address, old_page);if (new_page) {get_page (new_page); -mem_cgroup_commit_charge (new_page, memcg, false);page_add_new_anon_rmap (new_page, vma, addr, false);lru_cache_add_active_or_unevictable (new_page, vma); }else
mm/filemap.c renamed from mm/filemap.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +1 -1
@@ -2633,7 +2633,7 @@ void filemap_map_pages(struct vm_fault *if (vmf->pte) vmf->pte += xas.xa_index - last_pgoff; last_pgoff = xas.xa_index; -if (alloc_set_pte (vmf,NULL , page)) +if (alloc_set_pte (vmf, page)) goto unlock;unlock_page (page); goto next;
mm/huge_memory.c renamed from mm/huge_memory.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +3 -6
@@ -587,19 +587,19 @@ static vm_fault_t __do_huge_pmd_anonymou struct page *page, gfp_t gfp) {struct vm_area_struct *vma = vmf->vma; -struct mem_cgroup *memcg;pgtable_t pgtable;unsigned long haddr = vmf->address &HPAGE_PMD_MASK ;vm_fault_t ret =0 ;VM_BUG_ON_PAGE (!PageCompound (page), page); -if (mem_cgroup_try_charge_delay (page, vma->vm_mm, gfp, &memcg)) { +if (mem_cgroup_charge (page, vma->vm_mm, gfp, false)) {put_page (page);count_vm_event (THP_FAULT_FALLBACK );count_vm_event (THP_FAULT_FALLBACK_CHARGE );return VM_FAULT_FALLBACK ; } +cgroup_throttle_swaprate (page, gfp); pgtable =pte_alloc_one (vma->vm_mm);if (unlikely (!pgtable)) {
@@ -630,7 +630,6 @@ static vm_fault_t __do_huge_pmd_anonymouvm_fault_t ret2;spin_unlock (vmf->ptl); -mem_cgroup_cancel_charge (page, memcg);put_page (page);pte_free (vma->vm_mm, pgtable); ret2 =handle_userfault (vmf,VM_UFFD_MISSING );
@@ -640,7 +639,6 @@ static vm_fault_t __do_huge_pmd_anonymou entry =mk_huge_pmd (page, vma->vm_page_prot); entry =maybe_pmd_mkwrite (pmd_mkdirty (entry), vma); -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, haddr, true);lru_cache_add_active_or_unevictable (page, vma);pgtable_trans_huge_deposit (vma->vm_mm, vmf->pmd, pgtable);
@@ -649,7 +647,7 @@ static vm_fault_t __do_huge_pmd_anonymoumm_inc_nr_ptes (vma->vm_mm);spin_unlock (vmf->ptl);count_vm_event (THP_FAULT_ALLOC ); -count_memcg_events (memcg,THP_FAULT_ALLOC ,1 ); +count_memcg_event_mm (vma->vm_mm,THP_FAULT_ALLOC ); }return 0 ;
@@ -658,7 +656,6 @@ unlock_release: release:if (pgtable)pte_free (vma->vm_mm, pgtable); -mem_cgroup_cancel_charge (page, memcg);put_page (page);return ret;
mm/khugepaged.c renamed from mm/khugepaged.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +10 -25
@@ -1037,7 +1037,6 @@ static void collapse_huge_page(struct mmstruct page *new_page;spinlock_t *pmd_ptl, *pte_ptl;int isolated =0 , result =0 ; -struct mem_cgroup *memcg;struct vm_area_struct *vma;struct mmu_notifier_range range;gfp_t gfp;
@@ -1060,15 +1059,15 @@ static void collapse_huge_page(struct mm goto out_nolock; } -if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg))) { +if (unlikely (mem_cgroup_charge (new_page, mm, gfp, false))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out_nolock; } +count_memcg_page_event (new_page,THP_COLLAPSE_ALLOC );down_read (&mm->mmap_sem); result =hugepage_vma_revalidate (mm, address, &vma);if (result) { -mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1076,7 +1075,6 @@ static void collapse_huge_page(struct mm pmd =mm_find_pmd (mm, address);if (!pmd) { result =SCAN_PMD_NULL ; -mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1088,7 +1086,6 @@ static void collapse_huge_page(struct mm*/ if (unmapped && !__collapse_huge_page_swapin (mm, vma, address, pmd, referenced)) { -mem_cgroup_cancel_charge (new_page, memcg);up_read (&mm->mmap_sem); goto out_nolock; }
@@ -1175,9 +1172,7 @@ static void collapse_huge_page(struct mmspin_lock (pmd_ptl);BUG_ON (!pmd_none (*pmd)); -mem_cgroup_commit_charge (new_page, memcg, false);page_add_new_anon_rmap (new_page, vma, address, true); -count_memcg_events (memcg,THP_COLLAPSE_ALLOC ,1 );lru_cache_add_active_or_unevictable (new_page, vma);pgtable_trans_huge_deposit (mm, pmd, pgtable);set_pmd_at (mm, address, pmd, _pmd);
@@ -1191,10 +1186,11 @@ static void collapse_huge_page(struct mm out_up_write:up_write (&mm->mmap_sem); out_nolock: +if (!IS_ERR_OR_NULL (*hpage)) +mem_cgroup_uncharge (*hpage);trace_mm_collapse_huge_page (mm, isolated, result);return ; out: -mem_cgroup_cancel_charge (new_page, memcg); goto out_up_write; }
@@ -1618,7 +1614,6 @@ static void collapse_file(struct mm_strustruct address_space *mapping = file->f_mapping;gfp_t gfp;struct page *new_page; -struct mem_cgroup *memcg;pgoff_t index, end = start +HPAGE_PMD_NR ;LIST_HEAD (pagelist);XA_STATE_ORDER (xas, &mapping->i_pages, start,HPAGE_PMD_ORDER );
@@ -1637,10 +1632,11 @@ static void collapse_file(struct mm_stru goto out; } -if (unlikely (mem_cgroup_try_charge (new_page, mm, gfp, &memcg))) { +if (unlikely (mem_cgroup_charge (new_page, mm, gfp, false))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out; } +count_memcg_page_event (new_page,THP_COLLAPSE_ALLOC );/* This will be less messy when we use multi-index entries */ do {
@@ -1650,7 +1646,6 @@ static void collapse_file(struct mm_strubreak ;xas_unlock_irq (&xas);if (!xas_nomem (&xas,GFP_KERNEL )) { -mem_cgroup_cancel_charge (new_page, memcg); result =SCAN_FAIL ; goto out; }
@@ -1844,18 +1839,9 @@ out_unlock: }if (nr_none) { -struct lruvec *lruvec; -/* -* XXX: We have started try_charge and pinned the -* memcg, but the page isn't committed yet so we -* cannot use mod_lruvec_page_state(). This hackery -* will be cleaned up when remove the page->mapping -* dependency from memcg and fully charge above. -*/ - lruvec =mem_cgroup_lruvec (memcg,page_pgdat (new_page)); -__mod_lruvec_state (lruvec,NR_FILE_PAGES , nr_none); +__mod_lruvec_page_state (new_page,NR_FILE_PAGES , nr_none);if (is_shmem) -__mod_lruvec_state (lruvec,NR_SHMEM , nr_none); +__mod_lruvec_page_state (new_page,NR_SHMEM , nr_none); } xa_locked:
@@ -1893,7 +1879,6 @@ xa_unlocked:SetPageUptodate (new_page);page_ref_add (new_page,HPAGE_PMD_NR -1 ); -mem_cgroup_commit_charge (new_page, memcg, false);if (is_shmem) {set_page_dirty (new_page);
@@ -1901,7 +1886,6 @@ xa_unlocked: }else {lru_cache_add_file (new_page); } -count_memcg_events (memcg,THP_COLLAPSE_ALLOC ,1 );/* * Remove pte page tables, so we can re-fault the page as huge.
@@ -1948,13 +1932,14 @@ xa_unlocked:VM_BUG_ON (nr_none);xas_unlock_irq (&xas); -mem_cgroup_cancel_charge (new_page, memcg); new_page->mapping =NULL ; }unlock_page (new_page); out:VM_BUG_ON (!list_empty (&pagelist)); +if (!IS_ERR_OR_NULL (*hpage)) +mem_cgroup_uncharge (*hpage);/* TODO: tracepoints */ }
mm/memory.c renamed from mm/memory.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +10 -26
@@ -2645,7 +2645,6 @@ static vm_fault_t wp_page_copy(struct vmstruct page *new_page =NULL ;pte_t entry;int page_copied =0 ; -struct mem_cgroup *memcg;struct mmu_notifier_range range;if (unlikely (anon_vma_prepare (vma)))
@@ -2676,8 +2675,9 @@ static vm_fault_t wp_page_copy(struct vm } } -if (mem_cgroup_try_charge_delay (new_page, mm,GFP_KERNEL , &memcg)) +if (mem_cgroup_charge (new_page, mm,GFP_KERNEL , false)) goto oom_free_new; +cgroup_throttle_swaprate (new_page,GFP_KERNEL );__SetPageUptodate (new_page);
@@ -2710,7 +2710,6 @@ static vm_fault_t wp_page_copy(struct vm* thread doing COW. */ ptep_clear_flush_notify (vma, vmf->address, vmf->pte); -mem_cgroup_commit_charge (new_page, memcg, false);page_add_new_anon_rmap (new_page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (new_page, vma);/*
@@ -2749,8 +2748,6 @@ static vm_fault_t wp_page_copy(struct vm/* Free the old page.. */ new_page = old_page; page_copied =1 ; - }else { -mem_cgroup_cancel_charge (new_page, memcg); }if (new_page)
@@ -3088,7 +3085,6 @@ vm_fault_t do_swap_page(struct vm_fault {struct vm_area_struct *vma = vmf->vma;struct page *page =NULL , *swapcache; -struct mem_cgroup *memcg;swp_entry_t entry;pte_t pte;int locked;
@@ -3193,10 +3189,11 @@ vm_fault_t do_swap_page(struct vm_fault goto out_page; } -if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , &memcg)) { +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , true)) { ret =VM_FAULT_OOM ; goto out_page; } +cgroup_throttle_swaprate (page,GFP_KERNEL );/* * Back out if somebody else already faulted in this pte.
@@ -3243,11 +3240,9 @@ vm_fault_t do_swap_page(struct vm_fault/* ksm created a completely new copy */ if (unlikely (page != swapcache && swapcache)) { -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); }else { -mem_cgroup_commit_charge (page, memcg, true);do_page_add_anon_rmap (page, vma, vmf->address, exclusive);activate_page (page); }
@@ -3284,7 +3279,6 @@ unlock: out:return ret; out_nomap: -mem_cgroup_cancel_charge (page, memcg);pte_unmap_unlock (vmf->pte, vmf->ptl); out_page:unlock_page (page);
@@ -3305,7 +3299,6 @@ out_release:static vm_fault_t do_anonymous_page (struct vm_fault *vmf) {struct vm_area_struct *vma = vmf->vma; -struct mem_cgroup *memcg;struct page *page;vm_fault_t ret =0 ;pte_t entry;
@@ -3358,8 +3351,9 @@ static vm_fault_t do_anonymous_page(struif (!page) goto oom; -if (mem_cgroup_try_charge_delay (page, vma->vm_mm,GFP_KERNEL , &memcg)) +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , false)) goto oom_free_page; +cgroup_throttle_swaprate (page,GFP_KERNEL );/* * The memory barrier inside __SetPageUptodate makes sure that
@@ -3384,13 +3378,11 @@ static vm_fault_t do_anonymous_page(stru/* Deliver the page fault to userland, check inside PT lock */ if (userfaultfd_missing (vma)) {pte_unmap_unlock (vmf->pte, vmf->ptl); -mem_cgroup_cancel_charge (page, memcg);put_page (page);return handle_userfault (vmf,VM_UFFD_MISSING ); }inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES ); -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); setpte:
@@ -3402,7 +3394,6 @@ unlock:pte_unmap_unlock (vmf->pte, vmf->ptl);return ret; release: -mem_cgroup_cancel_charge (page, memcg);put_page (page); goto unlock; oom_free_page:
@@ -3607,7 +3598,6 @@ static vm_fault_t do_set_pmd(struct vm_f* mapping. If needed, the fucntion allocates page table or use pre-allocated. * * @vmf: fault environment -* @memcg: memcg to charge page (only for private mappings) * @page: page to map * * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
@@ -3618,8 +3608,7 @@ static vm_fault_t do_set_pmd(struct vm_f* * Return: %0 on success, %VM_FAULT_ code in case of error. */ -vm_fault_t alloc_set_pte (struct vm_fault *vmf,struct mem_cgroup *memcg, -struct page *page) +vm_fault_t alloc_set_pte (struct vm_fault *vmf,struct page *page) {struct vm_area_struct *vma = vmf->vma;bool write = vmf->flags &FAULT_FLAG_WRITE ;
@@ -3627,9 +3616,6 @@ vm_fault_t alloc_set_pte(struct vm_faultvm_fault_t ret;if (pmd_none (*vmf->pmd) &&PageTransCompound (page)) { -/* THP on COW? */ -VM_BUG_ON_PAGE (memcg, page); - ret =do_set_pmd (vmf, page);if (ret !=VM_FAULT_FALLBACK )return ret;
@@ -3652,7 +3638,6 @@ vm_fault_t alloc_set_pte(struct vm_fault/* copy-on-write page */ if (write && !(vma->vm_flags &VM_SHARED )) {inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES ); -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, vmf->address, false);lru_cache_add_active_or_unevictable (page, vma); }else {
@@ -3702,7 +3687,7 @@ vm_fault_t finish_fault(struct vm_faultif (!(vmf->vma->vm_flags &VM_SHARED )) ret =check_stable_address_space (vmf->vma->vm_mm);if (!ret) - ret =alloc_set_pte (vmf, vmf->memcg, page); + ret =alloc_set_pte (vmf, page);if (vmf->pte)pte_unmap_unlock (vmf->pte, vmf->ptl);return ret;
@@ -3862,11 +3847,11 @@ static vm_fault_t do_cow_fault(struct vmif (!vmf->cow_page)return VM_FAULT_OOM ; -if (mem_cgroup_try_charge_delay (vmf->cow_page, vma->vm_mm, -GFP_KERNEL , &vmf->memcg)) { +if (mem_cgroup_charge (vmf->cow_page, vma->vm_mm,GFP_KERNEL , false)) {put_page (vmf->cow_page);return VM_FAULT_OOM ; } +cgroup_throttle_swaprate (vmf->cow_page,GFP_KERNEL ); ret =__do_fault (vmf);if (unlikely (ret & (VM_FAULT_ERROR |VM_FAULT_NOPAGE |VM_FAULT_RETRY )))
@@ -3884,7 +3869,6 @@ static vm_fault_t do_cow_fault(struct vm goto uncharge_out;return ret; uncharge_out: -mem_cgroup_cancel_charge (vmf->cow_page, vmf->memcg);put_page (vmf->cow_page);return ret; }
mm/migrate.c renamed from mm/migrate.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +1 -4
@@ -2740,7 +2740,6 @@ static void migrate_vma_insert_page(stru {struct vm_area_struct *vma = migrate->vma;struct mm_struct *mm = vma->vm_mm; -struct mem_cgroup *memcg;bool flush = false;spinlock_t *ptl;pte_t entry;
@@ -2787,7 +2786,7 @@ static void migrate_vma_insert_page(struif (unlikely (anon_vma_prepare (vma))) goto abort; -if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , &memcg)) +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , false)) goto abort;/*
@@ -2832,7 +2831,6 @@ static void migrate_vma_insert_page(stru goto unlock_abort;inc_mm_counter (mm,MM_ANONPAGES ); -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, addr, false);if (!is_zone_device_page (page))lru_cache_add_active_or_unevictable (page, vma);
@@ -2855,7 +2853,6 @@ static void migrate_vma_insert_page(stru unlock_abort:pte_unmap_unlock (ptep, ptl); -mem_cgroup_cancel_charge (page, memcg); abort: *src &= ~MIGRATE_PFN_MIGRATE ; }
mm/swapfile.c renamed from mm/swapfile.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +1 -5
@@ -1892,7 +1892,6 @@ static int unuse_pte(struct vm_area_stru unsigned long addr, swp_entry_t entry, struct page *page) { struct page *swapcache; -struct mem_cgroup *memcg;spinlock_t *ptl;pte_t *pte;int ret =1 ;
@@ -1902,14 +1901,13 @@ static int unuse_pte(struct vm_area_struif (unlikely (!page))return -ENOMEM ; -if (mem_cgroup_try_charge (page, vma->vm_mm,GFP_KERNEL , &memcg)) { +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , true)) { ret = -ENOMEM ; goto out_nolock; } pte =pte_offset_map_lock (vma->vm_mm, pmd, addr, &ptl);if (unlikely (!pte_same_as_swp (*pte,swp_entry_to_pte (entry)))) { -mem_cgroup_cancel_charge (page, memcg); ret =0 ; goto out; }
@@ -1920,10 +1918,8 @@ static int unuse_pte(struct vm_area_struset_pte_at (vma->vm_mm, addr, pte,pte_mkold (mk_pte (page, vma->vm_page_prot)));if (page == swapcache) { -mem_cgroup_commit_charge (page, memcg, true);page_add_anon_rmap (page, vma, addr, false); }else {/* ksm created a completely new copy */ -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, vma, addr, false);lru_cache_add_active_or_unevictable (page, vma); }
mm/userfaultfd.c renamed from mm/userfaultfd.c~mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api +1 -4
@@ -56,7 +56,6 @@ static int mcopy_atomic_pte(struct mm_st struct page **pagep, bool wp_copy) { -struct mem_cgroup *memcg;pte_t _dst_pte, *dst_pte;spinlock_t *ptl;void *page_kaddr;
@@ -97,7 +96,7 @@ static int mcopy_atomic_pte(struct mm_st__SetPageUptodate (page); ret = -ENOMEM ; -if (mem_cgroup_try_charge (page, dst_mm,GFP_KERNEL , &memcg)) +if (mem_cgroup_charge (page, dst_mm,GFP_KERNEL , false)) goto out_release; _dst_pte =pte_mkdirty (mk_pte (page, dst_vma->vm_page_prot));
@@ -123,7 +122,6 @@ static int mcopy_atomic_pte(struct mm_st goto out_release_uncharge_unlock;inc_mm_counter (dst_mm,MM_ANONPAGES ); -mem_cgroup_commit_charge (page, memcg, false);page_add_new_anon_rmap (page, dst_vma, dst_addr, false);lru_cache_add_active_or_unevictable (page, dst_vma);
@@ -138,7 +136,6 @@ out:return ret; out_release_uncharge_unlock:pte_unmap_unlock (dst_pte, ptl); -mem_cgroup_cancel_charge (page, memcg); out_release:put_page (page); goto out;
_
Andrew Morton
maintainerPATCH 96/131 mm: memcontrol: drop unused try/commit/cancel charge API
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: drop unused try/commit/cancel charge API
There are no more users. RIP in peace.
---
include/linux/memcontrol.h | 36 --------- mm/memcontrol.c | 128 ++++------------------------------- 2 files changed, 17 insertions(+), 147 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-drop-unused-try-commit-cancel-charge-api +0 -36
@@ -355,14 +355,6 @@ static inline unsigned long mem_cgroup_penum mem_cgroup_protection mem_cgroup_protected (struct mem_cgroup *root,struct mem_cgroup *memcg); -int mem_cgroup_try_charge (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp); -int mem_cgroup_try_charge_delay (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,struct mem_cgroup **memcgp); -void mem_cgroup_commit_charge (struct page *page,struct mem_cgroup *memcg, -bool lrucare); -void mem_cgroup_cancel_charge (struct page *page,struct mem_cgroup *memcg); -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask,bool lrucare);
@@ -846,34 +838,6 @@ static inline enum mem_cgroup_protectionreturn MEMCG_PROT_NONE ; } -static inline int mem_cgroup_try_charge (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask, -struct mem_cgroup **memcgp) -{ - *memcgp =NULL ; -return 0 ; -} - -static inline int mem_cgroup_try_charge_delay (struct page *page, -struct mm_struct *mm, -gfp_t gfp_mask, -struct mem_cgroup **memcgp) -{ - *memcgp =NULL ; -return 0 ; -} - -static inline void mem_cgroup_commit_charge (struct page *page, -struct mem_cgroup *memcg, -bool lrucare) -{ -} - -static inline void mem_cgroup_cancel_charge (struct page *page, -struct mem_cgroup *memcg) -{ -} -static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask,bool lrucare) {
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-drop-unused-try-commit-cancel-charge-api +17 -111
184 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 97/131 mm: memcontrol: prepare swap controller setup for integration
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: prepare swap controller setup for integration
A few cleanups to streamline the swap controller setup:
- Replace the do_swap_account flag with cgroup_memory_noswap. This brings it in line with other functionality that is usually available unless explicitly opted out of - nosocket, nokmem.
- Remove the really_do_swap_account flag that stores the boot option and is later used to switch the do_swap_account. It's not clear why this indirection is/was necessary. Use do_swap_account directly.
- Minor coding style polishing
---
include/linux/memcontrol.h | 2 - mm/memcontrol.c | 59 ++++++++++++++++------------------- mm/swap_cgroup.c | 4 +- 3 files changed, 31 insertions(+), 34 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-prepare-swap-controller-setup-for-integration +1 -1
@@ -558,7 +558,7 @@ struct mem_cgroup *mem_cgroup_get_oom_grvoid mem_cgroup_print_oom_group (struct mem_cgroup *memcg);#ifdef CONFIG_MEMCG_SWAP -extern int do_swap_account; +extern bool cgroup_memory_noswap;#endif struct mem_cgroup *lock_page_memcg (struct page *page);
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-prepare-swap-controller-setup-for-integration +28 -31
133 lines not on this page · raw message
mm/swap_cgroup.c renamed from mm/swap_cgroup.c~mm-memcontrol-prepare-swap-controller-setup-for-integration +2 -2
@@ -171,7 +171,7 @@ int swap_cgroup_swapon(int type, unsigneunsigned long length;struct swap_cgroup_ctrl *ctrl; -if (!do_swap_account) +if (cgroup_memory_noswap)return 0 ; length =DIV_ROUND_UP (max_pages,SC_PER_PAGE );
@@ -209,7 +209,7 @@ void swap_cgroup_swapoff(int type)unsigned long i, length;struct swap_cgroup_ctrl *ctrl; -if (!do_swap_account) +if (cgroup_memory_noswap)return ;mutex_lock (&swap_cgroup_mutex);
_
Andrew Morton
maintainerPATCH 98/131 mm: memcontrol: make swap tracking an integral part of memory control
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: make swap tracking an integral part of memory control
Without swap page tracking, users that are otherwise memory controlled can easily escape their containment and allocate significant amounts of memory that they're not being charged for. That's because swap does readahead, but without the cgroup records of who owned the page at swapout, readahead pages don't get charged until somebody actually faults them into their page table and we can identify an owner task. This can be maliciously exploited with MADV_WILLNEED, which triggers arbitrary readahead allocations without charging the pages.
Make swap swap page tracking an integral part of memcg and remove the Kconfig options. In the first place, it was only made configurable to allow users to save some memory. But the overhead of tracking cgroup ownership per swap page is minimal - 2 byte per page, or 512k per 1G of swap, or 0.04%. Saving that at the expense of broken containment semantics is not something we should present as a coequal option.
The swapaccount=0 boot option will continue to exist, and it will eliminate the page_counter overhead and hide the swap control files, but it won't disable swap slot ownership tracking.
This patch makes sure we always have the cgroup records at swapin time; the next patch will fix the actual bug by charging readahead swap pages at swapin time rather than at fault time.
v2: fix double swap charge bug in cgroup1/cgroup2 code gating
---
init/Kconfig | 17 -------------- mm/memcontrol.c | 53 +++++++++++++++++++-------------------------- mm/swap_cgroup.c | 6 ----- 3 files changed, 24 insertions(+), 52 deletions(-)
init/Kconfig renamed from init/Kconfig~mm-memcontrol-make-swap-tracking-an-integral-part-of-memory-control +1 -16
@@ -819,24 +819,9 @@ config MEMCG Provides control over the memory footprint of tasks in a cgroup. config MEMCG_SWAP - bool "Swap controller" + bool depends on MEMCG && SWAP - help - Provides control over the swap space consumed by tasks in a cgroup. - -config MEMCG_SWAP_ENABLED - bool "Swap controller enabled by default" - depends on MEMCG_SWAP default y - help - Memory Resource Controller Swap Extension comes with its price in - a bigger memory consumption. General purpose distribution kernels - which want to enable the feature but keep it disabled by default - and let the user enable it by swapaccount=1 boot command line - parameter should have this option unselected. - For those who want to have the feature enabled by default should - select this option (if, for some reason, they need to disable it - then swapaccount=0 does the trick). config MEMCG_KMEM bool
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-make-swap-tracking-an-integral-part-of-memory-control +23 -30
132 lines not on this page · raw message
mm/swap_cgroup.c renamed from mm/swap_cgroup.c~mm-memcontrol-make-swap-tracking-an-integral-part-of-memory-control +0 -6
@@ -171,9 +171,6 @@ int swap_cgroup_swapon(int type, unsigneunsigned long length;struct swap_cgroup_ctrl *ctrl; -if (cgroup_memory_noswap) -return 0 ; - length =DIV_ROUND_UP (max_pages,SC_PER_PAGE ); array_size = length *sizeof (void *);
@@ -209,9 +206,6 @@ void swap_cgroup_swapoff(int type)unsigned long i, length;struct swap_cgroup_ctrl *ctrl; -if (cgroup_memory_noswap) -return ; -mutex_lock (&swap_cgroup_mutex); ctrl = &swap_cgroup_ctrl[type]; map = ctrl->map;
_
Andrew Morton
maintainerPATCH 99/131 mm: memcontrol: charge swapin pages on instantiation
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: charge swapin pages on instantiation
Right now, users that are otherwise memory controlled can easily escape their containment and allocate significant amounts of memory that they're not being charged for. That's because swap readahead pages are not being charged until somebody actually faults them into their page table. This can be exploited with MADV_WILLNEED, which triggers arbitrary readahead allocations without charging the pages.
There are additional problems with the delayed charging of swap pages:
1. To implement refault/workingset detection for anonymous pages, we need to have a target LRU available at swapin time, but the LRU is not determinable until the page has been charged.
2. To implement per-cgroup LRU locking, we need page->mem_cgroup to be stable when the page is isolated from the LRU; otherwise, the locks change under us. But swapcache gets charged after it's already on the LRU, and even if we cannot isolate it ourselves (since charging is not exactly optional).
The previous patch ensured we always maintain cgroup ownership records for swap pages. This patch moves the swapcache charging point from the fault handler to swapin time to fix all of the above problems.
v2: simplify swapin error checking (Joonsoo)
---
mm/memory.c | 15 +++++- mm/shmem.c | 14 +++--- mm/swap_state.c | 99 ++++++++++++++++++++++++++-------------------- mm/swapfile.c | 6 -- 4 files changed, 75 insertions(+), 59 deletions(-)
mm/memory.c renamed from mm/memory.c~mm-memcontrol-charge-swapin-pages-on-instantiation +11 -4
@@ -3125,9 +3125,20 @@ vm_fault_t do_swap_page(struct vm_fault page =alloc_page_vma (GFP_HIGHUSER_MOVABLE , vma, vmf->address);if (page) { +int err; +__SetPageLocked (page);__SetPageSwapBacked (page);set_page_private (page, entry.val); + +/* Tell memcg to use swap ownership records */ +SetPageSwapCache (page); + err =mem_cgroup_charge (page, vma->vm_mm, +GFP_KERNEL , false); +ClearPageSwapCache (page); +if (err) + goto out_page; +lru_cache_add_anon (page);swap_readpage (page, true); }
@@ -3189,10 +3200,6 @@ vm_fault_t do_swap_page(struct vm_fault goto out_page; } -if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , true)) { - ret =VM_FAULT_OOM ; - goto out_page; - }cgroup_throttle_swaprate (page,GFP_KERNEL );/*
mm/shmem.c renamed from mm/shmem.c~mm-memcontrol-charge-swapin-pages-on-instantiation +8 -6
@@ -623,13 +623,15 @@ static int shmem_add_to_page_cache(struc page->mapping = mapping; page->index = index; - error =mem_cgroup_charge (page, charge_mm, gfp,PageSwapCache (page)); -if (error) { -if (!PageSwapCache (page) &&PageTransHuge (page)) { -count_vm_event (THP_FILE_FALLBACK ); -count_vm_event (THP_FILE_FALLBACK_CHARGE ); +if (!PageSwapCache (page)) { + error =mem_cgroup_charge (page, charge_mm, gfp, false); +if (error) { +if (PageTransHuge (page)) { +count_vm_event (THP_FILE_FALLBACK ); +count_vm_event (THP_FILE_FALLBACK_CHARGE ); + } + goto error; } - goto error; }cgroup_throttle_swaprate (page, gfp);
mm/swapfile.c renamed from mm/swapfile.c~mm-memcontrol-charge-swapin-pages-on-instantiation +0 -6
@@ -1901,11 +1901,6 @@ static int unuse_pte(struct vm_area_struif (unlikely (!page))return -ENOMEM ; -if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , true)) { - ret = -ENOMEM ; - goto out_nolock; - } - pte =pte_offset_map_lock (vma->vm_mm, pmd, addr, &ptl);if (unlikely (!pte_same_as_swp (*pte,swp_entry_to_pte (entry)))) { ret =0 ;
@@ -1931,7 +1926,6 @@ static int unuse_pte(struct vm_area_struactivate_page (page); out:pte_unmap_unlock (pte, ptl); -out_nolock:if (page != swapcache) {unlock_page (page);put_page (page);
mm/swap_state.c renamed from mm/swap_state.c~mm-memcontrol-charge-swapin-pages-on-instantiation +56 -43
134 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 100/131 mm: memcontrol: document the new swap control behavior
In reply to Andrew MortonFrom: Alex Shi <[email protected]>
Subject: mm: memcontrol: document the new swap control behavior
---
Documentation/admin-guide/cgroup-v1/memory.rst | 19 +++++---------- 1 file changed, 7 insertions(+), 12 deletions(-)
Documentation/admin-guide/cgroup-v1/memory.rst renamed from Documentation/admin-guide/cgroup-v1/memory.rst~mm-memcontrol-document-the-new-swap-control-behavior +7 -12
@@ -199,11 +199,11 @@ An RSS page is unaccounted when it's ful unaccounted when it's removed from radix-tree. Even if RSS pages are fully unmapped (by kswapd), they may exist as SwapCache in the system until they are really freed. Such SwapCaches are also accounted. -A swapped-in page is not accounted until it's mapped. +A swapped-in page is accounted after adding into swapcache. Note: The kernel does swapin-readahead and reads multiple swaps at once. -This means swapped-in pages may contain pages for other tasks than a task -causing page fault. So, we avoid accounting at swap-in I/O. +Since page's memcg recorded into swap whatever memsw enabled, the page will +be accounted after swapin. At page migration, accounting information is kept.
@@ -222,18 +222,13 @@ the cgroup that brought it in -- this wi But see section 8.2: when moving a task to another cgroup, its pages may be recharged to the new cgroup, if move_charge_at_immigrate has been chosen. -Exception: If CONFIG_MEMCG_SWAP is not used. -When you do swapoff and make swapped-out pages of shmem(tmpfs) to -be backed into memory in force, charges for pages are accounted against the -caller of swapoff rather than the users of shmem. - -2.4 Swap Extension (CONFIG_MEMCG_SWAP) +2.4 Swap Extension -------------------------------------- -Swap Extension allows you to record charge for swap. A swapped-in page is -charged back to original page allocator if possible. +Swap usage is always recorded for each of cgroup. Swap Extension allows you to +read and limit it. -When swap is accounted, following files are added. +When CONFIG_SWAP is enabled, following files are added. - memory.memsw.usage_in_bytes. - memory.memsw.limit_in_bytes.
_
Andrew Morton
maintainerPATCH 101/131 mm: memcontrol: delete unused lrucare handling
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: delete unused lrucare handling
Swapin faults were the last event to charge pages after they had already been put on the LRU list. Now that we charge directly on swapin, the lrucare portion of the charge code is unused.
---
include/linux/memcontrol.h | 5 +-- kernel/events/uprobes.c | 3 - mm/filemap.c | 2 - mm/huge_memory.c | 2 - mm/khugepaged.c | 4 +- mm/memcontrol.c | 57 ++--------------------------------- mm/memory.c | 8 ++-- mm/migrate.c | 2 - mm/shmem.c | 2 - mm/swap_state.c | 2 - mm/userfaultfd.c | 2 - 11 files changed, 19 insertions(+), 70 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcontrol-delete-unused-lrucare-handling +2 -3
@@ -355,8 +355,7 @@ static inline unsigned long mem_cgroup_penum mem_cgroup_protection mem_cgroup_protected (struct mem_cgroup *root,struct mem_cgroup *memcg); -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask, -bool lrucare); +int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask);void mem_cgroup_uncharge (struct page *page);void mem_cgroup_uncharge_list (struct list_head *page_list);
@@ -839,7 +838,7 @@ static inline enum mem_cgroup_protection }static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, -gfp_t gfp_mask,bool lrucare) +gfp_t gfp_mask) {return 0 ; }
kernel/events/uprobes.c renamed from kernel/events/uprobes.c~mm-memcontrol-delete-unused-lrucare-handling +1 -2
@@ -167,8 +167,7 @@ static int __replace_page(struct vm_area addr + PAGE_SIZE);if (new_page) { - err =mem_cgroup_charge (new_page, vma->vm_mm,GFP_KERNEL , - false); + err =mem_cgroup_charge (new_page, vma->vm_mm,GFP_KERNEL );if (err)return err; }
mm/filemap.c renamed from mm/filemap.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -845,7 +845,7 @@ static int __add_to_page_cache_locked(st page->index = offset;if (!huge) { - error =mem_cgroup_charge (page, current->mm, gfp_mask, false); + error =mem_cgroup_charge (page, current->mm, gfp_mask);if (error) goto error; }
mm/huge_memory.c renamed from mm/huge_memory.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -593,7 +593,7 @@ static vm_fault_t __do_huge_pmd_anonymouVM_BUG_ON_PAGE (!PageCompound (page), page); -if (mem_cgroup_charge (page, vma->vm_mm, gfp, false)) { +if (mem_cgroup_charge (page, vma->vm_mm, gfp)) {put_page (page);count_vm_event (THP_FAULT_FALLBACK );count_vm_event (THP_FAULT_FALLBACK_CHARGE );
mm/khugepaged.c renamed from mm/khugepaged.c~mm-memcontrol-delete-unused-lrucare-handling +2 -2
@@ -1059,7 +1059,7 @@ static void collapse_huge_page(struct mm goto out_nolock; } -if (unlikely (mem_cgroup_charge (new_page, mm, gfp, false))) { +if (unlikely (mem_cgroup_charge (new_page, mm, gfp))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out_nolock; }
@@ -1632,7 +1632,7 @@ static void collapse_file(struct mm_stru goto out; } -if (unlikely (mem_cgroup_charge (new_page, mm, gfp, false))) { +if (unlikely (mem_cgroup_charge (new_page, mm, gfp))) { result =SCAN_CGROUP_CHARGE_FAIL ; goto out; }
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-delete-unused-lrucare-handling +4 -53
100 lines not on this page · raw message
mm/memory.c renamed from mm/memory.c~mm-memcontrol-delete-unused-lrucare-handling +4 -4
@@ -2675,7 +2675,7 @@ static vm_fault_t wp_page_copy(struct vm } } -if (mem_cgroup_charge (new_page, mm,GFP_KERNEL , false)) +if (mem_cgroup_charge (new_page, mm,GFP_KERNEL )) goto oom_free_new;cgroup_throttle_swaprate (new_page,GFP_KERNEL );
@@ -3134,7 +3134,7 @@ vm_fault_t do_swap_page(struct vm_fault/* Tell memcg to use swap ownership records */ SetPageSwapCache (page); err =mem_cgroup_charge (page, vma->vm_mm, -GFP_KERNEL , false); +GFP_KERNEL );ClearPageSwapCache (page);if (err) goto out_page;
@@ -3358,7 +3358,7 @@ static vm_fault_t do_anonymous_page(struif (!page) goto oom; -if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , false)) +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL )) goto oom_free_page;cgroup_throttle_swaprate (page,GFP_KERNEL );
@@ -3854,7 +3854,7 @@ static vm_fault_t do_cow_fault(struct vmif (!vmf->cow_page)return VM_FAULT_OOM ; -if (mem_cgroup_charge (vmf->cow_page, vma->vm_mm,GFP_KERNEL , false)) { +if (mem_cgroup_charge (vmf->cow_page, vma->vm_mm,GFP_KERNEL )) {put_page (vmf->cow_page);return VM_FAULT_OOM ; }
mm/migrate.c renamed from mm/migrate.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -2786,7 +2786,7 @@ static void migrate_vma_insert_page(struif (unlikely (anon_vma_prepare (vma))) goto abort; -if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL , false)) +if (mem_cgroup_charge (page, vma->vm_mm,GFP_KERNEL )) goto abort;/*
mm/shmem.c renamed from mm/shmem.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -624,7 +624,7 @@ static int shmem_add_to_page_cache(struc page->index = index;if (!PageSwapCache (page)) { - error =mem_cgroup_charge (page, charge_mm, gfp, false); + error =mem_cgroup_charge (page, charge_mm, gfp);if (error) {if (PageTransHuge (page)) {count_vm_event (THP_FILE_FALLBACK );
mm/swap_state.c renamed from mm/swap_state.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -435,7 +435,7 @@ struct page *__read_swap_cache_async(swp goto fail_unlock; } -if (mem_cgroup_charge (page,NULL , gfp_mask, false)) { +if (mem_cgroup_charge (page,NULL , gfp_mask)) {delete_from_swap_cache (page); goto fail_unlock; }
mm/userfaultfd.c renamed from mm/userfaultfd.c~mm-memcontrol-delete-unused-lrucare-handling +1 -1
@@ -96,7 +96,7 @@ static int mcopy_atomic_pte(struct mm_st__SetPageUptodate (page); ret = -ENOMEM ; -if (mem_cgroup_charge (page, dst_mm,GFP_KERNEL , false)) +if (mem_cgroup_charge (page, dst_mm,GFP_KERNEL )) goto out_release; _dst_pte =pte_mkdirty (mk_pte (page, dst_vma->vm_page_prot));
_
Andrew Morton
maintainerPATCH 102/131 mm: memcontrol: update page->mem_cgroup stability rules
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: memcontrol: update page->mem_cgroup stability rules
The previous patches have simplified the access rules around page->mem_cgroup somewhat:
1. We never change page->mem_cgroup while the page is isolated by somebody else. This was by far the biggest exception to our rules and it didn't stop at lock_page() or lock_page_memcg().
2. We charge pages before they get put into page tables now, so the somewhat fishy rule about "can be in page table as long as it's still locked" is now gone and boiled down to having an exclusive reference to the page.
Document the new rules. Any of the following will stabilize the page->mem_cgroup association:
- the page lock
- LRU isolation
- lock_page_memcg()
- exclusive access to the page
---
mm/memcontrol.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-)
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcontrol-update-page-mem_cgroup-stability-rules +7 -14
@@ -1201,9 +1201,8 @@ int mem_cgroup_scan_tasks(struct mem_cgr* @page: the page * @pgdat: pgdat of the page * -* This function is only safe when following the LRU page isolation -* and putback protocol: the LRU lock must be held, and the page must -* either be PageLRU() or the caller must have isolated/allocated it. +* This function relies on page->mem_cgroup being stable - see the +* access rules in commit_charge(). */ struct lruvec *mem_cgroup_page_lruvec (struct page *page,struct pglist_data *pgdat) {
@@ -2659,18 +2658,12 @@ static void commit_charge(struct page *p {VM_BUG_ON_PAGE (page->mem_cgroup, page);/* -* Nobody should be changing or seriously looking at -* page->mem_cgroup at this point: +* Any of the following ensures page->mem_cgroup stability: * -* - the page is uncharged -* -* - the page is off-LRU -* -* - an anonymous fault has exclusive page access, except for -* a locked page table -* -* - a page cache insertion, a swapin fault, or a migration -* have the page locked +* - the page lock +* - LRU isolation +* - lock_page_memcg() +* - exclusive reference */ page->mem_cgroup = memcg; }
_
Andrew Morton
maintainerPATCH 103/131 mm: fix LRU balancing effect of new transparent huge pages
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: fix LRU balancing effect of new transparent huge pages
The reclaim code that balances between swapping and cache reclaim tries to predict likely reuse based on in-memory reference patterns alone. This works in many cases, but when it fails it cannot detect when the cache is thrashing pathologically, or when we're in the middle of a swap storm.
The high seek cost of rotational drives under which the algorithm evolved also meant that mistakes could quickly result in lockups from too aggressive swapping (which is predominantly random IO). As a result, the balancing code has been tuned over time to a point where it mostly goes for page cache and defers swapping until the VM is under significant memory pressure.
The resulting strategy doesn't make optimal caching decisions - where optimal is the least amount of IO required to execute the workload.
The proliferation of fast random IO devices such as SSDs, in-memory compression such as zswap, and persistent memory technologies on the horizon, has made this undesirable behavior very noticable: Even in the presence of large amounts of cold anonymous memory and a capable swap device, the VM refuses to even seriously scan these pages, and can leave the page cache thrashing needlessly.
This series sets out to address this. Since commit ("a528910e12ec mm:
thrash detection-based file cache sizing") we have exact tracking of refault IO - the ultimate cost of reclaiming the wrong pages. This allows us to use an IO cost based balancing model that is more aggressive about scanning anonymous memory when the cache is thrashing, while being able to avoid unnecessary swap storms.
These patches base the LRU balance on the rate of refaults on each list, times the relative IO cost between swap device and filesystem (swappiness), in order to optimize reclaim for least IO cost incurred.
History
I floated these changes in 2016. At the time they were incomplete and full of workarounds due to a lack of infrastructure in the reclaim code:
We didn't have PageWorkingset, we didn't have hierarchical cgroup statistics, and problems with the cgroup swap controller. As swapping wasn't too high a priority then, the patches stalled out. With all dependencies in place now, here we are again with much cleaner, feature-complete patches.
I kept the acks for patches that stayed materially the same :-)
Below is a series of test results that demonstrate certain problematic behavior of the current code, as well as showcase the new code's more predictable and appropriate balancing decisions.
Test #1: No convergence
This test shows an edge case where the VM currently doesn't converge at all on a new file workingset with a stale anon/tmpfs set.
The test sets up a cold anon set the size of 3/4 RAM, then tries to establish a new file set half the size of RAM (flat access pattern).
The vanilla kernel refuses to even scan anon pages and never converges. The file set is perpetually served from the filesystem.
The first test kernel is with the series up to the workingset patch applied. This allows thrashing page cache to challenge the anonymous workingset. The VM then scans the lists based on the current scanned/rotated balancing algorithm. It converges on a stable state where all cold anon pages are pushed out and the fileset is served entirely from cache:
noconverge/5.7-rc5-mm noconverge/5.7-rc5-mm-workingset
Scanned 417719308.00 ( +0.00%) 64091155.00 ( -84.66%)
Reclaimed 417711094.00 ( +0.00%) 61640308.00 ( -85.24%)
Reclaim efficiency % 100.00 ( +0.00%) 96.18 ( -3.78%)
Scanned file 417719308.00 ( +0.00%) 59211118.00 ( -85.83%)
Scanned anon 0.00 ( +0.00%) 4880037.00 ( )
Swapouts 0.00 ( +0.00%) 2439957.00 ( )
Swapins 0.00 ( +0.00%) 257.00 ( )
Refaults 415246605.00 ( +0.00%) 59183722.00 ( -85.75%)
Restore refaults 0.00 ( +0.00%) 54988252.00 ( )
The second test kernel is with the full patch series applied, which replaces the scanned/rotated ratios with refault/swapin rate-based balancing. It evicts the cold anon pages more aggressively in the presence of a thrashing cache and the absence of swapins, and so converges with about 60% of the IO and reclaim activity:
noconverge/5.7-rc5-mm-workingset noconverge/5.7-rc5-mm-lrubalance
Scanned 64091155.00 ( +0.00%) 37579741.00 ( -41.37%)
Reclaimed 61640308.00 ( +0.00%) 35129293.00 ( -43.01%)
Reclaim efficiency % 96.18 ( +0.00%) 93.48 ( -2.78%)
Scanned file 59211118.00 ( +0.00%) 32708385.00 ( -44.76%)
Scanned anon 4880037.00 ( +0.00%) 4871356.00 ( -0.18%)
Swapouts 2439957.00 ( +0.00%) 2435565.00 ( -0.18%) Swapins 257.00 ( +0.00%) 262.00 ( +1.94%) Refaults 59183722.00 ( +0.00%) 32675667.00 ( -44.79%)
Restore refaults 54988252.00 ( +0.00%) 28480430.00 ( -48.21%)
We're triggering this case in host sideloading scenarios: When a host's primary workload is not saturating the machine (primary load is usually driven by user activity), we can optimistically sideload a batch job; if user activity picks up and the primary workload needs the whole host during this time, we freeze the sideload and rely on it getting pushed to swap. Frequently that swapping doesn't happen and the completely inactive sideload simply stays resident while the expanding primary worklad is struggling to gain ground.
Test #2: Kernel build
This test is a a kernel build that is slightly memory-restricted (make -j4 inside a 400M cgroup).
Despite the very aggressive swapping of cold anon pages in test #1, this test shows that the new kernel carefully balances swap against cache refaults when both the file and the cache set are pressured.
It shows the patched kernel to be slightly better at finding the coldest memory from the combined anon and file set to evict under pressure. The result is lower aggregate reclaim and paging activity:
z 5.7-rc5-mm 5.7-rc5-mm-lrubalance
Real time 210.60 ( +0.00%) 210.97 ( +0.18%) User time 745.42 ( +0.00%) 746.48 ( +0.14%) System time 69.78 ( +0.00%) 69.79 ( +0.02%) Scanned file 354682.00 ( +0.00%) 293661.00 ( -17.20%) Scanned anon 465381.00 ( +0.00%) 378144.00 ( -18.75%) Swapouts 185920.00 ( +0.00%) 147801.00 ( -20.50%) Swapins 34583.00 ( +0.00%) 32491.00 ( -6.05%) Refaults 212664.00 ( +0.00%) 172409.00 ( -18.93%) Restore refaults 48861.00 ( +0.00%) 80091.00 ( +63.91%) Total paging IO 433167.00 ( +0.00%) 352701.00 ( -18.58%)
Test #3: Overload
This next test is not about performance, but rather about the predictability of the algorithm. The current balancing behavior doesn't always lead to comprehensible results, which makes performance analysis and parameter tuning (swappiness e.g.) very difficult.
The test shows the balancing behavior under equivalent anon and file input. Anon and file sets are created of equal size (3/4 RAM), have the same access patterns (a hot-cold gradient), and synchronized access rates. Swappiness is raised from the default of 60 to 100 to indicate equal IO cost between swap and cache.
With the vanilla balancing code, anon scans make up around 9% of the total pages scanned, or a ~1:10 ratio. This is a surprisingly skewed ratio, and it's an outcome that is hard to explain given the input parameters to the VM.
The new balancing model targets a 1:2 balance: All else being equal, reclaiming a file page costs one page IO - the refault; reclaiming an anon page costs two IOs - the swapout and the swapin. In the test we observe a ~1:3 balance.
The scanned and paging IO numbers indicate that the anon LRU algorithm we have in place right now does a slightly worse job at picking the coldest pages compared to the file algorithm. There is ongoing work to improve this, like Joonsoo's anon workingset patches; however, it's difficult to compare the two aging strategies when the balancing between them is behaving unintuitively.
The slightly less efficient anon reclaim results in a deviation from the optimal 1:2 scan ratio we would like to see here - however, 1:3 is much closer to what we'd want to see in this test than the vanilla kernel's aging of 10+ cache pages for every anonymous one:
overload-100/5.7-rc5-mm-workingset overload-100/5.7-rc5-mm-lrubalance-realfile Scanned 533633725.00 ( +0.00%) 595687785.00 ( +11.63%) Reclaimed 494325440.00 ( +0.00%) 518154380.00 ( +4.82%) Reclaim efficiency % 92.63 ( +0.00%) 86.98 ( -6.03%) Scanned file 484532894.00 ( +0.00%) 456937722.00 ( -5.70%) Scanned anon 49100831.00 ( +0.00%) 138750063.00 ( +182.58%) Swapouts 8096423.00 ( +0.00%) 48982142.00 ( +504.98%) Swapins 10027384.00 ( +0.00%) 62325044.00 ( +521.55%) Refaults 479819973.00 ( +0.00%) 451309483.00 ( -5.94%) Restore refaults 426422087.00 ( +0.00%) 399914067.00 ( -6.22%) Total paging IO 497943780.00 ( +0.00%) 562616669.00 ( +12.99%)
Test #4: Parallel IO
It's important to note that these patches only affect the situation where the kernel has to reclaim workingset memory, which is usually a transitionary period. The vast majority of page reclaim occuring in a system is from trimming the ever-expanding page cache.
These patches don't affect cache trimming behavior. We never swap as long as we only have use-once cache moving through the file LRU, we only consider swapping when the cache is actively thrashing.
The following test demonstrates this. It has an anon workingset that takes up half of RAM and then writes a file that is twice the size of RAM out to disk.
As the cache is funneled through the inactive file list, no anon pages are scanned (aside from apparently some background noise of 10 pages):
5.7-rc5-mm 5.7-rc5-mm-lrubalance
Scanned 10714722.00 ( +0.00%) 10723445.00 ( +0.08%)
Reclaimed 10703596.00 ( +0.00%) 10712166.00 ( +0.08%)
Reclaim efficiency % 99.90 ( +0.00%) 99.89 ( -0.00%)
Scanned file 10714722.00 ( +0.00%) 10723435.00 ( +0.08%)
Scanned anon 0.00 ( +0.00%) 10.00 ( )
Swapouts 0.00 ( +0.00%) 7.00 ( ) Swapins 0.00 ( +0.00%) 0.00 ( +0.00%) Refaults 92.00 ( +0.00%) 41.00 ( -54.84%)
Restore refaults 0.00 ( +0.00%) 0.00 ( +0.00%)
Total paging IO 92.00 ( +0.00%) 48.00 ( -47.31%)
This patch (of 14):
Currently, THP are counted as single pages until they are split right before being swapped out. However, at that point the VM is already in the middle of reclaim, and adjusting the LRU balance then is useless.
Always account THP by the number of basepages, and remove the fixup from the splitting path.
---
mm/swap.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
mm/swap.c renamed from mm/swap.c~mm-fix-lru-balancing-effect-of-new-transparent-huge-pages +11 -14
@@ -279,13 +279,14 @@ void rotate_reclaimable_page(struct page }static void update_page_reclaim_stat (struct lruvec *lruvec, -int file,int rotated) +int file,int rotated, +unsigned int nr_pages) {struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; - reclaim_stat->recent_scanned[file]++; + reclaim_stat->recent_scanned[file] += nr_pages;if (rotated) - reclaim_stat->recent_rotated[file]++; + reclaim_stat->recent_rotated[file] += nr_pages; } static void __activate_page(struct page *page, struct lruvec *lruvec,
@@ -302,7 +303,7 @@ static void __activate_page(struct pagetrace_mm_lru_activate (page);__count_vm_event (PGACTIVATE ); -update_page_reclaim_stat (lruvec, file,1 ); +update_page_reclaim_stat (lruvec, file,1 ,hpage_nr_pages (page)); } }
@@ -564,7 +565,7 @@ static void lru_deactivate_file_fn(strucif (active)__count_vm_event (PGDEACTIVATE ); -update_page_reclaim_stat (lruvec, file,0 ); +update_page_reclaim_stat (lruvec, file,0 ,hpage_nr_pages (page)); } static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
@@ -580,7 +581,7 @@ static void lru_deactivate_fn(struct pagadd_page_to_lru_list (page, lruvec, lru);__count_vm_events (PGDEACTIVATE ,hpage_nr_pages (page)); -update_page_reclaim_stat (lruvec, file,0 ); +update_page_reclaim_stat (lruvec, file,0 ,hpage_nr_pages (page)); } }
@@ -605,7 +606,7 @@ static void lru_lazyfree_fn(struct page__count_vm_events (PGLAZYFREE ,hpage_nr_pages (page));count_memcg_page_event (page,PGLAZYFREE ); -update_page_reclaim_stat (lruvec,1 ,0 ); +update_page_reclaim_stat (lruvec,1 ,0 ,hpage_nr_pages (page)); } }
@@ -929,8 +930,6 @@ EXPORT_SYMBOL(__pagevec_release);void lru_add_page_tail (struct page *page,struct page *page_tail,struct lruvec *lruvec,struct list_head *list) { -const int file =0 ; -VM_BUG_ON_PAGE (!PageHead (page), page);VM_BUG_ON_PAGE (PageCompound (page_tail), page);VM_BUG_ON_PAGE (PageLRU (page_tail), page);
@@ -956,9 +955,6 @@ void lru_add_page_tail(struct page *pageadd_page_to_lru_list_tail (page_tail, lruvec, page_lru(page_tail)); } - -if (!PageUnevictable (page)) -update_page_reclaim_stat (lruvec, file,PageActive (page_tail)); }#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -1001,8 +997,9 @@ static void __pagevec_lru_add_fn(structif (page_evictable (page)) { lru =page_lru (page); -update_page_reclaim_stat (lruvec,page_is_file_lru (page), -PageActive (page)); +update_page_reclaim_stat (lruvec,is_file_lru (lru), +PageActive (page), +hpage_nr_pages (page));if (was_unevictable)count_vm_event (UNEVICTABLE_PGRESCUED ); }else {
_
Andrew Morton
maintainerPATCH 104/131 mm: keep separate anon and file statistics on page reclaim activity
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: keep separate anon and file statistics on page reclaim activity
Having statistics on pages scanned and pages reclaimed for both anon and file pages makes it easier to evaluate changes to LRU balancing.
While at it, clean up the stat-keeping mess for isolation, putback, reclaim stats etc. a bit: first the physical LRU operation (isolation and putback), followed by vmstats, reclaim_stats, and then vm events.
---
include/linux/vm_event_item.h | 4 ++++ mm/vmscan.c | 17 +++++++++-------- mm/vmstat.c | 4 ++++ 3 files changed, 17 insertions(+), 8 deletions(-)
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~mm-keep-separate-anon-and-file-statistics-on-page-reclaim-activity +4 -0
@@ -35,6 +35,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSPGSCAN_KSWAPD ,PGSCAN_DIRECT ,PGSCAN_DIRECT_THROTTLE , +PGSCAN_ANON , +PGSCAN_FILE , +PGSTEAL_ANON , +PGSTEAL_FILE , #ifdef CONFIG_NUMA PGSCAN_ZONE_RECLAIM_FAILED, #endif
mm/vmscan.c renamed from mm/vmscan.c~mm-keep-separate-anon-and-file-statistics-on-page-reclaim-activity +9 -8
@@ -1913,7 +1913,7 @@ shrink_inactive_list(unsigned long nr_tounsigned int nr_reclaimed =0 ;unsigned long nr_taken;struct reclaim_stat stat; -int file =is_file_lru (lru); +bool file =is_file_lru (lru);enum vm_event_item item;struct pglist_data *pgdat =lruvec_pgdat (lruvec);struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
@@ -1941,11 +1941,12 @@ shrink_inactive_list(unsigned long nr_to__mod_node_page_state (pgdat,NR_ISOLATED_ANON + file, nr_taken); reclaim_stat->recent_scanned[file] += nr_taken; - item =current_is_kswapd () ?PGSCAN_KSWAPD :PGSCAN_DIRECT ;if (!cgroup_reclaim (sc))__count_vm_events (item, nr_scanned);__count_memcg_events (lruvec_memcg (lruvec), item, nr_scanned); +__count_vm_events (PGSCAN_ANON + file, nr_scanned); +spin_unlock_irq (&pgdat->lru_lock);if (nr_taken ==0 )
@@ -1956,16 +1957,16 @@ shrink_inactive_list(unsigned long nr_tospin_lock_irq (&pgdat->lru_lock); +move_pages_to_lru (lruvec, &page_list); + +__mod_node_page_state (pgdat,NR_ISOLATED_ANON + file, -nr_taken); + reclaim_stat->recent_rotated[0 ] += stat.nr_activate[0 ]; + reclaim_stat->recent_rotated[1 ] += stat.nr_activate[1 ]; item =current_is_kswapd () ?PGSTEAL_KSWAPD :PGSTEAL_DIRECT ;if (!cgroup_reclaim (sc))__count_vm_events (item, nr_reclaimed);__count_memcg_events (lruvec_memcg (lruvec), item, nr_reclaimed); - reclaim_stat->recent_rotated[0 ] += stat.nr_activate[0 ]; - reclaim_stat->recent_rotated[1 ] += stat.nr_activate[1 ]; - -move_pages_to_lru (lruvec, &page_list); - -__mod_node_page_state (pgdat,NR_ISOLATED_ANON + file, -nr_taken); +__count_vm_events (PGSTEAL_ANON + file, nr_reclaimed);spin_unlock_irq (&pgdat->lru_lock);
mm/vmstat.c renamed from mm/vmstat.c~mm-keep-separate-anon-and-file-statistics-on-page-reclaim-activity +4 -0
@@ -1203,6 +1203,10 @@ const char * const vmstat_text[] = {"pgscan_kswapd" ,"pgscan_direct" ,"pgscan_direct_throttle" , +"pgscan_anon" , +"pgscan_file" , +"pgsteal_anon" , +"pgsteal_file" , #ifdef CONFIG_NUMA "zone_reclaim_failed",
_
Andrew Morton
maintainerPATCH 105/131 mm: allow swappiness that prefers reclaiming anon over the file workingset
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: allow swappiness that prefers reclaiming anon over the file workingset
With the advent of fast random IO devices (SSDs, PMEM) and in-memory swap devices such as zswap, it's possible for swap to be much faster than filesystems, and for swapping to be preferable over thrashing filesystem caches.
Allow setting swappiness - which defines the rough relative IO cost of cache misses between page cache and swap-backed pages - to reflect such situations by making the swap-preferred range configurable.
---
Documentation/admin-guide/sysctl/vm.rst | 23 +++++++++++++++++----- kernel/sysctl.c | 3 +- mm/vmscan.c | 2 - 3 files changed, 21 insertions(+), 7 deletions(-)
Documentation/admin-guide/sysctl/vm.rst renamed from Documentation/admin-guide/sysctl/vm.rst~mm-allow-swappiness-that-prefers-reclaiming-anon-over-the-file-workingset +18 -5
@@ -831,14 +831,27 @@ tooling to work, you can do:: swappiness ========== -This control is used to define how aggressive the kernel will swap -memory pages. Higher values will increase aggressiveness, lower values -decrease the amount of swap. A value of 0 instructs the kernel not to -initiate swap until the amount of free and file-backed pages is less -than the high water mark in a zone. +This control is used to define the rough relative IO cost of swapping +and filesystem paging, as a value between 0 and 200. At 100, the VM +assumes equal IO cost and will thus apply memory pressure to the page +cache and swap-backed pages equally; lower values signify more +expensive swap IO, higher values indicates cheaper. + +Keep in mind that filesystem IO patterns under memory pressure tend to +be more efficient than swap's random IO. An optimal value will require +experimentation and will also be workload-dependent. The default value is 60. +For in-memory swap, like zram or zswap, as well as hybrid setups that +have swap on faster devices than the filesystem, values beyond 100 can +be considered. For example, if the random IO against the swap device +is on average 2x faster than IO from the filesystem, swappiness should +be 133 (x + 2x = 200, 2x = 133.33). + +At 0, the kernel will not initiate swap until the amount of free and +file-backed pages is less than the high watermark in a zone. + unprivileged_userfaultfd ========================
kernel/sysctl.c renamed from kernel/sysctl.c~mm-allow-swappiness-that-prefers-reclaiming-anon-over-the-file-workingset +2 -1
@@ -131,6 +131,7 @@ static unsigned long zero_ul;static unsigned long one_ul =1 ;static unsigned long long_max =LONG_MAX ;static int one_hundred =100 ; +static int two_hundred =200 ;static int one_thousand =1000 ;#ifdef CONFIG_PRINTK static int ten_thousand =10000 ;
@@ -1391,7 +1392,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = &two_hundred, },#ifdef CONFIG_HUGETLB_PAGE {
mm/vmscan.c renamed from mm/vmscan.c~mm-allow-swappiness-that-prefers-reclaiming-anon-over-the-file-workingset +1 -1
@@ -161,7 +161,7 @@ struct scan_control {#endif /* -* From 0 .. 100. Higher means more swappy. +* From 0 .. 200. Higher means more swappy. */ int vm_swappiness =60 ;/*
_
Andrew Morton
maintainerPATCH 106/131 mm: fold and remove lru_cache_add_anon() and lru_cache_add_file()
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: fold and remove lru_cache_add_anon() and lru_cache_add_file()
They're the same function, and for the purpose of all callers they are equivalent to lru_cache_add().
---
fs/cifs/file.c | 10 ++++----- fs/fuse/dev.c | 2 - include/linux/swap.h | 2 - mm/khugepaged.c | 8 +------ mm/memory.c | 2 - mm/shmem.c | 6 ++--- mm/swap.c | 42 +++++++++-------------------------------- mm/swap_state.c | 2 - 8 files changed, 23 insertions(+), 51 deletions(-)
fs/cifs/file.c renamed from fs/cifs/file.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +5 -5
@@ -4162,7 +4162,7 @@ cifs_readv_complete(struct work_struct *for (i =0 ; i < rdata->nr_pages; i++) {struct page *page = rdata->pages[i]; -lru_cache_add_file (page); +lru_cache_add (page);if (rdata->result ==0 || (rdata->result == -EAGAIN && got_bytes)) {
@@ -4232,7 +4232,7 @@ readpages_fill_pages(struct TCP_Server_I* fill them until the writes are flushed. */ zero_user (page,0 ,PAGE_SIZE ); -lru_cache_add_file (page); +lru_cache_add (page);flush_dcache_page (page);SetPageUptodate (page);unlock_page (page);
@@ -4242,7 +4242,7 @@ readpages_fill_pages(struct TCP_Server_Icontinue ; }else {/* no need to hold page hostage */ -lru_cache_add_file (page); +lru_cache_add (page);unlock_page (page);put_page (page); rdata->pages[i] =NULL ;
@@ -4437,7 +4437,7 @@ static int cifs_readpages(struct file *f/* best to give up if we're out of mem */ list_for_each_entry_safe (page, tpage, &tmplist, lru) {list_del (&page->lru); -lru_cache_add_file (page); +lru_cache_add (page);unlock_page (page);put_page (page); }
@@ -4475,7 +4475,7 @@ static int cifs_readpages(struct file *fadd_credits_and_wake_if (server, &rdata->credits,0 );for (i =0 ; i < rdata->nr_pages; i++) { page = rdata->pages[i]; -lru_cache_add_file (page); +lru_cache_add (page);unlock_page (page);put_page (page); }
fs/fuse/dev.c renamed from fs/fuse/dev.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +1 -1
@@ -840,7 +840,7 @@ static int fuse_try_move_page(struct fusget_page (newpage);if (!(buf->flags &PIPE_BUF_FLAG_LRU )) -lru_cache_add_file (newpage); +lru_cache_add (newpage); err =0 ;spin_lock (&cs->req->waitq.lock);
include/linux/swap.h renamed from include/linux/swap.h~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +0 -2
@@ -335,8 +335,6 @@ extern unsigned long nr_free_pagecache_p/* linux/mm/swap.c */ extern void lru_cache_add (struct page *); -extern void lru_cache_add_anon (struct page *page); -extern void lru_cache_add_file (struct page *page);extern void lru_add_page_tail (struct page *page,struct page *page_tail,struct lruvec *lruvec,struct list_head *head);extern void activate_page (struct page *);
mm/khugepaged.c renamed from mm/khugepaged.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +2 -6
@@ -1879,13 +1879,9 @@ xa_unlocked:SetPageUptodate (new_page);page_ref_add (new_page,HPAGE_PMD_NR -1 ); - -if (is_shmem) { +if (is_shmem)set_page_dirty (new_page); -lru_cache_add_anon (new_page); - }else { -lru_cache_add_file (new_page); - } +lru_cache_add (new_page);/* * Remove pte page tables, so we can re-fault the page as huge.
mm/memory.c renamed from mm/memory.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +1 -1
@@ -3139,7 +3139,7 @@ vm_fault_t do_swap_page(struct vm_faultif (err) goto out_page; -lru_cache_add_anon (page); +lru_cache_add (page);swap_readpage (page, true); } }else {
mm/shmem.c renamed from mm/shmem.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +3 -3
@@ -1609,7 +1609,7 @@ static int shmem_replace_page(struct pag*/ oldpage = newpage; }else { -lru_cache_add_anon (newpage); +lru_cache_add (newpage); *pagep = newpage; }
@@ -1860,7 +1860,7 @@ alloc_nohuge: charge_mm);if (error) goto unacct; -lru_cache_add_anon (page); +lru_cache_add (page);spin_lock_irq (&info->lock); info->alloced +=compound_nr (page);
@@ -2376,7 +2376,7 @@ static int shmem_mfill_atomic_pte(structif (!pte_none (*dst_pte)) goto out_release_unlock; -lru_cache_add_anon (page); +lru_cache_add (page);spin_lock_irq (&info->lock); info->alloced++;
mm/swap.c renamed from mm/swap.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +10 -32
57 lines not on this page · raw message
mm/swap_state.c renamed from mm/swap_state.c~mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file +1 -1
@@ -442,7 +442,7 @@ struct page *__read_swap_cache_async(swp/* Caller will initiate read into locked page */ SetPageWorkingset (page); -lru_cache_add_anon (page); +lru_cache_add (page); *new_page_allocated = true;return page;
_
Andrew Morton
maintainerPATCH 107/131 mm: workingset: let cache workingset challenge anon
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: workingset: let cache workingset challenge anon
We activate cache refaults with reuse distances in pages smaller than the size of the total cache. This allows new pages with competitive access frequencies to establish themselves, as well as challenge and potentially displace pages on the active list that have gone cold.
However, that assumes that active cache can only replace other active cache in a competition for the hottest memory. This is not a great default assumption. The page cache might be thrashing while there are enough completely cold and unused anonymous pages sitting around that we'd only have to write to swap once to stop all IO from the cache.
Activate cache refaults when their reuse distance in pages is smaller than the total userspace workingset, including anonymous pages.
Reclaim can still decide how to balance pressure among the two LRUs depending on the IO situation. Rotational drives will prefer avoiding random IO from swap and go harder after cache. But fundamentally, hot cache should be able to compete with anon pages for a place in RAM.
---
mm/workingset.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)
mm/workingset.c renamed from mm/workingset.c~mm-workingset-let-cache-workingset-challenge-anon +12 -5
@@ -277,8 +277,8 @@ void workingset_refault(struct page *pagstruct mem_cgroup *eviction_memcg;struct lruvec *eviction_lruvec;unsigned long refault_distance; +unsigned long workingset_size;struct pglist_data *pgdat; -unsigned long active_file;struct mem_cgroup *memcg;unsigned long eviction;struct lruvec *lruvec;
@@ -310,7 +310,6 @@ void workingset_refault(struct page *pag goto out; eviction_lruvec =mem_cgroup_lruvec (eviction_memcg, pgdat); refault =atomic_long_read (&eviction_lruvec->inactive_age); - active_file =lruvec_page_state (eviction_lruvec,NR_ACTIVE_FILE );/* * Calculate the refault distance
@@ -345,10 +344,18 @@ void workingset_refault(struct page *pag/* * Compare the distance to the existing workingset size. We -* don't act on pages that couldn't stay resident even if all -* the memory was available to the page cache. +* don't activate pages that couldn't stay resident even if +* all the memory was available to the page cache. Whether +* cache can compete with anon or not depends on having swap. */ -if (refault_distance > active_file) + workingset_size =lruvec_page_state (eviction_lruvec,NR_ACTIVE_FILE ); +if (mem_cgroup_get_nr_swap_pages (memcg) >0 ) { + workingset_size +=lruvec_page_state (eviction_lruvec, +NR_INACTIVE_ANON ); + workingset_size +=lruvec_page_state (eviction_lruvec, +NR_ACTIVE_ANON ); + } +if (refault_distance > workingset_size) goto out;SetPageActive (page);
_
Andrew Morton
maintainerPATCH 108/131 mm: remove use-once cache bias from LRU balancing
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: remove use-once cache bias from LRU balancing
When the splitlru patches divided page cache and swap-backed pages into separate LRU lists, the pressure balance between the lists was biased to account for the fact that streaming IO can cause memory pressure with a flood of pages that are used only once. New page cache additions would tip the balance toward the file LRU, and repeat access would neutralize that bias again. This ensured that page reclaim would always go for used-once cache first.
Since e9868505987a ("mm,vmscan: only evict file pages when we have plenty"), page reclaim generally skips over swap-backed memory entirely as long as there is used-once cache present, and will apply the LRU balancing when only repeatedly accessed cache pages are left - at which point the previous use-once bias will have been neutralized. This makes the use-once cache balancing bias unnecessary.
---
mm/swap.c | 5 ----- 1 file changed, 5 deletions(-)
mm/swap.c renamed from mm/swap.c~mm-remove-use-once-cache-bias-from-lru-balancing +0 -5
@@ -293,7 +293,6 @@ static void __activate_page(struct page void *arg) {if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) { -int file =page_is_file_lru (page);int lru =page_lru_base_type (page);del_page_from_lru_list (page, lruvec, lru);
@@ -303,7 +302,6 @@ static void __activate_page(struct pagetrace_mm_lru_activate (page);__count_vm_event (PGACTIVATE ); -update_page_reclaim_stat (lruvec, file,1 ,hpage_nr_pages (page)); } }
@@ -975,9 +973,6 @@ static void __pagevec_lru_add_fn(structif (page_evictable (page)) { lru =page_lru (page); -update_page_reclaim_stat (lruvec,is_file_lru (lru), -PageActive (page), -hpage_nr_pages (page));if (was_unevictable)count_vm_event (UNEVICTABLE_PGRESCUED ); }else {
_
Andrew Morton
maintainerPATCH 109/131 mm: vmscan: drop unnecessary div0 avoidance rounding in get_scan_count()
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: vmscan: drop unnecessary div0 avoidance rounding in get_scan_count()
When we calculate the relative scan pressure between the anon and file LRU lists, we have to assume that reclaim_stat can contain zeroes. To avoid div0 crashes, we add 1 to all denominators like so:
anon_prio = swappiness;
file_prio = 200 - anon_prio;
[...]
/*
* The amount of pressure on anon vs file pages is inversely
* proportional to the fraction of recently scanned pages on
* each list that were recently referenced and in active use.
*/
ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
ap /= reclaim_stat->recent_rotated[0] + 1;
fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
fp /= reclaim_stat->recent_rotated[1] + 1;
spin_unlock_irq(&pgdat->lru_lock);
fraction[0] = ap;
fraction[1] = fp;
denominator = ap + fp + 1;While reclaim_stat can contain 0, it's not actually possible for ap + fp to be 0. One of anon_prio or file_prio could be zero, but they must still add up to 200. And the reclaim_stat fraction, due to the +1 in there, is always at least 1. So if one of the two numerators is 0, the other one can't be. ap + fp is always at least 1. Drop the + 1.
---
mm/vmscan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmscan.c renamed from mm/vmscan.c~mm-vmscan-drop-unnecessary-div0-avoidance-rounding-in-get_scan_count +1 -1
@@ -2348,7 +2348,7 @@ static void get_scan_count(struct lruvec fraction[0 ] = ap; fraction[1 ] = fp; - denominator = ap + fp +1 ; + denominator = ap + fp; out:for_each_evictable_lru (lru) {int file =is_file_lru (lru);
_
Andrew Morton
maintainerPATCH 110/131 mm: base LRU balancing on an explicit cost model
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: base LRU balancing on an explicit cost model
Currently, scan pressure between the anon and file LRU lists is balanced based on a mixture of reclaim efficiency and a somewhat vague notion of "value" of having certain pages in memory over others. That concept of value is problematic, because it has caused us to count any event that remotely makes one LRU list more or less preferrable for reclaim, even when these events are not directly comparable and impose very different costs on the system. One example is referenced file pages that we still deactivate and referenced anonymous pages that we actually rotate back to the head of the list.
There is also conceptual overlap with the LRU algorithm itself. By rotating recently used pages instead of reclaiming them, the algorithm already biases the applied scan pressure based on page value. Thus, when rebalancing scan pressure due to rotations, we should think of reclaim cost, and leave assessing the page value to the LRU algorithm.
Lastly, considering both value-increasing as well as value-decreasing events can sometimes cause the same type of event to be counted twice, i.e. how rotating a page increases the LRU value, while reclaiming it succesfully decreases the value. In itself this will balance out fine, but it quietly skews the impact of events that are only recorded once.
The abstract metric of "value", the murky relationship with the LRU algorithm, and accounting both negative and positive events make the current pressure balancing model hard to reason about and modify.
This patch switches to a balancing model of accounting the concrete, actually observed cost of reclaiming one LRU over another. For now, that cost includes pages that are scanned but rotated back to the list head. Subsequent patches will add consideration for IO caused by refaulting of recently evicted pages.
Replace struct zone_reclaim_stat with two cost counters in the lruvec, and make everything that affects cost go through a new lru_note_cost() function.
---
include/linux/mmzone.h | 21 ++++++------------ include/linux/swap.h | 2 + mm/memcontrol.c | 18 +++++---------- mm/swap.c | 21 +++++++----------- mm/vmscan.c | 44 +++++++++++++++++++-------------------- 5 files changed, 46 insertions(+), 60 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-base-lru-balancing-on-an-explicit-cost-model +7 -14
@@ -242,19 +242,6 @@ static inline bool is_active_lru(enum lrreturn (lru ==LRU_ACTIVE_ANON || lru ==LRU_ACTIVE_FILE ); } -struct zone_reclaim_stat { -/* -* The pageout code in vmscan.c keeps track of how many of the -* mem/swap backed and file backed pages are referenced. -* The higher the rotated/scanned ratio, the more valuable -* that cache is. -* -* The anon LRU stats live in [0], file LRU stats in [1] -*/ -unsigned long recent_rotated[2 ]; -unsigned long recent_scanned[2 ]; -}; -enum lruvec_flags {LRUVEC_CONGESTED ,/* lruvec has many dirty pages * backed by a congested BDI
@@ -263,7 +250,13 @@ enum lruvec_flags {struct lruvec {struct list_head lists[NR_LRU_LISTS ]; -struct zone_reclaim_stat reclaim_stat; +/* +* These track the cost of reclaiming one LRU - file or anon - +* over the other. As the observed cost of reclaiming one LRU +* increases, the reclaim scan balance tips toward the other. +*/ +unsigned long anon_cost; +unsigned long file_cost;/* Evictions & activations on the inactive file list */ atomic_long_t inactive_age;/* Refaults at the time of last reclaim cycle */
include/linux/swap.h renamed from include/linux/swap.h~mm-base-lru-balancing-on-an-explicit-cost-model +2 -0
@@ -334,6 +334,8 @@ extern unsigned long nr_free_pagecache_p/* linux/mm/swap.c */ +extern void lru_note_cost (struct lruvec *lruvec,bool file, +unsigned int nr_pages);extern void lru_cache_add (struct page *);extern void lru_add_page_tail (struct page *page,struct page *page_tail,struct lruvec *lruvec,struct list_head *head);
mm/memcontrol.c renamed from mm/memcontrol.c~mm-base-lru-balancing-on-an-explicit-cost-model +6 -12
@@ -3853,23 +3853,17 @@ static int memcg_stat_show(struct seq_fi {pg_data_t *pgdat;struct mem_cgroup_per_node *mz; -struct zone_reclaim_stat *rstat; -unsigned long recent_rotated[2 ] = {0 ,0 }; -unsigned long recent_scanned[2 ] = {0 ,0 }; +unsigned long anon_cost =0 ; +unsigned long file_cost =0 ;for_each_online_pgdat (pgdat) { mz =mem_cgroup_nodeinfo (memcg, pgdat->node_id); - rstat = &mz->lruvec.reclaim_stat; - recent_rotated[0 ] += rstat->recent_rotated[0 ]; - recent_rotated[1 ] += rstat->recent_rotated[1 ]; - recent_scanned[0 ] += rstat->recent_scanned[0 ]; - recent_scanned[1 ] += rstat->recent_scanned[1 ]; + anon_cost += mz->lruvec.anon_cost; + file_cost += mz->lruvec.file_cost; } -seq_printf (m,"recent_rotated_anon %lu\n" , recent_rotated[0 ]); -seq_printf (m,"recent_rotated_file %lu\n" , recent_rotated[1 ]); -seq_printf (m,"recent_scanned_anon %lu\n" , recent_scanned[0 ]); -seq_printf (m,"recent_scanned_file %lu\n" , recent_scanned[1 ]); +seq_printf (m,"anon_cost %lu\n" , anon_cost); +seq_printf (m,"file_cost %lu\n" , file_cost); }#endif
mm/swap.c renamed from mm/swap.c~mm-base-lru-balancing-on-an-explicit-cost-model +9 -12
@@ -278,15 +278,12 @@ void rotate_reclaimable_page(struct page } } -static void update_page_reclaim_stat (struct lruvec *lruvec, -int file,int rotated, -unsigned int nr_pages) -{ -struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; - - reclaim_stat->recent_scanned[file] += nr_pages; -if (rotated) - reclaim_stat->recent_rotated[file] += nr_pages; +void lru_note_cost (struct lruvec *lruvec,bool file,unsigned int nr_pages) +{ +if (file) + lruvec->file_cost += nr_pages; +else + lruvec->anon_cost += nr_pages; } static void __activate_page(struct page *page, struct lruvec *lruvec,
@@ -541,7 +538,7 @@ static void lru_deactivate_file_fn(strucif (active)__count_vm_event (PGDEACTIVATE ); -update_page_reclaim_stat (lruvec, file,0 ,hpage_nr_pages (page)); +lru_note_cost (lruvec, !file,hpage_nr_pages (page)); } static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
@@ -557,7 +554,7 @@ static void lru_deactivate_fn(struct pagadd_page_to_lru_list (page, lruvec, lru);__count_vm_events (PGDEACTIVATE ,hpage_nr_pages (page)); -update_page_reclaim_stat (lruvec, file,0 ,hpage_nr_pages (page)); +lru_note_cost (lruvec, !file,hpage_nr_pages (page)); } }
@@ -582,7 +579,7 @@ static void lru_lazyfree_fn(struct page__count_vm_events (PGLAZYFREE ,hpage_nr_pages (page));count_memcg_page_event (page,PGLAZYFREE ); -update_page_reclaim_stat (lruvec,1 ,0 ,hpage_nr_pages (page)); +lru_note_cost (lruvec,0 ,hpage_nr_pages (page)); } }
mm/vmscan.c renamed from mm/vmscan.c~mm-base-lru-balancing-on-an-explicit-cost-model +22 -22
104 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 111/131 mm: deactivations shouldn't bias the LRU balance
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: deactivations shouldn't bias the LRU balance
Operations like MADV_FREE, FADV_DONTNEED etc. currently move any affected active pages to the inactive list to accelerate their reclaim (good) but also steer page reclaim toward that LRU type, or away from the other (bad).
The reason why this is undesirable is that such operations are not part of the regular page aging cycle, and rather a fluke that doesn't say much about the remaining pages on that list; they might all be in heavy use, and once the chunk of easy victims has been purged, the VM continues to apply elevated pressure on those remaining hot pages. The other LRU, meanwhile, might have easily reclaimable pages, and there was never a need to steer away from it in the first place.
As the previous patch outlined, we should focus on recording actually observed cost to steer the balance rather than speculating about the potential value of one LRU list over the other. In that spirit, leave explicitely deactivated pages to the LRU algorithm to pick up, and let rotations decide which list is the easiest to reclaim.
---
mm/swap.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-)
mm/swap.c renamed from mm/swap.c~mm-deactivations-shouldnt-bias-the-lru-balance +1 -6
@@ -498,7 +498,7 @@ void lru_cache_add_active_or_unevictablestatic void lru_deactivate_file_fn (struct page *page,struct lruvec *lruvec,void *arg) { -int lru, file; +int lru;bool active;if (!PageLRU (page))
@@ -512,7 +512,6 @@ static void lru_deactivate_file_fn(strucreturn ; active =PageActive (page); - file =page_is_file_lru (page); lru =page_lru_base_type (page);del_page_from_lru_list (page, lruvec, lru + active);
@@ -538,14 +537,12 @@ static void lru_deactivate_file_fn(strucif (active)__count_vm_event (PGDEACTIVATE ); -lru_note_cost (lruvec, !file,hpage_nr_pages (page)); }static void lru_deactivate_fn (struct page *page,struct lruvec *lruvec,void *arg) {if (PageLRU (page) &&PageActive (page) && !PageUnevictable (page)) { -int file =page_is_file_lru (page);int lru =page_lru_base_type (page);del_page_from_lru_list (page, lruvec, lru +LRU_ACTIVE );
@@ -554,7 +551,6 @@ static void lru_deactivate_fn(struct pagadd_page_to_lru_list (page, lruvec, lru);__count_vm_events (PGDEACTIVATE ,hpage_nr_pages (page)); -lru_note_cost (lruvec, !file,hpage_nr_pages (page)); } }
@@ -579,7 +575,6 @@ static void lru_lazyfree_fn(struct page__count_vm_events (PGLAZYFREE ,hpage_nr_pages (page));count_memcg_page_event (page,PGLAZYFREE ); -lru_note_cost (lruvec,0 ,hpage_nr_pages (page)); } }
_
Andrew Morton
maintainerPATCH 112/131 mm: only count actual rotations as LRU reclaim cost
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: only count actual rotations as LRU reclaim cost
When shrinking the active file list we rotate referenced pages only when they're in an executable mapping. The others get deactivated. When it comes to balancing scan pressure, though, we count all referenced pages as rotated, even the deactivated ones. Yet they do not carry the same cost to the system: the deactivated page *might* refault later on, but the deactivation is tangible progress toward freeing pages; rotations on the other hand cost time and effort without getting any closer to freeing memory.
Don't treat both events as equal. The following patch will hook up LRU balancing to cache and anon refaults, which are a much more concrete cost signal for reclaiming one list over the other. Thus, remove the maybe-IO cost bias from page references, and only note the CPU cost for actual rotations that prevent the pages from getting reclaimed.
---
mm/vmscan.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
mm/vmscan.c renamed from mm/vmscan.c~mm-only-count-actual-rotations-as-lru-reclaim-cost +3 -5
@@ -2054,7 +2054,6 @@ static void shrink_active_list(unsignedif (page_referenced (page,0 , sc->target_mem_cgroup, &vm_flags)) { - nr_rotated +=hpage_nr_pages (page);/* * Identify referenced, file-backed active pages and * give them one more trip around the active list. So
@@ -2065,6 +2064,7 @@ static void shrink_active_list(unsigned* so we ignore them here. */ if ((vm_flags &VM_EXEC ) &&page_is_file_lru (page)) { + nr_rotated +=hpage_nr_pages (page);list_add (&page->lru, &l_active);continue ; }
@@ -2080,10 +2080,8 @@ static void shrink_active_list(unsigned*/ spin_lock_irq (&pgdat->lru_lock);/* -* Count referenced pages from currently used mappings as rotated, -* even though only some of them are actually re-activated. This -* helps balance scan pressure between file and anonymous pages in -* get_scan_count. +* Rotating pages costs CPU without actually +* progressing toward the reclaim goal. */ lru_note_cost (lruvec, file, nr_rotated);
_
Andrew Morton
maintainerPATCH 113/131 mm: balance LRU lists based on relative thrashing
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: balance LRU lists based on relative thrashing
Since the LRUs were split into anon and file lists, the VM has been balancing between page cache and anonymous pages based on per-list ratios of scanned vs. rotated pages. In most cases that tips page reclaim towards the list that is easier to reclaim and has the fewest actively used pages, but there are a few problems with it:
1. Refaults and LRU rotations are weighted the same way, even though one costs IO and the other costs a bit of CPU.
2. The less we scan an LRU list based on already observed rotations, the more we increase the sampling interval for new references, and rotations become even more likely on that list. This can enter a death spiral in which we stop looking at one list completely until the other one is all but annihilated by page reclaim.
Since commit a528910e12ec ("mm: thrash detection-based file cache sizing") we have refault detection for the page cache. Along with swapin events, they are good indicators of when the file or anon list, respectively, is too small for its workingset and needs to grow.
For example, if the page cache is thrashing, the cache pages need more time in memory, while there may be colder pages on the anonymous list. Likewise, if swapped pages are faulting back in, it indicates that we reclaim anonymous pages too aggressively and should back off.
Replace LRU rotations with refaults and swapins as the basis for relative reclaim cost of the two LRUs. This will have the VM target list balances that incur the least amount of IO on aggregate.
---
include/linux/swap.h | 3 +-- mm/swap.c | 11 +++++++---- mm/swap_state.c | 5 +++++ mm/vmscan.c | 39 ++++++++++----------------------------- mm/workingset.c | 4 ++++ 5 files changed, 27 insertions(+), 35 deletions(-)
include/linux/swap.h renamed from include/linux/swap.h~mm-balance-lru-lists-based-on-relative-thrashing +1 -2
@@ -334,8 +334,7 @@ extern unsigned long nr_free_pagecache_p/* linux/mm/swap.c */ -extern void lru_note_cost (struct lruvec *lruvec,bool file, -unsigned int nr_pages); +extern void lru_note_cost (struct page *);extern void lru_cache_add (struct page *);extern void lru_add_page_tail (struct page *page,struct page *page_tail,struct lruvec *lruvec,struct list_head *head);
mm/swap.c renamed from mm/swap.c~mm-balance-lru-lists-based-on-relative-thrashing +7 -4
@@ -278,12 +278,15 @@ void rotate_reclaimable_page(struct page } } -void lru_note_cost (struct lruvec *lruvec,bool file,unsigned int nr_pages) +void lru_note_cost (struct page *page) { -if (file) - lruvec->file_cost += nr_pages; +struct lruvec *lruvec =mem_cgroup_page_lruvec (page,page_pgdat (page)); + +/* Record new data point */ +if (page_is_file_lru (page)) + lruvec->file_cost++;else - lruvec->anon_cost += nr_pages; + lruvec->anon_cost++; } static void __activate_page(struct page *page, struct lruvec *lruvec,
mm/swap_state.c renamed from mm/swap_state.c~mm-balance-lru-lists-based-on-relative-thrashing +5 -0
@@ -440,6 +440,11 @@ struct page *__read_swap_cache_async(swp goto fail_unlock; } +/* XXX: Move to lru_cache_add() when it supports new vs putback */ +spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock); +/* Caller will initiate read into locked page */ SetPageWorkingset (page);lru_cache_add (page);
mm/vmscan.c renamed from mm/vmscan.c~mm-balance-lru-lists-based-on-relative-thrashing +10 -29
@@ -1958,12 +1958,6 @@ shrink_inactive_list(unsigned long nr_tomove_pages_to_lru (lruvec, &page_list);__mod_node_page_state (pgdat,NR_ISOLATED_ANON + file, -nr_taken); -/* -* Rotating pages costs CPU without actually -* progressing toward the reclaim goal. -*/ -lru_note_cost (lruvec,0 , stat.nr_activate[0 ]); -lru_note_cost (lruvec,1 , stat.nr_activate[1 ]); item =current_is_kswapd () ?PGSTEAL_KSWAPD :PGSTEAL_DIRECT ;if (!cgroup_reclaim (sc))__count_vm_events (item, nr_reclaimed);
@@ -2079,11 +2073,6 @@ static void shrink_active_list(unsigned* Move pages back to the lru list. */ spin_lock_irq (&pgdat->lru_lock); -/* -* Rotating pages costs CPU without actually -* progressing toward the reclaim goal. -*/ -lru_note_cost (lruvec, file, nr_rotated); nr_activate =move_pages_to_lru (lruvec, &l_active); nr_deactivate =move_pages_to_lru (lruvec, &l_inactive);
@@ -2298,22 +2287,23 @@ static void get_scan_count(struct lruvec scan_balance =SCAN_FRACT ;/* -* With swappiness at 100, anonymous and file have the same priority. -* This scanning priority is essentially the inverse of IO cost. +* Calculate the pressure balance between anon and file pages. +* +* The amount of pressure we put on each LRU is inversely +* proportional to the cost of reclaiming each list, as +* determined by the share of pages that are refaulting, times +* the relative IO cost of bringing back a swapped out +* anonymous page vs reloading a filesystem page (swappiness). +* +* With swappiness at 100, anon and file have equal IO cost. */ anon_prio = swappiness; file_prio =200 - anon_prio;/* -* OK, so we have swap space and a fair amount of page cache -* pages. We use the recently rotated / recently scanned -* ratios to determine how valuable each cache is. -* * Because workloads change over time (and to avoid overflow) * we keep these statistics as a floating average, which ends -* up weighing recent references more than old ones. -* -* anon in [0], file in [1] +* up weighing recent refaults more than old ones. */ anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
@@ -2328,15 +2318,6 @@ static void get_scan_count(struct lruvec lruvec->file_cost /=2 ; totalcost /=2 ; } - -/* -* The amount of pressure on anon vs file pages is inversely -* proportional to the assumed cost of reclaiming each list, -* as determined by the share of pages that are likely going -* to refault or rotate on each list (recently referenced), -* times the relative IO cost of bringing back a swapped out -* anonymous page vs reloading a filesystem page (swappiness). -*/ ap = anon_prio * (totalcost +1 ); ap /= lruvec->anon_cost +1 ;
mm/workingset.c renamed from mm/workingset.c~mm-balance-lru-lists-based-on-relative-thrashing +4 -0
@@ -365,6 +365,10 @@ void workingset_refault(struct page *pag/* Page was active prior to eviction */ if (workingset) {SetPageWorkingset (page); +/* XXX: Move to lru_cache_add() when it supports new vs putback */ +spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock);inc_lruvec_state (lruvec,WORKINGSET_RESTORE ); } out:
_
Re: [patch 113/131] mm: balance LRU lists based on relative thrashing
Review
Quoted 2 lines · Andrew Morton
(quoted diff) +1
-0+ /* XXX: Move to lru_cache_add() when it supports new vs putback */
Hi Hannes,
Sorry for a bit lost, would you like to explain a bit more of your idea here?
mm/swap_state.c struct page *__read_swap_cache_async(swp · line 447
Quoted 4 lines
mm/swap_state.c renamed from mm/swap_state.c~mm-balance-lru-lists-based-on-relative-thrashing +4
-0+spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock); +
What could we see here w/o the lru_lock?
Thanks
Alex
Johannes Weiner
maintainerRe: [patch 113/131] mm: balance LRU lists based on relative thrashing
Quoted 10 lines · Alex Shi
Review
Quoted 2 lines · Andrew Morton
(quoted diff) +1
-0+ /* XXX: Move to lru_cache_add() when it supports new vs putback */Hi Hannes,
Sorry for a bit lost, would you like to explain a bit more of your idea here?
Quoted 7 lines
mm/swap_state.c struct page *__read_swap_cache_async(swp · line 447
Quoted 4 lines
mm/swap_state.c renamed from mm/swap_state.c~mm-balance-lru-lists-based-on-relative-thrashing +4
-0+spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock); +What could we see here w/o the
lru_lock?
It'll just be part of the existing LRU locking in pagevec_lru_move_fn(), when the new pages are added to the LRU in batch. See this older patch for example:
https://lore.kernel.org/linux-mm/[email protected]/
I didn't include it in this series to reduce conflict with Joonsoo's WIP series that also operates in this area and does something similar:
Re: [patch 113/131] mm: balance LRU lists based on relative thrashing
2020년 6월 9일 (화) 오후 11:46, Johannes Weiner <[email protected]>님이 작성:
Quoted 20 lines · Alex Shi
Quoted 10 lines · Andrew Morton
Quoted 2 lines
+ /* XXX: Move to
lru_cache_add()when it supports new vs putback */Hi Hannes,
Sorry for a bit lost, would you like to explain a bit more of your idea here?
Quoted 7 lines
mm/swap_state.c struct page *__read_swap_cache_async(swp · line 447
Quoted 4 lines
mm/swap_state.c renamed from mm/swap_state.c~mm-balance-lru-lists-based-on-relative-thrashing +4
-0+spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock); +What could we see here w/o the
lru_lock?
Quoted 8 lines
It'll just be part of the existing LRU locking in
pagevec_lru_move_fn(), when the new pages are added to the LRU in batch. See this older patch for example:https://lore.kernel.org/linux-mm/[email protected]/
I didn't include it in this series to reduce conflict with Joonsoo's WIP series that also operates in this area and does something similar:
Thanks!
Quoted 1 line
I haven't completed the rebase of my series but I guess that referenced patch "https://lkml.org/lkml/2020/4/3/63" would be removed in the next version.
Before the I/O cost model, a new anonymous page contributes to the LRU reclaim balance. But, now, a new anonymous page doesn't contributes to the I/O cost so this adjusting patch would not be needed anymore.
If anyone wants to change this part,
"/* XXX: Move to lru_cache_add() when it supports new vs putback */", feel free to do it.
Thanks.
Re: [patch 113/131] mm: balance LRU lists based on relative thrashing
Quoted 20 lines · Joonsoo Kim
2020년 6월 9일 (화) 오후 11:46, Johannes Weiner <[email protected]>님이 작성:
Quoted 19 lines · Alex Shi
Quoted 17 lines · Andrew Morton
Quoted 2 lines
+ /* XXX: Move to
lru_cache_add()when it supports new vs putback */Hi Hannes,
Sorry for a bit lost, would you like to explain a bit more of your idea here?
mm/swap_state.c struct page *__read_swap_cache_async(swp · line 447
Quoted 4 lines
mm/swap_state.c renamed from mm/swap_state.c~mm-balance-lru-lists-based-on-relative-thrashing +4
-0+spin_lock_irq (&page_pgdat (page)->lru_lock); +lru_note_cost (page); +spin_unlock_irq (&page_pgdat (page)->lru_lock); +What could we see here w/o the
lru_lock?
Why I want to know the lru_lock protection here is that currently we have 5 lru lists but only guarded by one lock, that would cause much contention when different apps active on a server.
I guess originally we have only one lru_lock, since 5 locks would cause cacheline bouncing if we put them together, or a bit cacheline waste to separate them in cacheline. But after we have qspinlock, each of cpu will just loop lock on their cacheline, no interfer to others. It would much much relief the performance drop by cacheline bounce.
And we could use page.mapping bits to store the using lru list index for the page. As a quick thought, I guess, except the 5 locks for 5 lists, we still need 1 more lock for common lruvec data or for others which relay on lru_lock now, like mlock, hpage_nr_pages.. That's the reason I want to know everything under lru_lock. :)
Any comments for this idea? :)
Thanks
Alex
Quoted 10 lines
Quoted 9 lines
It'll just be part of the existing LRU locking in
pagevec_lru_move_fn(), when the new pages are added to the LRU in batch. See this older patch for example:https://lore.kernel.org/linux-mm/[email protected]/
I didn't include it in this series to reduce conflict with Joonsoo's WIP series that also operates in this area and does something similar:
Quoted 6 lines
Thanks!
Quoted 1 line
I haven't completed the rebase of my series but I guess that referenced patch "https://lkml.org/lkml/2020/4/3/63" would be removed in the next version.
Thanks a lot for the info, Johannes&Joonsoo! A long history for a interesting idea. :)
Quoted 8 lines
Before the I/O cost model, a new anonymous page contributes to the LRU reclaim balance. But, now, a new anonymous page doesn't contributes to the I/O cost so this adjusting patch would not be needed anymore.
If anyone wants to change this part,
"/* XXX: Move tolru_cache_add()when it supports new vs putback */", feel free to do it.
Andrew Morton
maintainerPATCH 114/131 mm: vmscan: determine anon/file pressure balance at the reclaim root
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: vmscan: determine anon/file pressure balance at the reclaim root
We split the LRU lists into anon and file, and we rebalance the scan pressure between them when one of them begins thrashing: if the file cache experiences workingset refaults, we increase the pressure on anonymous pages; if the workload is stalled on swapins, we increase the pressure on the file cache instead.
With cgroups and their nested LRU lists, we currently don't do this correctly. While recursive cgroup reclaim establishes a relative LRU order among the pages of all involved cgroups, LRU pressure balancing is done on an individual cgroup LRU level. As a result, when one cgroup is thrashing on the filesystem cache while a sibling may have cold anonymous pages, pressure doesn't get equalized between them.
This patch moves LRU balancing decision to the root of reclaim - the same level where the LRU order is established.
It does this by tracking LRU cost recursively, so that every level of the cgroup tree knows the aggregate LRU cost of all memory within its domain. When the page scanner calculates the scan balance for any given individual cgroup's LRU list, it uses the values from the ancestor cgroup that initiated the reclaim cycle.
If one sibling is then thrashing on the cache, it will tip the pressure balance inside its ancestors, and the next hierarchical reclaim iteration will go more after the anon pages in the tree.
---
include/linux/memcontrol.h | 13 +++++++++++ mm/swap.c | 32 +++++++++++++++++++++++---- mm/vmscan.c | 41 ++++++++++++++--------------------- 3 files changed, 57 insertions(+), 29 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-vmscan-determine-anon-file-pressure-balance-at-the-reclaim-root +13 -0
@@ -1242,6 +1242,19 @@ static inline void dec_lruvec_page_statemod_lruvec_page_state (page, idx,-1 ); } +static inline struct lruvec *parent_lruvec (struct lruvec *lruvec) +{ +struct mem_cgroup *memcg; + + memcg =lruvec_memcg (lruvec); +if (!memcg) +return NULL ; + memcg =parent_mem_cgroup (memcg); +if (!memcg) +return NULL ; +return mem_cgroup_lruvec (memcg,lruvec_pgdat (lruvec)); +} +#ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain *mem_cgroup_wb_domain (struct bdi_writeback *wb);
mm/swap.c renamed from mm/swap.c~mm-vmscan-determine-anon-file-pressure-balance-at-the-reclaim-root +27 -5
@@ -282,11 +282,33 @@ void lru_note_cost(struct page *page) {struct lruvec *lruvec =mem_cgroup_page_lruvec (page,page_pgdat (page)); -/* Record new data point */ -if (page_is_file_lru (page)) - lruvec->file_cost++; -else - lruvec->anon_cost++; +do { +unsigned long lrusize; + +/* Record cost event */ +if (page_is_file_lru (page)) + lruvec->file_cost++; +else + lruvec->anon_cost++; + +/* +* Decay previous events +* +* Because workloads change over time (and to avoid +* overflow) we keep these statistics as a floating +* average, which ends up weighing recent refaults +* more than old ones. +*/ + lrusize =lruvec_page_state (lruvec,NR_INACTIVE_ANON ) + +lruvec_page_state (lruvec,NR_ACTIVE_ANON ) + +lruvec_page_state (lruvec,NR_INACTIVE_FILE ) + +lruvec_page_state (lruvec,NR_ACTIVE_FILE ); + +if (lruvec->file_cost + lruvec->anon_cost > lrusize /4 ) { + lruvec->file_cost /=2 ; + lruvec->anon_cost /=2 ; + } + }while ((lruvec =parent_lruvec (lruvec))); } static void __activate_page(struct page *page, struct lruvec *lruvec,
mm/vmscan.c renamed from mm/vmscan.c~mm-vmscan-determine-anon-file-pressure-balance-at-the-reclaim-root +17 -24
76 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 115/131 mm: vmscan: reclaim writepage is IO cost
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: vmscan: reclaim writepage is IO cost
The VM tries to balance reclaim pressure between anon and file so as to reduce the amount of IO incurred due to the memory shortage. It already counts refaults and swapins, but in addition it should also count writepage calls during reclaim.
For swap, this is obvious: it's IO that wouldn't have occurred if the anonymous memory hadn't been under memory pressure. From a relative balancing point of view this makes sense as well: even if anon is cold and reclaimable, a cache that isn't thrashing may have equally cold pages that don't require IO to reclaim.
For file writeback, it's trickier: some of the reclaim writepage IO would have likely occurred anyway due to dirty expiration. But not all of it - premature writeback reduces batching and generates additional writes. Since the flushers are already woken up by the time the VM starts writing cache pages one by one, let's assume that we'e likely causing writes that wouldn't have happened without memory pressure. In addition, the per-page cost of IO would have probably been much cheaper if written in larger batches from the flusher thread rather than the single-page-writes from kswapd.
For our purposes - getting the trend right to accelerate convergence on a stable state that doesn't require paging at all - this is sufficiently accurate. If we later wanted to optimize for sustained thrashing, we can still refine the measurements.
Count all writepage calls from kswapd as IO cost toward the LRU that the page belongs to.
Why do this dynamically? Don't we know in advance that anon pages require IO to reclaim, and so could build in a static bias?
First, scanning is not the same as reclaiming. If all the anon pages are referenced, we may not swap for a while just because we're scanning the anon list. During this time, however, it's important that we age anonymous memory and the page cache at the same rate so that their hot-cold gradients are comparable. Everything else being equal, we still want to reclaim the coldest memory overall.
Second, we keep copies in swap unless the page changes. If there is swap-backed data that's mostly read (tmpfs file) and has been swapped out before, we can reclaim it without incurring additional IO.
---
include/linux/swap.h | 4 +++- include/linux/vmstat.h | 1 + mm/swap.c | 16 ++++++++++------ mm/swap_state.c | 2 +- mm/vmscan.c | 3 +++ mm/workingset.c | 2 +- 6 files changed, 19 insertions(+), 9 deletions(-)
include/linux/swap.h renamed from include/linux/swap.h~mm-vmscan-reclaim-writepage-is-io-cost +3 -1
@@ -334,7 +334,9 @@ extern unsigned long nr_free_pagecache_p/* linux/mm/swap.c */ -extern void lru_note_cost (struct page *); +extern void lru_note_cost (struct lruvec *lruvec,bool file, +unsigned int nr_pages); +extern void lru_note_cost_page (struct page *);extern void lru_cache_add (struct page *);extern void lru_add_page_tail (struct page *page,struct page *page_tail,struct lruvec *lruvec,struct list_head *head);
include/linux/vmstat.h renamed from include/linux/vmstat.h~mm-vmscan-reclaim-writepage-is-io-cost +1 -0
@@ -26,6 +26,7 @@ struct reclaim_stat {unsigned nr_congested;unsigned nr_writeback;unsigned nr_immediate; +unsigned nr_pageout;unsigned nr_activate[2 ];unsigned nr_ref_keep;unsigned nr_unmap_fail;
mm/swap.c renamed from mm/swap.c~mm-vmscan-reclaim-writepage-is-io-cost +10 -6
@@ -278,18 +278,16 @@ void rotate_reclaimable_page(struct page } } -void lru_note_cost (struct page *page) +void lru_note_cost (struct lruvec *lruvec,bool file,unsigned int nr_pages) { -struct lruvec *lruvec =mem_cgroup_page_lruvec (page,page_pgdat (page)); - do { unsigned long lrusize; /* Record cost event */ - if (page_is_file_lru(page)) - lruvec->file_cost++; + if (file) + lruvec->file_cost += nr_pages; else - lruvec->anon_cost++; + lruvec->anon_cost += nr_pages; /* * Decay previous events
@@ -311,6 +309,12 @@ void lru_note_cost(struct page *page) }while ((lruvec =parent_lruvec (lruvec))); } +void lru_note_cost_page (struct page *page) +{ +lru_note_cost (mem_cgroup_page_lruvec (page,page_pgdat (page)), +page_is_file_lru (page),hpage_nr_pages (page)); +} +static void __activate_page (struct page *page,struct lruvec *lruvec,void *arg) {
mm/swap_state.c renamed from mm/swap_state.c~mm-vmscan-reclaim-writepage-is-io-cost +1 -1
@@ -442,7 +442,7 @@ struct page *__read_swap_cache_async(swp/* XXX: Move to lru_cache_add() when it supports new vs putback */ spin_lock_irq (&page_pgdat (page)->lru_lock); -lru_note_cost (page); +lru_note_cost_page (page);spin_unlock_irq (&page_pgdat (page)->lru_lock);/* Caller will initiate read into locked page */
mm/vmscan.c renamed from mm/vmscan.c~mm-vmscan-reclaim-writepage-is-io-cost +3 -0
@@ -1359,6 +1359,8 @@ static unsigned int shrink_page_list(strcase PAGE_ACTIVATE : goto activate_locked;case PAGE_SUCCESS : + stat->nr_pageout +=hpage_nr_pages (page); +if (PageWriteback (page)) goto keep;if (PageDirty (page))
@@ -1964,6 +1966,7 @@ shrink_inactive_list(unsigned long nr_tomove_pages_to_lru (lruvec, &page_list);__mod_node_page_state (pgdat,NR_ISOLATED_ANON + file, -nr_taken); +lru_note_cost (lruvec, file, stat.nr_pageout); item =current_is_kswapd () ?PGSTEAL_KSWAPD :PGSTEAL_DIRECT ;if (!cgroup_reclaim (sc))__count_vm_events (item, nr_reclaimed);
mm/workingset.c renamed from mm/workingset.c~mm-vmscan-reclaim-writepage-is-io-cost +1 -1
@@ -367,7 +367,7 @@ void workingset_refault(struct page *pagSetPageWorkingset (page);/* XXX: Move to lru_cache_add() when it supports new vs putback */ spin_lock_irq (&page_pgdat (page)->lru_lock); -lru_note_cost (page); +lru_note_cost_page (page);spin_unlock_irq (&page_pgdat (page)->lru_lock);inc_lruvec_state (lruvec,WORKINGSET_RESTORE ); }
_
Andrew Morton
maintainerPATCH 116/131 mm: vmscan: limit the range of LRU type balancing
In reply to Andrew MortonFrom: Johannes Weiner <[email protected]>
Subject: mm: vmscan: limit the range of LRU type balancing
When LRU cost only shows up on one list, we abruptly stop scanning that list altogether. That's an extreme reaction: by the time the other list starts thrashing and the pendulum swings back, we may have no recent age information on the first list anymore, and we could have significant latencies until the scanner has caught up.
Soften this change in the feedback system by ensuring that no list receives less than a third of overall pressure, and only distribute the other 66% according to LRU cost. This ensures that we maintain a minimum rate of aging on the entire workingset while it's being pressured, while still allowing a generous rate of convergence when the relative sizes of the lists need to adjust.
---
mm/vmscan.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)
mm/vmscan.c renamed from mm/vmscan.c~mm-vmscan-limit-the-range-of-lru-type-balancing +13 -9
@@ -2237,12 +2237,11 @@ static void get_scan_count(struct lruvec unsigned long *nr) {struct mem_cgroup *memcg =lruvec_memcg (lruvec); +unsigned long anon_cost, file_cost, total_cost;int swappiness =mem_cgroup_swappiness (memcg);u64 fraction[2 ];u64 denominator =0 ;/* gcc */ -unsigned long anon_prio, file_prio;enum scan_balance scan_balance; -unsigned long totalcost;unsigned long ap, fp;enum lru_list lru;
@@ -2301,17 +2300,22 @@ static void get_scan_count(struct lruvec* the relative IO cost of bringing back a swapped out * anonymous page vs reloading a filesystem page (swappiness). * +* Although we limit that influence to ensure no list gets +* left behind completely: at least a third of the pressure is +* applied, before swappiness. +* * With swappiness at 100, anon and file have equal IO cost. */ - anon_prio = swappiness; - file_prio =200 - anon_prio; + total_cost = sc->anon_cost + sc->file_cost; + anon_cost = total_cost + sc->anon_cost; + file_cost = total_cost + sc->file_cost; + total_cost = anon_cost + file_cost; - totalcost = sc->anon_cost + sc->file_cost; - ap = anon_prio * (totalcost +1 ); - ap /= sc->anon_cost +1 ; + ap = swappiness * (total_cost +1 ); + ap /= anon_cost +1 ; - fp = file_prio * (totalcost +1 ); - fp /= sc->file_cost +1 ; + fp = (200 - swappiness) * (total_cost +1 ); + fp /= file_cost +1 ; fraction[0 ] = ap; fraction[1 ] = fp;
_
Andrew Morton
maintainerPATCH 117/131 mm: swap: fix vmstats for huge pages
In reply to Andrew MortonFrom: Shakeel Butt <[email protected]>
Subject: mm: swap: fix vmstats for huge pages
Many of the callbacks called by pagevec_lru_move_fn() does not correctly update the vmstats for huge pages. Fix that. Also __pagevec_lru_add_fn() use the irq-unsafe alternative to update the stat as the irqs are already disabled.
---
mm/swap.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-)
mm/swap.c renamed from mm/swap.c~mm-swap-fix-vmstats-for-huge-pages +8 -6
@@ -241,7 +241,7 @@ static void pagevec_move_tail_fn(structdel_page_from_lru_list (page, lruvec,page_lru (page));ClearPageActive (page);add_page_to_lru_list_tail (page, lruvec,page_lru (page)); - (*pgmoved)++; + (*pgmoved) +=hpage_nr_pages (page); } }
@@ -327,7 +327,7 @@ static void __activate_page(struct pageadd_page_to_lru_list (page, lruvec, lru);trace_mm_lru_activate (page); -__count_vm_event (PGACTIVATE ); +__count_vm_events (PGACTIVATE ,hpage_nr_pages (page)); } }
@@ -529,6 +529,7 @@ static void lru_deactivate_file_fn(struc {int lru;bool active; +int nr_pages =hpage_nr_pages (page);if (!PageLRU (page))return ;
@@ -561,11 +562,11 @@ static void lru_deactivate_file_fn(struc* We moves tha page into tail of inactive. */ add_page_to_lru_list_tail (page, lruvec, lru); -__count_vm_event (PGROTATED ); +__count_vm_events (PGROTATED , nr_pages); }if (active) -__count_vm_event (PGDEACTIVATE ); +__count_vm_events (PGDEACTIVATE , nr_pages); } static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
@@ -960,6 +961,7 @@ static void __pagevec_lru_add_fn(struct {enum lru_list lru;int was_unevictable =TestClearPageUnevictable (page); +int nr_pages =hpage_nr_pages (page);VM_BUG_ON_PAGE (PageLRU (page), page);
@@ -995,13 +997,13 @@ static void __pagevec_lru_add_fn(structif (page_evictable (page)) { lru =page_lru (page);if (was_unevictable) -count_vm_event (UNEVICTABLE_PGRESCUED ); +__count_vm_events (UNEVICTABLE_PGRESCUED , nr_pages); }else { lru =LRU_UNEVICTABLE ;ClearPageActive (page);SetPageUnevictable (page);if (!was_unevictable) -count_vm_event (UNEVICTABLE_PGCULLED ); +__count_vm_events (UNEVICTABLE_PGCULLED , nr_pages); }add_page_to_lru_list (page, lruvec, lru);
_
Andrew Morton
maintainerPATCH 118/131 mm: swap: memcg: fix memcg stats for huge pages
In reply to Andrew MortonFrom: Shakeel Butt <[email protected]>
Subject: mm: swap: memcg: fix memcg stats for huge pages
The commit 2262185c5b28 ("mm: per-cgroup memory reclaim stats") added PGLAZYFREE, PGACTIVATE & PGDEACTIVATE stats for cgroups but missed couple of places and PGLAZYFREE missed huge page handling. Fix that. Also for PGLAZYFREE use the irq-unsafe function to update as the irq is already disabled.
---
mm/swap.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)
mm/swap.c renamed from mm/swap.c~mm-swap-memcg-fix-memcg-stats-for-huge-pages +16 -5
@@ -320,6 +320,7 @@ static void __activate_page(struct page {if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) {int lru =page_lru_base_type (page); +int nr_pages =hpage_nr_pages (page);del_page_from_lru_list (page, lruvec, lru);SetPageActive (page);
@@ -327,7 +328,9 @@ static void __activate_page(struct pageadd_page_to_lru_list (page, lruvec, lru);trace_mm_lru_activate (page); -__count_vm_events (PGACTIVATE ,hpage_nr_pages (page)); +__count_vm_events (PGACTIVATE , nr_pages); +__count_memcg_events (lruvec_memcg (lruvec),PGACTIVATE , + nr_pages); } }
@@ -565,8 +568,11 @@ static void lru_deactivate_file_fn(struc__count_vm_events (PGROTATED , nr_pages); } -if (active) +if (active) {__count_vm_events (PGDEACTIVATE , nr_pages); +__count_memcg_events (lruvec_memcg (lruvec),PGDEACTIVATE , + nr_pages); + } } static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
@@ -574,13 +580,16 @@ static void lru_deactivate_fn(struct pag {if (PageLRU (page) &&PageActive (page) && !PageUnevictable (page)) {int lru =page_lru_base_type (page); +int nr_pages =hpage_nr_pages (page);del_page_from_lru_list (page, lruvec, lru +LRU_ACTIVE );ClearPageActive (page);ClearPageReferenced (page);add_page_to_lru_list (page, lruvec, lru); -__count_vm_events (PGDEACTIVATE ,hpage_nr_pages (page)); +__count_vm_events (PGDEACTIVATE , nr_pages); +__count_memcg_events (lruvec_memcg (lruvec),PGDEACTIVATE , + nr_pages); } }
@@ -590,6 +599,7 @@ static void lru_lazyfree_fn(struct pageif (PageLRU (page) &&PageAnon (page) &&PageSwapBacked (page) && !PageSwapCache (page) && !PageUnevictable (page)) {bool active =PageActive (page); +int nr_pages =hpage_nr_pages (page);del_page_from_lru_list (page, lruvec,LRU_INACTIVE_ANON + active);
@@ -603,8 +613,9 @@ static void lru_lazyfree_fn(struct pageClearPageSwapBacked (page);add_page_to_lru_list (page, lruvec,LRU_INACTIVE_FILE ); -__count_vm_events (PGLAZYFREE ,hpage_nr_pages (page)); -count_memcg_page_event (page,PGLAZYFREE ); +__count_vm_events (PGLAZYFREE , nr_pages); +__count_memcg_events (lruvec_memcg (lruvec),PGLAZYFREE , + nr_pages); } }
_
Andrew Morton
maintainerPATCH 119/131 tools/vm/page_owner_sort.c: filter out unneeded line
In reply to Andrew MortonFrom: Changhee Han <[email protected]>
Subject: tools/vm/page_owner_sort.c: filter out unneeded line
To see a sorted result from page_owner, there must be a tiresome preprocessing step before running page_owner_sort. This patch simply filters out lines which start with "PFN" while reading the page owner report.
---
Documentation/vm/page_owner.rst | 3 +-- tools/vm/page_owner_sort.c | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-)
Documentation/vm/page_owner.rst renamed from Documentation/vm/page_owner.rst~tools-vm-page_owner_sort-filter-out-unneeded-line +1 -2
@@ -83,8 +83,7 @@ Usage 4) Analyze information from page owner:: cat /sys/kernel/debug/page_owner > page_owner_full.txt - grep -v ^PFN page_owner_full.txt > page_owner.txt - ./page_owner_sort page_owner.txt sorted_page_owner.txt + ./page_owner_sort page_owner_full.txt sorted_page_owner.txt See the result about who allocated each page in the ``sorted_page_owner.txt``.
tools/vm/page_owner_sort.c renamed from tools/vm/page_owner_sort.c~tools-vm-page_owner_sort-filter-out-unneeded-line +3 -2
@@ -4,8 +4,7 @@* * Example use: * cat /sys/kernel/debug/page_owner > page_owner_full.txt -* grep -v ^PFN page_owner_full.txt > page_owner.txt -* ./page_owner_sort page_owner.txt sorted_page_owner.txt +* ./page_owner_sort page_owner_full.txt sorted_page_owner.txt * * See Documentation/vm/page_owner.rst */
@@ -38,6 +37,8 @@ int read_block(char *buf, int buf_size,while (buf_end - curr >1 &&fgets (curr, buf_end - curr, fin)) {if (*curr =='\n' )/* empty line */ return curr - buf; +if (!strncmp (curr,"PFN" ,3 )) +continue ; curr +=strlen (curr); }
_
Andrew Morton
maintainerPATCH 120/131 mm, mempolicy: fix up gup usage in lookup_node
In reply to Andrew MortonFrom: Michal Hocko <[email protected]>
Subject: mm, mempolicy: fix up gup usage in lookup_node
ba841078cd05 ("mm/mempolicy: Allow lookup_node() to handle fatal signal") has added a special casing for 0 return value because that was a possible gup return value when interrupted by fatal signal. This has been fixed by ae46d2aa6a7f ("mm/gup: Let __get_user_pages_locked() return -EINTR for fatal signal") in the mean time so ba841078cd05 can be reverted.
This patch however doesn't go all the way to revert it because the check for 0 is wrong and confusing here. Firstly it is inherently unsafe to access the page when get_user_pages_locked returns 0 (aka no page returned).
Fortunatelly this will not happen because get_user_pages_locked will not return 0 when nr_pages > 0 unless FOLL_NOWAIT is specified which is not the case here. Document this potential error code in gup code while we are at it.
---
mm/gup.c | 5 +++++ mm/mempolicy.c | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-)
mm/gup.c renamed from mm/gup.c~mm-mempolicy-fix-up-gup-usage-in-lookup_node +5 -0
@@ -989,6 +989,7 @@ static int check_vma_flags(struct vm_are* -- If nr_pages is >0, but no pages were pinned, returns -errno. * -- If nr_pages is >0, and some pages were pinned, returns the number of * pages pinned. Again, this may be less than nr_pages. +* -- 0 return value is possible when the fault would need to be retried. * * The caller is responsible for releasing returned @pages, via put_page(). *
@@ -1265,6 +1266,10 @@ retry: }EXPORT_SYMBOL_GPL (fixup_user_fault); +/* +* Please note that this function, unlike __get_user_pages will not +* return 0 for nr_pages > 0 without FOLL_NOWAIT +*/ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
mm/mempolicy.c renamed from mm/mempolicy.c~mm-mempolicy-fix-up-gup-usage-in-lookup_node +1 -4
@@ -927,10 +927,7 @@ static int lookup_node(struct mm_structint locked =1 ; err =get_user_pages_locked (addr &PAGE_MASK ,1 ,0 , &p, &locked); -if (err ==0 ) { -/* E.g. GUP interrupted by fatal signal */ - err = -EFAULT ; - }else if (err >0 ) { +if (err >0 ) { err =page_to_nid (p);put_page (p); }
_
Andrew Morton
maintainerPATCH 121/131 include/linux/memblock.h: fix minor typo and unclear comment
In reply to Andrew MortonFrom: chenqiwu <[email protected]>
Subject: include/linux/memblock.h: fix minor typo and unclear comment
Fix a minor typo "usabe->usable" for the current discription of member variable "memory" in struct memblock.
BTW, I think it's unclear the member variable "base" in struct memblock_type is currently described as the physical address of memory region, change it to base address of the region is clearer since the variable is decorated as phys_addr_t.
---
include/linux/memblock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/memblock.h renamed from include/linux/memblock.h~mm-memblock-fix-minor-typo-and-unclear-comment +2 -2
@@ -41,7 +41,7 @@ enum memblock_flags {/** * struct memblock_region - represents a memory region -* @base: physical address of the region +* @base: base address of the region * @size: size of the region * @flags: memory region attributes * @nid: NUMA node id
@@ -75,7 +75,7 @@ struct memblock_type {* struct memblock - memblock allocator metadata * @bottom_up: is bottom up direction? * @current_limit: physical address of the current allocation limit -* @memory: usabe memory regions +* @memory: usable memory regions * @reserved: reserved memory regions * @physmem: all physical memory */
_
Andrew Morton
maintainerPATCH 122/131 sparc32: register memory occupied by kernel as memblock.memory
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: sparc32: register memory occupied by kernel as memblock.memory
sparc32 never registered the memory occupied by the kernel image with memblock_add() and it only reserved this memory with meblock_reserve().
With openbios as system firmware, the memory occupied by the kernel is reserved in openbios and removed from mem.available. The prom setup code in the kernel uses mem.available to set up the memory banks and essentially there is a hole for the memory occupied by the kernel image.
Later in bootmem_init() this memory is memblock_reserve()d.
Up until recently, memmap initialization would call __init_single_page() for the pages in that hole, the free_low_memory_core_early() would mark them as reserved and everything would be Ok.
After the change in memmap initialization introduced by the commit "mm:memmap_init: iterate over memblock regions rather that check each PFN", the hole is skipped and the page structs for it are not initialized. And when they are passed from memblock to page allocator as reserved, the latter gets confused.
Simply registering the memory occupied by the kernel with memblock_add() resolves this issue.
Tested on qemu-system-sparc with Debian Etch [1] userspace.
[1] https://people.debian.org/~aurel32/qemu/sparc/debian_etch_sparc_small.qcow2
---
arch/sparc/mm/init_32.c | 1 + 1 file changed, 1 insertion(+)
arch/sparc/mm/init_32.c renamed from arch/sparc/mm/init_32.c~sparc32-register-memory-occupied-by-kernel-as-memblockmemory +1 -0
@@ -193,6 +193,7 @@ unsigned long __init bootmem_init(unsign/* Reserve the kernel text/data/bss. */ size = (start_pfn <<PAGE_SHIFT ) - phys_base;memblock_reserve (phys_base, size); +memblock_add (phys_base, size); size =memblock_phys_mem_size () -memblock_reserved_size (); *pages_avail = (size >>PAGE_SHIFT ) - high_pages;
_
Andrew Morton
maintainerPATCH 123/131 hugetlbfs: get unmapped area below TASK_UNMAPPED_BASE for hugetlbfs
In reply to Andrew MortonFrom: Shijie Hu <[email protected]>
Subject: hugetlbfs: get unmapped area below TASK_UNMAPPED_BASE for hugetlbfs
In a 32-bit program, running on arm64 architecture. When the address space below mmap base is completely exhausted, shmat() for huge pages will return ENOMEM, but shmat() for normal pages can still success on no-legacy mode. This seems not fair.
For normal pages, the calling trace of get_unmapped_area() is:
=> mm->get_unmapped_area() if on legacy mode, => arch_get_unmapped_area() => vm_unmapped_area() if on no-legacy mode, => arch_get_unmapped_area_topdown() => vm_unmapped_area()
For huge pages, the calling trace of get_unmapped_area() is:
=> file->f_op->get_unmapped_area() => hugetlb_get_unmapped_area() => vm_unmapped_area()
To solve this issue, we only need to make hugetlb_get_unmapped_area() take the same way as mm->get_unmapped_area(). Add *bottomup() and *topdown() for hugetlbfs, and check current mm->get_unmapped_area() to decide which one to use. If mm->get_unmapped_area is equal to arch_get_unmapped_area_topdown(), hugetlb_get_unmapped_area() calls topdown routine, otherwise calls bottomup routine.
---
fs/hugetlbfs/inode.c | 67 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 8 deletions(-)
fs/hugetlbfs/inode.c renamed from fs/hugetlbfs/inode.c~hugetlbfs-get-unmapped-area-below-task_unmapped_base-for-hugetlbfs +59 -8
@@ -38,6 +38,7 @@#include <linux/uio.h> #include <linux/uaccess.h> +#include <linux/sched/mm.h> static const struct super_operations hugetlbfs_ops;static const struct address_space_operations hugetlbfs_aops;
@@ -191,13 +192,60 @@ out:#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA static unsigned long +hugetlb_get_unmapped_area_bottomup (struct file *file,unsigned long addr, +unsigned long len,unsigned long pgoff,unsigned long flags) +{ +struct hstate *h =hstate_file (file); +struct vm_unmapped_area_info info; + + info.flags =0 ; + info.length = len; + info.low_limit = current->mm->mmap_base; + info.high_limit =TASK_SIZE ; + info.align_mask =PAGE_MASK & ~huge_page_mask (h); + info.align_offset =0 ; +return vm_unmapped_area (&info); +} + +static unsigned long +hugetlb_get_unmapped_area_topdown (struct file *file,unsigned long addr, +unsigned long len,unsigned long pgoff,unsigned long flags) +{ +struct hstate *h =hstate_file (file); +struct vm_unmapped_area_info info; + + info.flags =VM_UNMAPPED_AREA_TOPDOWN ; + info.length = len; + info.low_limit =max (PAGE_SIZE , mmap_min_addr); + info.high_limit = current->mm->mmap_base; + info.align_mask =PAGE_MASK & ~huge_page_mask (h); + info.align_offset =0 ; + addr =vm_unmapped_area (&info); + +/* +* A failed mmap() very likely causes application failure, +* so fall back to the bottom-up function here. This scenario +* can happen with large stack limits and large mmap() +* allocations. +*/ +if (unlikely (offset_in_page (addr))) { +VM_BUG_ON (addr != -ENOMEM ); + info.flags =0 ; + info.low_limit = current->mm->mmap_base; + info.high_limit =TASK_SIZE ; + addr =vm_unmapped_area (&info); + } + +return addr; +} + +static unsigned long hugetlb_get_unmapped_area (struct file *file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags) {struct mm_struct *mm = current->mm;struct vm_area_struct *vma;struct hstate *h =hstate_file (file); -struct vm_unmapped_area_info info;if (len & ~huge_page_mask (h))return -EINVAL ;
@@ -218,13 +266,16 @@ hugetlb_get_unmapped_area(struct file *f return addr; } - info.flags =0 ; - info.length = len; - info.low_limit =TASK_UNMAPPED_BASE ; - info.high_limit =TASK_SIZE ; - info.align_mask =PAGE_MASK & ~huge_page_mask (h); - info.align_offset =0 ; -return vm_unmapped_area (&info); +/* +* Use mm->get_unmapped_area value as a hint to use topdown routine. +* If architectures have special needs, they should define their own +* version of hugetlb_get_unmapped_area. +*/ +if (mm->get_unmapped_area == arch_get_unmapped_area_topdown) +return hugetlb_get_unmapped_area_topdown (file, addr, len, + pgoff, flags); +return hugetlb_get_unmapped_area_bottomup (file, addr, len, + pgoff, flags); }#endif
_
Andrew Morton
maintainerPATCH 124/131 mm: thp: don't need to drain lru cache when splitting and mlocking THP
In reply to Andrew MortonFrom: Yang Shi <[email protected]>
Subject: mm: thp: don't need to drain lru cache when splitting and mlocking THP
Since commit 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival") THP would not stay in pagevec anymore. So the optimization made by commit d965432234db ("thp: increase split_huge_page() success rate") doesn't make sense anymore, which tries to unpin munlocked THPs from pagevec by draining pagevec.
Draining lru cache before isolating THP in mlock path is also unnecessary. b676b293fb48 ("mm, thp: fix mapped pages avoiding unevictable list on mlock") added it and 9a73f61bdb8a ("thp, mlock: do not mlock PTE-mapped file huge pages") accidentally carried it over after the above optimization went in.
---
mm/huge_memory.c | 7 ------- 1 file changed, 7 deletions(-)
mm/huge_memory.c renamed from mm/huge_memory.c~mm-thp-dont-need-drain-lru-cache-when-splitting-and-mlocking-thp +0 -7
@@ -1378,7 +1378,6 @@ struct page *follow_trans_huge_pmd(struc goto skip_mlock;if (!trylock_page (page)) goto skip_mlock; -lru_add_drain ();if (page->mapping && !PageDoubleMap (page))mlock_vma_page (page);unlock_page (page);
@@ -2582,7 +2581,6 @@ int split_huge_page_to_list(struct pagestruct anon_vma *anon_vma =NULL ;struct address_space *mapping =NULL ;int count, mapcount, extra_pins, ret; -bool mlocked;unsigned long flags;pgoff_t end;
@@ -2641,14 +2639,9 @@ int split_huge_page_to_list(struct page goto out_unlock; } - mlocked =PageMlocked (head);unmap_page (head);VM_BUG_ON_PAGE (compound_mapcount (head), head); -/* Make sure the page is not on per-CPU pagevec as it takes pin */ -if (mlocked) -lru_add_drain (); -/* prevent PageLRU to go away from under us, and freeze lru stats */ spin_lock_irqsave (&pgdata->lru_lock, flags);
_
Andrew Morton
maintainerPATCH 125/131 powerpc/mm: drop platform defined pmd_mknotpresent()
In reply to Andrew MortonFrom: Anshuman Khandual <[email protected]>
Subject: powerpc/mm: drop platform defined pmd_mknotpresent()
Patch series "mm/thp: Rename pmd_mknotpresent() as pmd_mknotvalid()", v2.
This series renames pmd_mknotpresent() as pmd_mknotvalid(). Before that it drops an existing pmd_mknotpresent() definition from powerpc platform which was never required as it defines it's pmdp_invalidate() through subscribing __HAVE_ARCH_PMDP_INVALIDATE. This does not create any functional change.
This rename was suggested by Catalin during a previous discussion while we were trying to change the THP helpers on arm64 platform for migration.
https://patchwork.kernel.org/patch/11019637/
This patch (of 2):
Platform needs to define pmd_mknotpresent() for generic pmdp_invalidate() only when __HAVE_ARCH_PMDP_INVALIDATE is not subscribed. Otherwise platform specific pmd_mknotpresent() is not required. Hence just drop it.
---
arch/powerpc/include/asm/book3s/64/pgtable.h | 4 ---- 1 file changed, 4 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h renamed from arch/powerpc/include/asm/book3s/64/pgtable.h~powerpc-mm-drop-platform-defined-pmd_mknotpresent +0 -4
@@ -1168,10 +1168,6 @@ static inline int pmd_large(pmd_t pmd)return !!(pmd_raw (pmd) &cpu_to_be64 (_PAGE_PTE)); } -static inline pmd_t pmd_mknotpresent (pmd_t pmd) -{ -return __pmd (pmd_val (pmd) & ~_PAGE_PRESENT); -}/* * For radix we should always find H_PAGE_HASHPTE zero. Hence * the below will work for radix too
_
Andrew Morton
maintainerPATCH 126/131 mm/thp: rename pmd_mknotpresent() as pmd_mkinvalid()
In reply to Andrew MortonFrom: Anshuman Khandual <[email protected]>
Subject: mm/thp: rename pmd_mknotpresent() as pmd_mkinvalid()
pmd_present() is expected to test positive after pmdp_mknotpresent() as the PMD entry still points to a valid huge page in memory. pmdp_mknotpresent() implies that given PMD entry is just invalidated from MMU perspective while still holding on to pmd_page() referred valid huge page thus also clearing pmd_present() test. This creates the following situation which is counter intuitive.
[pmd_present(pmd_mknotpresent(pmd)) = true]
This renames pmd_mknotpresent() as pmd_mkinvalid() reflecting the helper's functionality more accurately while changing the above mentioned situation as follows. This does not create any functional change.
[pmd_present(pmd_mkinvalid(pmd)) = true]
This is not applicable for platforms that define own pmdp_invalidate() via __HAVE_ARCH_PMDP_INVALIDATE. Suggestion for renaming came during a previous discussion here.
https://patchwork.kernel.org/patch/11019637/
---
arch/arc/include/asm/hugepage.h | 2 +- arch/arm/include/asm/pgtable-3level.h | 2 +- arch/arm64/include/asm/pgtable.h | 2 +- arch/mips/include/asm/pgtable.h | 2 +- arch/x86/include/asm/pgtable.h | 2 +- arch/x86/mm/kmmio.c | 2 +- mm/pgtable-generic.c | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-)
arch/arc/include/asm/hugepage.h renamed from arch/arc/include/asm/hugepage.h~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -26,7 +26,7 @@ static inline pmd_t pte_pmd(pte_t pte)#define pmd_mkold (pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))#define pmd_mkyoung (pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))#define pmd_mkhuge (pmd) pte_pmd(pte_mkhuge(pmd_pte(pmd))) -#define pmd_mknotpresent (pmd) pte_pmd(pte_mknotpresent(pmd_pte(pmd))) +#define pmd_mkinvalid (pmd) pte_pmd(pte_mknotpresent(pmd_pte(pmd)))#define pmd_mkclean (pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))#define pmd_write (pmd) pte_write(pmd_pte(pmd))
arch/arm64/include/asm/pgtable.h renamed from arch/arm64/include/asm/pgtable.h~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -366,7 +366,7 @@ static inline int pmd_protnone(pmd_t pmd#define pmd_mkclean (pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))#define pmd_mkdirty (pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))#define pmd_mkyoung (pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) -#define pmd_mknotpresent (pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID)) +#define pmd_mkinvalid (pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID))#define pmd_thp_or_huge (pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
arch/arm/include/asm/pgtable-3level.h renamed from arch/arm/include/asm/pgtable-3level.h~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -221,7 +221,7 @@ PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);#define pmdp_establish generic_pmdp_establish/* represent a notpresent pmd by faulting entry, this is used by pmdp_invalidate */ -static inline pmd_t pmd_mknotpresent (pmd_t pmd) +static inline pmd_t pmd_mkinvalid (pmd_t pmd) {return __pmd (pmd_val (pmd) & ~L_PMD_SECT_VALID ); }
arch/mips/include/asm/pgtable.h renamed from arch/mips/include/asm/pgtable.h~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -631,7 +631,7 @@ static inline pmd_t pmd_modify(pmd_t pmdreturn pmd; } -static inline pmd_t pmd_mknotpresent (pmd_t pmd) +static inline pmd_t pmd_mkinvalid (pmd_t pmd) {pmd_val (pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY);
arch/x86/include/asm/pgtable.h renamed from arch/x86/include/asm/pgtable.h~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -624,7 +624,7 @@ static inline pud_t pfn_pud(unsigned lonreturn __pud (pfn |check_pgprot (pgprot)); } -static inline pmd_t pmd_mknotpresent (pmd_t pmd) +static inline pmd_t pmd_mkinvalid (pmd_t pmd) {return pfn_pmd (pmd_pfn (pmd),__pgprot (pmd_flags (pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
arch/x86/mm/kmmio.c renamed from arch/x86/mm/kmmio.c~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -130,7 +130,7 @@ static void clear_pmd_presence(pmd_t *pmpmdval_t v =pmd_val (*pmd);if (clear) { *old = v; - new_pmd =pmd_mknotpresent (*pmd); + new_pmd =pmd_mkinvalid (*pmd); }else {/* Presume this has been called with clear==true previously */ new_pmd =__pmd (*old);
mm/pgtable-generic.c renamed from mm/pgtable-generic.c~mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid +1 -1
@@ -194,7 +194,7 @@ pgtable_t pgtable_trans_huge_withdraw(stpmd_t pmdp_invalidate (struct vm_area_struct *vma,unsigned long address,pmd_t *pmdp) { -pmd_t old =pmdp_establish (vma, address, pmdp,pmd_mknotpresent (*pmdp)); +pmd_t old =pmdp_establish (vma, address, pmdp,pmd_mkinvalid (*pmdp));flush_pmd_tlb_range (vma, address, address +HPAGE_PMD_SIZE );return old; }
_
Andrew Morton
maintainerPATCH 127/131 drivers/base/memory.c: cache memory blocks in xarray to accelerate lookup
In reply to Andrew MortonFrom: Scott Cheloha <[email protected]>
Subject: drivers/base/memory.c: cache memory blocks in xarray to accelerate lookup
Searching for a particular memory block by id is an O(n) operation because each memory block's underlying device is kept in an unsorted linked list on the subsystem bus.
We can cut the lookup cost to O(log n) if we cache each memory block in an xarray. This time complexity improvement is significant on systems with many memory blocks. For example:
1. A 128GB POWER9 VM with 256MB memblocks has 512 blocks. With this change memory_dev_init() completes ~12ms faster and walk_memory_blocks() completes ~12ms faster.
Before:
[ 0.005042] memory_dev_init: adding memory blocks [ 0.021591] memory_dev_init: added memory blocks [ 0.022699] walk_memory_blocks: walking memory blocks [ 0.038730] walk_memory_blocks: walked memory blocks 0-511
After:
[ 0.005057] memory_dev_init: adding memory blocks [ 0.009415] memory_dev_init: added memory blocks [ 0.010519] walk_memory_blocks: walking memory blocks [ 0.014135] walk_memory_blocks: walked memory blocks 0-511
2. A 256GB POWER9 LPAR with 256MB memblocks has 1024 blocks. With this change memory_dev_init() completes ~88ms faster and walk_memory_blocks() completes ~87ms faster.
Before:
[ 0.252246] memory_dev_init: adding memory blocks [ 0.395469] memory_dev_init: added memory blocks [ 0.409413] walk_memory_blocks: walking memory blocks [ 0.433028] walk_memory_blocks: walked memory blocks 0-511 [ 0.433094] walk_memory_blocks: walking memory blocks [ 0.500244] walk_memory_blocks: walked memory blocks 131072-131583
After:
[ 0.245063] memory_dev_init: adding memory blocks [ 0.299539] memory_dev_init: added memory blocks [ 0.313609] walk_memory_blocks: walking memory blocks [ 0.315287] walk_memory_blocks: walked memory blocks 0-511 [ 0.315349] walk_memory_blocks: walking memory blocks [ 0.316988] walk_memory_blocks: walked memory blocks 131072-131583
3. A 32TB POWER9 LPAR with 256MB memblocks has 131072 blocks. With this change we complete memory_dev_init() ~37 minutes faster and walk_memory_blocks() at least ~30 minutes faster. The exact timing for walk_memory_blocks() is missing, though I observed that the soft lockups in walk_memory_blocks() disappeared with the change, suggesting that lower bound.
Before:
[ 13.703907] memory_dev_init: adding blocks [ 2287.406099] memory_dev_init: added all blocks [ 2347.494986] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 2527.625378] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 2707.761977] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 2887.899975] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3068.028318] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3248.158764] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3428.287296] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3608.425357] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3788.554572] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 3968.695071] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160 [ 4148.823970] [c000000014c5bb60] [c000000000869af4] walk_memory_blocks+0x94/0x160
After:
[ 13.696898] memory_dev_init: adding blocks [ 15.660035] memory_dev_init: added all blocks
(the walk_memory_blocks traces disappear)
There should be no significant negative impact for machines with few memory blocks. A sparse xarray has a small footprint and an O(log n) lookup is negligibly slower than an O(n) lookup for only the smallest number of memory blocks.
1. A 16GB x86 machine with 128MB memblocks has 132 blocks. With this change memory_dev_init() completes ~300us faster and walk_memory_blocks() completes no faster or slower. The improvement is pretty close to noise.
Before:
[ 0.224752] memory_dev_init: adding memory blocks [ 0.227116] memory_dev_init: added memory blocks [ 0.227183] walk_memory_blocks: walking memory blocks [ 0.227183] walk_memory_blocks: walked memory blocks 0-131
After:
[ 0.224911] memory_dev_init: adding memory blocks [ 0.226935] memory_dev_init: added memory blocks [ 0.227089] walk_memory_blocks: walking memory blocks [ 0.227089] walk_memory_blocks: walked memory blocks 0-131
---
drivers/base/memory.c | 44 +++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-)
drivers/base/memory.c renamed from drivers/base/memory.c~drivers-base-memoryc-cache-memory-blocks-in-xarray-to-accelerate-lookup +32 -12
@@ -21,6 +21,7 @@#include <linux/mm.h> #include <linux/stat.h> #include <linux/slab.h> +#include <linux/xarray.h> #include <linux/atomic.h> #include <linux/uaccess.h>
@@ -74,6 +75,13 @@ static struct bus_type memory_subsys = { .offline = memory_subsys_offline, }; +/* +* Memory blocks are cached in a local radix tree to avoid +* a costly linear search for the corresponding device on +* the subsystem bus. +*/ +static DEFINE_XARRAY (memory_blocks); +static BLOCKING_NOTIFIER_HEAD (memory_chain);int register_memory_notifier (struct notifier_block *nb)
@@ -489,22 +497,23 @@ int __weak arch_get_memory_phys_device(ureturn 0 ; } -/* A reference for the returned memory block device is acquired. */ +/* +* A reference for the returned memory block device is acquired. +* +* Called under device_hotplug_lock. +*/ static struct memory_block *find_memory_block_by_id (unsigned long block_id) { -struct device *dev; +struct memory_block *mem; - dev =subsys_find_device_by_id (&memory_subsys, block_id,NULL ); -return dev ?to_memory_block (dev) :NULL ; + mem =xa_load (&memory_blocks, block_id); +if (mem) +get_device (&mem->dev); +return mem; }/* -* For now, we have a linear search to go find the appropriate -* memory_block corresponding to a particular phys_index. If -* this gets to be a real problem, we can always use a radix -* tree or something here. -* -* This could be made generic for all device subsystems. +* Called under device_hotplug_lock. */ struct memory_block *find_memory_block (struct mem_section *section) {
@@ -548,9 +557,16 @@ int register_memory(struct memory_block memory->dev.offline = memory->state ==MEM_OFFLINE ; ret =device_register (&memory->dev); -if (ret) +if (ret) {put_device (&memory->dev); - +return ret; + } + ret =xa_err (xa_store (&memory_blocks, memory->dev.id, memory, +GFP_KERNEL )); +if (ret) { +put_device (&memory->dev); +device_unregister (&memory->dev); + }return ret; }
@@ -604,6 +620,8 @@ static void unregister_memory(struct memif (WARN_ON_ONCE (memory->dev.bus != &memory_subsys))return ; +WARN_ON (xa_erase (&memory_blocks, memory->dev.id) ==NULL ); +/* drop the ref. we got via find_memory_block() */ put_device (&memory->dev);device_unregister (&memory->dev);
@@ -750,6 +768,8 @@ void __init memory_dev_init(void)* * In case func() returns an error, walking is aborted and the error is * returned. +* +* Called under device_hotplug_lock. */ int walk_memory_blocks (unsigned long start,unsigned long size,void *arg,walk_memory_blocks_func_t func)
_
Andrew Morton
maintainerPATCH 128/131 mm: add DEBUG_WX support
In reply to Andrew MortonFrom: Zong Li <[email protected]>
Subject: mm: add DEBUG_WX support
Patch series "Extract DEBUG_WX to shared use".
Some architectures support DEBUG_WX function, it's verbatim from each others, so extract to mm/Kconfig.debug for shared use.
PPC and ARM ports don't support generic page dumper yet, so we only refine x86 and arm64 port in this patch series.
For RISC-V port, the DEBUG_WX support depends on other patches which be merged already:
- RISC-V page table dumper
- Support strict kernel memory permissions for security
This patch (of 4):
Some architectures support DEBUG_WX function, it's verbatim from each others. Extract to mm/Kconfig.debug for shared use.
---
mm/Kconfig.debug | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
mm/Kconfig.debug renamed from mm/Kconfig.debug~mm-add-debug_wx-support +32 -0
@@ -118,6 +118,38 @@ config DEBUG_RODATA_TEST ---help--- This option enables a testcase for the setting rodata read-only. +config ARCH_HAS_DEBUG_WX + bool + +config DEBUG_WX + bool "Warn on W+X mappings at boot" + depends on ARCH_HAS_DEBUG_WX + depends on MMU + select PTDUMP_CORE + help + Generate a warning if any W+X mappings are found at boot. + + This is useful for discovering cases where the kernel is leaving W+X + mappings after applying NX, as such mappings are a security risk. + + Look for a message in dmesg output like this: + + <arch>/mm: Checked W+X mappings: passed, no W+X pages found. + + or like this, if the check failed: + + <arch>/mm: Checked W+X mappings: failed, <N> W+X pages found. + + Note that even if the check fails, your kernel is possibly + still fine, as W+X mappings are not a security hole in + themselves, what they do is that they make the exploitation + of other unfixed kernel bugs easier. + + There is no runtime or memory usage effect of this option + once the kernel has booted up - it's a one time check. + + If in doubt, say "Y". + config GENERIC_PTDUMP bool
_
Andrew Morton
maintainerPATCH 129/131 riscv: support DEBUG_WX
In reply to Andrew MortonFrom: Zong Li <[email protected]>
Subject: riscv: support DEBUG_WX
Support DEBUG_WX to check whether there are mapping with write and execute permission at the same time.
---
arch/riscv/Kconfig | 1 + arch/riscv/include/asm/ptdump.h | 11 +++++++++++ arch/riscv/mm/init.c | 3 +++ 3 files changed, 15 insertions(+)
arch/riscv/include/asm/ptdump.h renamed from arch/riscv/include/asm/ptdump.h~riscv-support-debug_wx +11 -0
@@ -8,4 +8,15 @@void ptdump_check_wx (void ); +#ifdef CONFIG_DEBUG_WX +static inline void debug_checkwx (void ) +{ +ptdump_check_wx (); +} +#else +static inline void debug_checkwx (void ) +{ +} +#endif +#endif /* _ASM_RISCV_PTDUMP_H */
arch/riscv/Kconfig renamed from arch/riscv/Kconfig~riscv-support-debug_wx +1 -0
@@ -16,6 +16,7 @@ config RISCV select OF_EARLY_FLATTREE select OF_IRQ select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_DEBUG_WX select ARCH_WANT_FRAME_POINTERS select CLONE_BACKWARDS select COMMON_CLK
arch/riscv/mm/init.c renamed from arch/riscv/mm/init.c~riscv-support-debug_wx +3 -0
@@ -19,6 +19,7 @@#include <asm/sections.h> #include <asm/pgtable.h> #include <asm/io.h> +#include <asm/ptdump.h> #include "../kernel/head.h"
@@ -514,6 +515,8 @@ void mark_rodata_ro(void)set_memory_ro (rodata_start, (data_start - rodata_start) >>PAGE_SHIFT );set_memory_nx (rodata_start, (data_start - rodata_start) >>PAGE_SHIFT );set_memory_nx (data_start, (max_low - data_start) >>PAGE_SHIFT ); + + debug_checkwx(); }#endif
_
Andrew Morton
maintainerPATCH 130/131 x86: mm: use ARCH_HAS_DEBUG_WX instead of arch defined
In reply to Andrew MortonFrom: Zong Li <[email protected]>
Subject: x86: mm: use ARCH_HAS_DEBUG_WX instead of arch defined
Extract DEBUG_WX to mm/Kconfig.debug for shared use. Change to use ARCH_HAS_DEBUG_WX instead of DEBUG_WX defined by arch port.
---
arch/x86/Kconfig | 1 + arch/x86/Kconfig.debug | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-)
arch/x86/Kconfig renamed from arch/x86/Kconfig~x86-mm-use-arch_has_debug_wx-instead-of-arch-defined +1 -0
@@ -81,6 +81,7 @@ config X86 select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_UBSAN_SANITIZE_ALL + select ARCH_HAS_DEBUG_WX select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT
arch/x86/Kconfig.debug renamed from arch/x86/Kconfig.debug~x86-mm-use-arch_has_debug_wx-instead-of-arch-defined +0 -27
@@ -72,33 +72,6 @@ config EFI_PGT_DUMP issues with the mapping of the EFI runtime regions into that table. -config DEBUG_WX - bool "Warn on W+X mappings at boot" - select PTDUMP_CORE - ---help--- - Generate a warning if any W+X mappings are found at boot. - - This is useful for discovering cases where the kernel is leaving - W+X mappings after applying NX, as such mappings are a security risk. - - Look for a message in dmesg output like this: - - x86/mm: Checked W+X mappings: passed, no W+X pages found. - - or like this, if the check failed: - - x86/mm: Checked W+X mappings: FAILED, <N> W+X pages found. - - Note that even if the check fails, your kernel is possibly - still fine, as W+X mappings are not a security hole in - themselves, what they do is that they make the exploitation - of other unfixed kernel bugs easier. - - There is no runtime or memory usage effect of this option - once the kernel has booted up - it's a one time check. - - If in doubt, say "Y". - config DEBUG_TLBFLUSH bool "Set upper limit of TLB entries to flush one-by-one" depends on DEBUG_KERNEL
_
Andrew Morton
maintainerPATCH 131/131 arm64: mm: use ARCH_HAS_DEBUG_WX instead of arch defined
In reply to Andrew MortonFrom: Zong Li <[email protected]>
Subject: arm64: mm: use ARCH_HAS_DEBUG_WX instead of arch defined
Extract DEBUG_WX to mm/Kconfig.debug for shared use. Change to use ARCH_HAS_DEBUG_WX instead of DEBUG_WX defined by arch port.
---
arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 29 ----------------------------- 2 files changed, 1 insertion(+), 29 deletions(-)
arch/arm64/Kconfig renamed from arch/arm64/Kconfig~arm64-mm-use-arch_has_debug_wx-instead-of-arch-defined +1 -0
@@ -9,6 +9,7 @@ config ARM64 select ACPI_MCFG if (ACPI && PCI) select ACPI_SPCR_TABLE if ACPI select ACPI_PPTT if ACPI + select ARCH_HAS_DEBUG_WX select ARCH_BINFMT_ELF_STATE select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED
arch/arm64/Kconfig.debug renamed from arch/arm64/Kconfig.debug~arm64-mm-use-arch_has_debug_wx-instead-of-arch-defined +0 -29
@@ -23,35 +23,6 @@ config ARM64_RANDOMIZE_TEXT_OFFSET of TEXT_OFFSET and platforms must not require a specific value. -config DEBUG_WX - bool "Warn on W+X mappings at boot" - select PTDUMP_CORE - ---help--- - Generate a warning if any W+X mappings are found at boot. - - This is useful for discovering cases where the kernel is leaving - W+X mappings after applying NX, as such mappings are a security risk. - This check also includes UXN, which should be set on all kernel - mappings. - - Look for a message in dmesg output like this: - - arm64/mm: Checked W+X mappings: passed, no W+X pages found. - - or like this, if the check failed: - - arm64/mm: Checked W+X mappings: FAILED, <N> W+X pages found. - - Note that even if the check fails, your kernel is possibly - still fine, as W+X mappings are not a security hole in - themselves, what they do is that they make the exploitation - of other unfixed kernel bugs easier. - - There is no runtime or memory usage effect of this option - once the kernel has booted up - it's a one time check. - - If in doubt, say "Y". - config DEBUG_EFI depends on EFI && DEBUG_INFO bool "UEFI debugging"
_
+ lib-test-get_count_order-long-in-test_bitopsc-fix.patch added to -mm tree
In reply to Andrew MortonThe patch titled Subject: lib-test-get_count_order-long-in-test_bitopsc-fix has been added to the -mm tree. Its filename is lib-test-get_count_order-long-in-test_bitopsc-fix.patch
The patch titled
Subject: lib-test-get_count_order-long-in-test_bitopsc-fix
has been added to the -mm tree. Its filename is
lib-test-get_count_order-long-in-test_bitopsc-fix.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/lib-test-get_count_order-long-in-test_bitopsc-fix.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/lib-test-get_count_order-long-in-test_bitopsc-fix.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated there every 3-4 working days
------------------------------------------------------
From: Andrew Morton <[email protected]>
Subject: lib-test-get_count_order-long-in-test_bitopsc-fix
define local `i'
---
lib/test_bitops.c | 2 ++ 1 file changed, 2 insertions(+)
lib/test_bitops.c renamed from lib/test_bitops.c~lib-test-get_count_order-long-in-test_bitopsc-fix +2 -0
@@ -50,6 +50,8 @@ unsigned long order_comb_long[][2] = {static int __init test_bitops_startup (void ) { +int i; +pr_warn ("Loaded test module\n" );set_bit (BITOPS_4 , g_bitmap);set_bit (BITOPS_7 , g_bitmap);
_
Patches currently in -mm which might be from [email protected] are
drivers-tty-serial-sh-scic-suppress-uninitialized-var-warning.patch mm.patch mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix.patch arch-kunmap-remove-duplicate-kunmap-implementations-fix.patch arch-kmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch arch-kunmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch kmap-consolidate-kmap_prot-definitions-checkpatch-fixes.patch mm-replace-zero-length-array-with-flexible-array-member-fix.patch mm-hugetlb-fix-a-typo-in-comment-manitained-maintained-v2-checkpatch-fixes.patch lib-test-get_count_order-long-in-test_bitopsc-fix.patch seq_file-introduce-define_seq_attribute-helper-macro-checkpatch-fixes.patch ipc-convert-ipcs_idr-to-xarray-update-fix.patch
linux-next-pre.patch
linux-next-rejects.patch
linux-next-git-rejects.patch
linux-next-post.patch
kernel-add-panic_on_taint-fix.patch mm-consolidate-pgd_index-and-pgd_offset_k-definitions-fix.patch mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix.patch mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix-fix.patch mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix-fix-fix.patch mmap-locking-api-rename-mmap_sem-to-mmap_lock-fix.patch mmap-locking-api-convert-mmap_sem-comments-fix.patch mmap-locking-api-convert-mmap_sem-comments-fix-fix.patch mmap-locking-api-convert-mmap_sem-comments-fix-fix-fix.patch mm-pass-task-and-mm-to-do_madvise.patch mm-introduce-external-memory-hinting-api-fix-2-fix.patch mm-support-vector-address-ranges-for-process_madvise-fix-fix-fix-fix-fix.patch maccess-unify-the-probe-kernel-arch-hooks-fix.patch bpf-bpf_seq_printf-handle-potentially-unsafe-format-string-better.patch maccess-always-use-strict-semantics-for-probe_kernel_read-fix.patch x86-use-non-set_fs-based-maccess-routines-checkpatch-fixes.patch doc-cgroup-update-note-about-conditions-when-oom-killer-is-invoked-fix.patch sh-convert-ins-outs-macros-to-inline-functions-checkpatch-fixes.patch kernel-forkc-export-kernel_thread-to-modules.patch
Andrew Morton
maintainermmotm 2020-06-03-17-54 uploaded
In reply to Andrew MortonThe mm-of-the-moment snapshot 2020-06-03-17-54 has been uploaded to
http://www.ozlabs.org/~akpm/mmotm/
mmotm-readme.txt says
README for mm-of-the-moment:
http://www.ozlabs.org/~akpm/mmotm/
This is a snapshot of my -mm patch queue. Uploaded at random hopefully more than once a week.
You will need quilt to apply these patches to the latest Linus release (5.x or 5.x-rcY). The series file is in broken-out.tar.gz and is duplicated in
http://ozlabs.org/~akpm/mmotm/series
The file broken-out.tar.gz contains two datestamp files: .DATE and .DATE-yyyy-mm-dd-hh-mm-ss. Both contain the string yyyy-mm-dd-hh-mm-ss, followed by the base kernel version against which this patch series is to be applied.
This tree is partially included in linux-next. To see which patches are included in linux-next, consult the `series' file. Only the patches within the #NEXT_PATCHES_START/#NEXT_PATCHES_END markers are included in linux-next.
A full copy of the full kernel tree with the linux-next and mmotm patches already applied is available through git within an hour of the mmotm release. Individual mmotm releases are tagged. The master branch always points to the latest release, so it's constantly rebasing.
https://github.com/hnaz/linux-mm
The directory http://www.ozlabs.org/~akpm/mmots/ (mm-of-the-second) contains daily snapshots of the -mm tree. It is updated more frequently than mmotm, and is untested.
A git copy of this tree is also available at
https://github.com/hnaz/linux-mm
This mmotm tree contains the following patches against 5.7:
(patches marked "*" will be included in linux-next)
origin.patch
* mm-slub-fix-a-memory-leak-in-sysfs_slab_add.patch
* memcg-optimize-memorynuma_stat-like-memorystat.patch
* mm-gup-move-__get_user_pages_fast-down-a-few-lines-in-gupc.patch
* mm-gup-refactor-and-de-duplicate-gup_fast-code.patch
* mm-gup-introduce-pin_user_pages_fast_only.patch
* drm-i915-convert-get_user_pages-pin_user_pages.patch
* mm-gup-might_lock_readmmap_sem-in-get_user_pages_fast.patch
* kasan-stop-tests-being-eliminated-as-dead-code-with-fortify_source.patch
* stringh-fix-incompatibility-between-fortify_source-and-kasan.patch
* mm-clarify-__gfp_memalloc-usage.patch
* mm-memblock-replace-dereferences-of-memblock_regionnid-with-api-calls.patch
* mm-make-early_pfn_to_nid-and-related-defintions-close-to-each-other.patch
* mm-remove-config_have_memblock_node_map-option.patch
* mm-free_area_init-use-maximal-zone-pfns-rather-than-zone-sizes.patch
* mm-use-free_area_init-instead-of-free_area_init_nodes.patch
* alpha-simplify-detection-of-memory-zone-boundaries.patch
* arm-simplify-detection-of-memory-zone-boundaries.patch
* arm64-simplify-detection-of-memory-zone-boundaries-for-uma-configs.patch
* csky-simplify-detection-of-memory-zone-boundaries.patch
* m68k-mm-simplify-detection-of-memory-zone-boundaries.patch
* parisc-simplify-detection-of-memory-zone-boundaries.patch
* sparc32-simplify-detection-of-memory-zone-boundaries.patch
* unicore32-simplify-detection-of-memory-zone-boundaries.patch
* xtensa-simplify-detection-of-memory-zone-boundaries.patch
* mm-memmap_init-iterate-over-memblock-regions-rather-that-check-each-pfn.patch
* mm-remove-early_pfn_in_nid-and-config_nodes_span_other_nodes.patch
* mm-free_area_init-allow-defining-max_zone_pfn-in-descending-order.patch
* mm-rename-free_area_init_node-to-free_area_init_memoryless_node.patch
* mm-clean-up-free_area_init_node-and-its-helpers.patch
* mm-simplify-find_min_pfn_with_active_regions.patch
* docs-vm-update-memory-models-documentation.patch
* mm-page_allocc-bad_-is-not-necessary-when-pagehwpoison.patch
* mm-page_allocc-bad_flags-is-not-necessary-for-bad_page.patch
* mm-page_allocc-rename-free_pages_check_bad-to-check_free_page_bad.patch
* mm-page_allocc-rename-free_pages_check-to-check_free_page.patch
* mm-page_allocc-extract-check__page_bad-common-part-to-page_bad_reason.patch
* mmpage_alloccma-conditionally-prefer-cma-pageblocks-for-movable-allocations.patch
* mm-remove-unused-free_bootmem_with_active_regions.patch
* mm-page_allocc-only-tune-sysctl_lowmem_reserve_ratio-value-once-when-changing-it.patch
* mm-page_allocc-clear-out-zone-lowmem_reserve-if-the-zone-is-empty.patch
* mm-vmstatc-do-not-show-lowmem-reserve-protection-information-of-empty-zone.patch
* mm-page_alloc-use-ac-high_zoneidx-for-classzone_idx.patch
* mm-page_alloc-integrate-classzone_idx-and-high_zoneidx.patch
* mm-page_allocc-use-node_mask_none-in-build_zonelists.patch
* mm-rename-gfpflags_to_migratetype-to-gfp_migratetype-for-same-convention.patch
* mm-reset-numa-stats-for-boot-pagesets.patch
* mm-page_alloc-reset-the-zone-watermark_boost-early.patch
* mm-page_alloc-restrict-and-formalize-compound_page_dtors.patch
* mm-call-touch_nmi_watchdog-on-max-order-boundaries-in-deferred-init.patch
* mm-initialize-deferred-pages-with-interrupts-enabled.patch
* mm-call-cond_resched-from-deferred_init_memmap.patch
* padata-remove-exit-routine.patch
* padata-initialize-earlier.patch
* padata-allocate-work-structures-for-parallel-jobs-from-a-pool.patch
* padata-add-basic-support-for-multithreaded-jobs.patch
* mm-dont-track-number-of-pages-during-deferred-initialization.patch
* mm-parallelize-deferred_init_memmap.patch
* mm-make-deferred-inits-max-threads-arch-specific.patch
* padata-document-multithreaded-jobs.patch
* mm-page_allocc-add-missing-line-breaks.patch
* khugepaged-add-self-test.patch
* khugepaged-do-not-stop-collapse-if-less-than-half-ptes-are-referenced.patch
* khugepaged-drain-all-lru-caches-before-scanning-pages.patch
* khugepaged-drain-lru-add-pagevec-after-swapin.patch
* khugepaged-allow-to-collapse-a-page-shared-across-fork.patch
* khugepaged-allow-to-collapse-pte-mapped-compound-pages.patch
* thp-change-cow-semantics-for-anon-thp.patch
* khugepaged-introduce-max_ptes_shared-tunable.patch
* hugetlbfs-add-arch_hugetlb_valid_size.patch
* hugetlbfs-move-hugepagesz=-parsing-to-arch-independent-code.patch
* hugetlbfs-remove-hugetlb_add_hstate-warning-for-existing-hstate.patch
* hugetlbfs-clean-up-command-line-processing.patch
* hugetlbfs-move-hugepagesz=-parsing-to-arch-independent-code-fix.patch
* mm-hugetlb-avoid-unnecessary-check-on-pud-and-pmd-entry-in-huge_pte_offset.patch
* arm64-mm-drop-__have_arch_huge_ptep_get.patch
* mm-hugetlb-define-a-generic-fallback-for-is_hugepage_only_range.patch
* mm-hugetlb-define-a-generic-fallback-for-arch_clear_hugepage_flags.patch
* mm-simplify-calling-a-compound-page-destructor.patch
* mm-vmscanc-use-update_lru_size-in-update_lru_sizes.patch
* mm-vmscan-count-layzfree-pages-and-fix-nr_isolated_-mismatch.patch
* mm-vmscanc-change-prototype-for-shrink_page_list.patch
* mm-vmscan-update-the-comment-of-should_continue_reclaim.patch
* mm-fix-numa-node-file-count-error-in-replace_page_cache.patch
* mm-memcontrol-fix-stat-corrupting-race-in-charge-moving.patch
* mm-memcontrol-drop-compound-parameter-from-memcg-charging-api.patch
* mm-shmem-remove-rare-optimization-when-swapin-races-with-hole-punching.patch
* mm-memcontrol-move-out-cgroup-swaprate-throttling.patch
* mm-memcontrol-convert-page-cache-to-a-new-mem_cgroup_charge-api.patch
* mm-memcontrol-prepare-uncharging-for-removal-of-private-page-type-counters.patch
* mm-memcontrol-prepare-move_account-for-removal-of-private-page-type-counters.patch
* mm-memcontrol-prepare-cgroup-vmstat-infrastructure-for-native-anon-counters.patch
* mm-memcontrol-switch-to-native-nr_file_pages-and-nr_shmem-counters.patch
* mm-memcontrol-switch-to-native-nr_anon_mapped-counter.patch
* mm-memcontrol-switch-to-native-nr_anon_thps-counter.patch
* mm-memcontrol-convert-anon-and-file-thp-to-new-mem_cgroup_charge-api.patch
* mm-memcontrol-drop-unused-try-commit-cancel-charge-api.patch
* mm-memcontrol-prepare-swap-controller-setup-for-integration.patch
* mm-memcontrol-make-swap-tracking-an-integral-part-of-memory-control.patch
* mm-memcontrol-charge-swapin-pages-on-instantiation.patch
* mm-memcontrol-document-the-new-swap-control-behavior.patch
* mm-memcontrol-delete-unused-lrucare-handling.patch
* mm-memcontrol-update-page-mem_cgroup-stability-rules.patch
* mm-fix-lru-balancing-effect-of-new-transparent-huge-pages.patch
* mm-keep-separate-anon-and-file-statistics-on-page-reclaim-activity.patch
* mm-allow-swappiness-that-prefers-reclaiming-anon-over-the-file-workingset.patch
* mm-fold-and-remove-lru_cache_add_anon-and-lru_cache_add_file.patch
* mm-workingset-let-cache-workingset-challenge-anon.patch
* mm-remove-use-once-cache-bias-from-lru-balancing.patch
* mm-vmscan-drop-unnecessary-div0-avoidance-rounding-in-get_scan_count.patch
* mm-base-lru-balancing-on-an-explicit-cost-model.patch
* mm-deactivations-shouldnt-bias-the-lru-balance.patch
* mm-only-count-actual-rotations-as-lru-reclaim-cost.patch
* mm-balance-lru-lists-based-on-relative-thrashing.patch
* mm-vmscan-determine-anon-file-pressure-balance-at-the-reclaim-root.patch
* mm-vmscan-reclaim-writepage-is-io-cost.patch
* mm-vmscan-limit-the-range-of-lru-type-balancing.patch
* mm-swap-fix-vmstats-for-huge-pages.patch
* mm-swap-memcg-fix-memcg-stats-for-huge-pages.patch
* tools-vm-page_owner_sort-filter-out-unneeded-line.patch
* mm-mempolicy-fix-up-gup-usage-in-lookup_node.patch
* mm-memblock-fix-minor-typo-and-unclear-comment.patch
* sparc32-register-memory-occupied-by-kernel-as-memblockmemory.patch
* hugetlbfs-get-unmapped-area-below-task_unmapped_base-for-hugetlbfs.patch
* mm-thp-dont-need-drain-lru-cache-when-splitting-and-mlocking-thp.patch
* powerpc-mm-drop-platform-defined-pmd_mknotpresent.patch
* mm-thp-rename-pmd_mknotpresent-as-pmd_mknotvalid.patch
* drivers-base-memoryc-cache-memory-blocks-in-xarray-to-accelerate-lookup.patch
* mm-add-debug_wx-support.patch
* riscv-support-debug_wx.patch
* x86-mm-use-arch_has_debug_wx-instead-of-arch-defined.patch
* arm64-mm-use-arch_has_debug_wx-instead-of-arch-defined.patch
* checkpatch-test-git_dir-changes.patch
* proc-kpageflags-prevent-an-integer-overflow-in-stable_page_flags.patch
* proc-kpageflags-do-not-use-uninitialized-struct-pages.patch
* kcov-cleanup-debug-messages.patch
* kcov-fix-potential-use-after-free-in-kcov_remote_start.patch
* kcov-move-t-kcov-assignments-into-kcov_start-stop.patch
* kcov-move-t-kcov_sequence-assignment.patch
* kcov-use-t-kcov_mode-as-enabled-indicator.patch
* kcov-collect-coverage-from-interrupts.patch
* usb-core-kcov-collect-coverage-from-usb-complete-callback.patch
* lib-lzo-fix-ambiguous-encoding-bug-in-lzo-rle.patch
* ocfs2-clear-links-count-in-ocfs2_mknod-if-an-error-occurs.patch
* ocfs2-fix-ocfs2-corrupt-when-iputting-an-inode.patch
* drivers-tty-serial-sh-scic-suppress-uninitialized-var-warning.patch
* ramfs-support-o_tmpfile.patch
* kernel-watchdog-flush-all-printk-nmi-buffers-when-hardlockup-detected.patch mm.patch
* mm-mmap-fix-the-adjusted-length-error.patch
* mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations.patch
* mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix.patch
* mm-add-comments-on-pglist_data-zones.patch
* arch-kmap-remove-bug_on.patch
* arch-xtensa-move-kmap-build-bug-out-of-the-way.patch
* arch-kmap-remove-redundant-arch-specific-kmaps.patch
* arch-kunmap-remove-duplicate-kunmap-implementations.patch
* arch-kunmap-remove-duplicate-kunmap-implementations-fix.patch
* x86powerpcmicroblaze-kmap-move-preempt-disable.patch
* arch-kmap_atomic-consolidate-duplicate-code.patch
* arch-kmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch
* arch-kunmap_atomic-consolidate-duplicate-code.patch
* arch-kunmap_atomic-consolidate-duplicate-code-fix.patch
* arch-kunmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch
* arch-kmap-ensure-kmap_prot-visibility.patch
* arch-kmap-dont-hard-code-kmap_prot-values.patch
* arch-kmap-define-kmap_atomic_prot-for-all-archs.patch
* drm-remove-drm-specific-kmap_atomic-code.patch
* drm-remove-drm-specific-kmap_atomic-code-fix.patch
* kmap-remove-kmap_atomic_to_page.patch
* parisc-kmap-remove-duplicate-kmap-code.patch
* sparc-remove-unnecessary-includes.patch
* kmap-consolidate-kmap_prot-definitions.patch
* kmap-consolidate-kmap_prot-definitions-checkpatch-fixes.patch
* mm-vmstat-add-events-for-pmd-based-thp-migration-without-split.patch
* mm-vmstat-add-events-for-pmd-based-thp-migration-without-split-fix.patch
* mm-vmstat-add-events-for-pmd-based-thp-migration-without-split-update.patch
* mm-add-kvfree_sensitive-for-freeing-sensitive-data-objects.patch
* mm-memory_hotplug-refrain-from-adding-memory-into-an-impossible-node.patch
* powerpc-pseries-hotplug-memory-stop-checking-is_mem_section_removable.patch
* mm-memory_hotplug-remove-is_mem_section_removable.patch
* mm-memory_hotplug-set-node_start_pfn-of-hotadded-pgdat-to-0.patch
* mm-memory_hotplug-handle-memblocks-only-with-config_arch_keep_memblock.patch
* mm-memory_hotplug-introduce-add_memory_driver_managed.patch
* kexec_file-dont-place-kexec-images-on-ioresource_mem_driver_managed.patch
* device-dax-add-memory-via-add_memory_driver_managed.patch
* mm-replace-zero-length-array-with-flexible-array-member.patch
* mm-replace-zero-length-array-with-flexible-array-member-fix.patch
* mm-memory_hotplug-fix-a-typo-in-comment-recoreded-recorded.patch
* mm-ksm-fix-a-typo-in-comment-alreaady-already.patch
* mm-ksm-fix-a-typo-in-comment-alreaady-already-v2.patch
* mm-mmap-fix-a-typo-in-comment-compatbility-compatibility.patch
* mm-hugetlb-fix-a-typo-in-comment-manitained-maintained.patch
* mm-hugetlb-fix-a-typo-in-comment-manitained-maintained-v2.patch
* mm-hugetlb-fix-a-typo-in-comment-manitained-maintained-v2-checkpatch-fixes.patch
* mm-vmsan-fix-some-typos-in-comment.patch
* mm-compaction-fix-a-typo-in-comment-pessemistic-pessimistic.patch
* mm-memblock-fix-a-typo-in-comment-implict-implicit.patch
* mm-list_lru-fix-a-typo-in-comment-numbesr-numbers.patch
* mm-filemap-fix-a-typo-in-comment-unneccssary-unnecessary.patch
* mm-frontswap-fix-some-typos-in-frontswapc.patch
* mm-memcg-fix-some-typos-in-memcontrolc.patch
* mm-fix-a-typo-in-comment-strucure-structure.patch
* mm-slub-fix-a-typo-in-comment-disambiguiation-disambiguation.patch
* mm-sparse-fix-a-typo-in-comment-convienence-convenience.patch
* mm-page-writeback-fix-a-typo-in-comment-effictive-effective.patch
* mm-memory-fix-a-typo-in-comment-attampt-attempt.patch
* mm-use-false-for-bool-variable.patch
* mm-return-true-in-cpupid_pid_unset.patch
* zcomp-use-array_size-for-backends-list.patch
* info-task-hung-in-generic_file_write_iter.patch
* info-task-hung-in-generic_file_write-fix.patch
* kernel-hung_taskc-monitor-killed-tasks.patch
* proc-rename-catch-function-argument.patch
* x86-mm-define-mm_p4d_folded.patch
* mm-debug-add-tests-validating-architecture-page-table-helpers.patch
* mm-debug-add-tests-validating-architecture-page-table-helpers-v17.patch
* mm-debug-add-tests-validating-architecture-page-table-helpers-v18.patch
* userc-make-uidhash_table-static.patch
* get_maintainer-add-email-addresses-from-yaml-files.patch
* get_maintainer-fix-unexpected-behavior-for-path-to-file-double-slashes.patch
* lib-math-avoid-trailing-n-hidden-in-pr_fmt.patch
* lib-add-might_fault-to-strncpy_from_user.patch
* lib-optimize-cpumask_local_spread.patch
* lib-test_lockupc-make-test_inode-static.patch
* lib-zlib-remove-outdated-and-incorrect-pre-increment-optimization.patch
* percpu_ref-use-a-more-common-logging-style.patch
* lib-flex_proportionsc-cleanup-__fprop_inc_percpu_max.patch
* lib-make-a-test-module-with-set-clear-bit.patch
* bitops-avoid-clang-shift-count-overflow-warnings.patch
* bitops-simplify-get_count_order_long.patch
* bitops-use-the-same-mechanism-for-get_count_order.patch
* lib-test-get_count_order-long-in-test_bitopsc.patch
* lib-test-get_count_order-long-in-test_bitopsc-fix.patch
* checkpatch-additional-maintainer-section-entry-ordering-checks.patch
* checkpatch-look-for-c99-comments-in-ctx_locate_comment.patch
* checkpatch-disallow-git-and-file-fix.patch
* checkpatch-use-patch-subject-when-reading-from-stdin.patch
* checkpatch-use-patch-subject-when-reading-from-stdin-fix.patch
* fs-binfmt_elf-remove-redundant-elf_map-ifndef.patch
* elfnote-mark-all-note-sections-shf_alloc.patch
* init-allow-distribution-configuration-of-default-init.patch
* fat-dont-allow-to-mount-if-the-fat-length-==-0.patch
* fat-improve-the-readahead-for-fat-entries.patch
* fs-seq_filec-seq_read-update-pr_info_ratelimited.patch
* seq_file-introduce-define_seq_attribute-helper-macro.patch
* seq_file-introduce-define_seq_attribute-helper-macro-checkpatch-fixes.patch
* mm-vmstat-convert-to-use-define_seq_attribute-macro.patch
* kernel-kprobes-convert-to-use-define_seq_attribute-macro.patch
* exec-simplify-the-copy_strings_kernel-calling-convention.patch
* exec-open-code-copy_string_kernel.patch
* exec-change-uselib2-is_sreg-failure-to-eacces.patch
* exec-relocate-s_isreg-check.patch
* exec-relocate-path_noexec-check.patch
* fs-include-fmode_exec-when-converting-flags-to-f_mode.patch
* umh-fix-refcount-underflow-in-fork_usermode_blob.patch
* rapidio-avoid-data-race-between-file-operation-callbacks-and-mport_cdev_add.patch
* rapidio-convert-get_user_pages-pin_user_pages.patch
* relay-handle-alloc_percpu-returning-null-in-relay_open.patch
* kernel-relayc-fix-read_pos-error-when-multiple-readers.patch
* aio-simplify-read_events.patch
* selftests-x86-pkeys-move-selftests-to-arch-neutral-directory.patch
* selftests-vm-pkeys-rename-all-references-to-pkru-to-a-generic-name.patch
* selftests-vm-pkeys-move-generic-definitions-to-header-file.patch
* selftests-vm-pkeys-move-some-definitions-to-arch-specific-header.patch
* selftests-vm-pkeys-make-gcc-check-arguments-of-sigsafe_printf.patch
* selftests-vm-pkeys-use-sane-types-for-pkey-register.patch
* selftests-vm-pkeys-add-helpers-for-pkey-bits.patch
* selftests-vm-pkeys-fix-pkey_disable_clear.patch
* selftests-vm-pkeys-fix-assertion-in-pkey_disable_set-clear.patch
* selftests-vm-pkeys-fix-alloc_random_pkey-to-make-it-really-random.patch
* selftests-vm-pkeys-use-the-correct-huge-page-size.patch
* selftests-vm-pkeys-introduce-generic-pkey-abstractions.patch
* selftests-vm-pkeys-introduce-powerpc-support.patch
* selftests-vm-pkeys-introduce-powerpc-support-fix.patch
* selftests-vm-pkeys-fix-number-of-reserved-powerpc-pkeys.patch
* selftests-vm-pkeys-fix-assertion-in-test_pkey_alloc_exhaust.patch
* selftests-vm-pkeys-improve-checks-to-determine-pkey-support.patch
* selftests-vm-pkeys-associate-key-on-a-mapped-page-and-detect-access-violation.patch
* selftests-vm-pkeys-associate-key-on-a-mapped-page-and-detect-write-violation.patch
* selftests-vm-pkeys-detect-write-violation-on-a-mapped-access-denied-key-page.patch
* selftests-vm-pkeys-introduce-a-sub-page-allocator.patch
* selftests-vm-pkeys-test-correct-behaviour-of-pkey-0.patch
* selftests-vm-pkeys-override-access-right-definitions-on-powerpc.patch
* selftests-vm-pkeys-override-access-right-definitions-on-powerpc-fix.patch
* selftests-vm-pkeys-use-the-correct-page-size-on-powerpc.patch
* selftests-vm-pkeys-fix-multilib-builds-for-x86.patch
* tools-testing-selftests-vm-remove-duplicate-headers.patch
* ubsan-fix-gcc-10-warnings.patch
* ipc-msg-add-missing-annotation-for-freeque.patch
* ipc-use-a-work-queue-to-free_ipc.patch
* ipc-convert-ipcs_idr-to-xarray.patch
* ipc-convert-ipcs_idr-to-xarray-update.patch
* ipc-convert-ipcs_idr-to-xarray-update-fix.patch
* linux-next-pre.patch
linux-next.patch
linux-next-rejects.patch
linux-next-git-rejects.patch
* linux-next-post.patch
* dynamic_debug-add-an-option-to-enable-dynamic-debug-for-modules-only.patch
* dynamic_debug-add-an-option-to-enable-dynamic-debug-for-modules-only-v2.patch
* kernel-add-panic_on_taint.patch
* kernel-add-panic_on_taint-fix.patch
* xarrayh-correct-return-code-for-xa_store_bhirq.patch
* kernel-sysctl-support-setting-sysctl-parameters-from-kernel-command-line.patch
* kernel-sysctl-support-handling-command-line-aliases.patch
* kernel-hung_task-convert-hung_task_panic-boot-parameter-to-sysctl.patch
* tools-testing-selftests-sysctl-sysctlsh-support-config_test_sysctl=y.patch
* lib-test_sysctl-support-testing-of-sysctl-boot-parameter.patch
* lib-test_sysctl-support-testing-of-sysctl-boot-parameter-fix.patch
* kernel-watchdogc-convert-soft-hardlockup-boot-parameters-to-sysctl-aliases.patch
* kernel-hung_taskc-introduce-sysctl-to-print-all-traces-when-a-hung-task-is-detected.patch
* panic-add-sysctl-to-dump-all-cpus-backtraces-on-oops-event.patch
* kernel-sysctl-ignore-out-of-range-taint-bits-introduced-via-kerneltainted.patch
* stacktrace-cleanup-inconsistent-variable-type.patch
* amdgpu-a-null-mm-does-not-mean-a-thread-is-a-kthread.patch
* kernel-move-use_mm-unuse_mm-to-kthreadc.patch
* kernel-move-use_mm-unuse_mm-to-kthreadc-v2.patch
* kernel-better-document-the-use_mm-unuse_mm-api-contract.patch
* kernel-better-document-the-use_mm-unuse_mm-api-contract-v2.patch
* kernel-better-document-the-use_mm-unuse_mm-api-contract-v2-fix.patch
* kernel-better-document-the-use_mm-unuse_mm-api-contract-fix-2.patch
* kernel-set-user_ds-in-kthread_use_mm.patch
* mm-kmemleak-silence-kcsan-splats-in-checksum.patch
* kallsyms-printk-add-loglvl-to-print_ip_sym.patch
* alpha-add-show_stack_loglvl.patch
* arc-add-show_stack_loglvl.patch
* arm-asm-add-loglvl-to-c_backtrace.patch
* arm-add-loglvl-to-unwind_backtrace.patch
* arm-add-loglvl-to-dump_backtrace.patch
* arm-wire-up-dump_backtrace_entrystm.patch
* arm-add-show_stack_loglvl.patch
* arm64-add-loglvl-to-dump_backtrace.patch
* arm64-add-show_stack_loglvl.patch
* c6x-add-show_stack_loglvl.patch
* csky-add-show_stack_loglvl.patch
* h8300-add-show_stack_loglvl.patch
* hexagon-add-show_stack_loglvl.patch
* ia64-pass-log-level-as-arg-into-ia64_do_show_stack.patch
* ia64-add-show_stack_loglvl.patch
* m68k-add-show_stack_loglvl.patch
* microblaze-add-loglvl-to-microblaze_unwind_inner.patch
* microblaze-add-loglvl-to-microblaze_unwind.patch
* microblaze-add-show_stack_loglvl.patch
* mips-add-show_stack_loglvl.patch
* nds32-add-show_stack_loglvl.patch
* nios2-add-show_stack_loglvl.patch
* openrisc-add-show_stack_loglvl.patch
* parisc-add-show_stack_loglvl.patch
* powerpc-add-show_stack_loglvl.patch
* riscv-add-show_stack_loglvl.patch
* s390-add-show_stack_loglvl.patch
* sh-add-loglvl-to-dump_mem.patch
* sh-remove-needless-printk.patch
* sh-add-loglvl-to-printk_address.patch
* sh-add-loglvl-to-show_trace.patch
* sh-add-show_stack_loglvl.patch
* sparc-add-show_stack_loglvl.patch
* um-sysrq-remove-needless-variable-sp.patch
* um-add-show_stack_loglvl.patch
* unicore32-remove-unused-pmode-argument-in-c_backtrace.patch
* unicore32-add-loglvl-to-c_backtrace.patch
* unicore32-add-show_stack_loglvl.patch
* x86-add-missing-const-qualifiers-for-log_lvl.patch
* x86-add-show_stack_loglvl.patch
* xtensa-add-loglvl-to-show_trace.patch
* xtensa-add-loglvl-to-show_trace-fix.patch
* xtensa-add-show_stack_loglvl.patch
* sysrq-use-show_stack_loglvl.patch
* x86-amd_gart-print-stacktrace-for-a-leak-with-kern_err.patch
* power-use-show_stack_loglvl.patch
* kdb-dont-play-with-console_loglevel.patch
* sched-print-stack-trace-with-kern_info.patch
* kernel-use-show_stack_loglvl.patch
* kernel-rename-show_stack_loglvl-=-show_stack.patch
* mm-frontswap-mark-various-intentional-data-races.patch
* mm-page_io-mark-various-intentional-data-races.patch
* mm-page_io-mark-various-intentional-data-races-v2.patch
* mm-swap_state-mark-various-intentional-data-races.patch
* mm-filemap-fix-a-data-race-in-filemap_fault.patch
* mm-swapfile-fix-and-annotate-various-data-races.patch
* mm-swapfile-fix-and-annotate-various-data-races-v2.patch
* mm-page_counter-fix-various-data-races-at-memsw.patch
* mm-memcontrol-fix-a-data-race-in-scan-count.patch
* mm-list_lru-fix-a-data-race-in-list_lru_count_one.patch
* mm-mempool-fix-a-data-race-in-mempool_free.patch
* mm-util-annotate-an-data-race-at-vm_committed_as.patch
* mm-rmap-annotate-a-data-race-at-tlb_flush_batched.patch
* mm-annotate-a-data-race-in-page_zonenum.patch
* mm-swap-annotate-data-races-for-lru_rotate_pvecs.patch
* mm-gupc-convert-to-use-get_user_pagepages_fast_only.patch
* mm-gup-update-pin_user_pagesrst-for-case-3-mmu-notifiers.patch
* mm-gup-introduce-pin_user_pages_locked.patch
* mm-gup-introduce-pin_user_pages_locked-v2.patch
* mm-gup-frame_vector-convert-get_user_pages-pin_user_pages.patch
* mm-gup-documentation-fix-for-pin_user_pages-apis.patch
* docs-mm-gup-pin_user_pagesrst-add-a-case-5.patch
* vhost-convert-get_user_pages-pin_user_pages.patch
* h8300-remove-usage-of-__arch_use_5level_hack.patch
* arm-add-support-for-folded-p4d-page-tables.patch
* arm-add-support-for-folded-p4d-page-tables-fix.patch
* arm64-add-support-for-folded-p4d-page-tables.patch
* arm64-add-support-for-folded-p4d-page-tables-fix.patch
* hexagon-remove-__arch_use_5level_hack.patch
* ia64-add-support-for-folded-p4d-page-tables.patch
* nios2-add-support-for-folded-p4d-page-tables.patch
* openrisc-add-support-for-folded-p4d-page-tables.patch
* powerpc-add-support-for-folded-p4d-page-tables.patch
* powerpc-add-support-for-folded-p4d-page-tables-fix-2.patch
* sh-fault-modernize-printing-of-kernel-messages.patch
* sh-drop-__pxd_offset-macros-that-duplicate-pxd_index-ones.patch
* sh-add-support-for-folded-p4d-page-tables.patch
* unicore32-remove-__arch_use_5level_hack.patch
* asm-generic-remove-pgtable-nop4d-hackh.patch
* mm-remove-__arch_has_5level_hack-and-include-asm-generic-5level-fixuph.patch
* net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy.patch
* mm-mmapc-add-more-sanity-checks-to-get_unmapped_area.patch
* mm-mmapc-do-not-allow-mappings-outside-of-allowed-limits.patch
* mm-dont-include-asm-pgtableh-if-linux-mmh-is-already-included.patch
* mm-introduce-include-linux-pgtableh.patch
* mm-reorder-includes-after-introduction-of-linux-pgtableh.patch
* csky-replace-definitions-of-__pxd_offset-with-pxd_index.patch
* m68k-mm-motorola-move-comment-about-page-table-allocation-funcitons.patch
* m68k-mm-move-cachenocahe_page-definitions-close-to-their-user.patch
* x86-mm-simplify-init_trampoline-and-surrounding-logic.patch
* x86-mm-simplify-init_trampoline-and-surrounding-logic-fix.patch
* mm-pgtable-add-shortcuts-for-accessing-kernel-pmd-and-pte.patch
* mm-pgtable-add-shortcuts-for-accessing-kernel-pmd-and-pte-fix.patch
* mm-consolidate-pte_index-and-pte_offset_-definitions.patch
* mm-consolidate-pmd_index-and-pmd_offset-definitions.patch
* mm-consolidate-pud_index-and-pud_offset-definitions.patch
* mm-consolidate-pgd_index-and-pgd_offset_k-definitions.patch
* mm-consolidate-pgd_index-and-pgd_offset_k-definitions-fix.patch
* arm-fix-the-flush_icache_range-arguments-in-set_fiq_handler.patch
* nds32-unexport-flush_icache_page.patch
* powerpc-unexport-flush_icache_user_range.patch
* unicore32-remove-flush_cache_user_range.patch
* asm-generic-fix-the-inclusion-guards-for-cacheflushh.patch
* asm-generic-dont-include-linux-mmh-in-cacheflushh.patch
* asm-generic-dont-include-linux-mmh-in-cacheflushh-fix.patch
* asm-generic-improve-the-flush_dcache_page-stub.patch
* alpha-use-asm-generic-cacheflushh.patch
* arm64-use-asm-generic-cacheflushh.patch
* c6x-use-asm-generic-cacheflushh.patch
* hexagon-use-asm-generic-cacheflushh.patch
* ia64-use-asm-generic-cacheflushh.patch
* microblaze-use-asm-generic-cacheflushh.patch
* m68knommu-use-asm-generic-cacheflushh.patch
* openrisc-use-asm-generic-cacheflushh.patch
* powerpc-use-asm-generic-cacheflushh.patch
* riscv-use-asm-generic-cacheflushh.patch
* armsparcunicore32-remove-flush_icache_user_range.patch
* mm-rename-flush_icache_user_range-to-flush_icache_user_page.patch
* asm-generic-add-a-flush_icache_user_range-stub.patch
* sh-implement-flush_icache_user_range.patch
* xtensa-implement-flush_icache_user_range.patch
* xtensa-implement-flush_icache_user_range-fix.patch
* arm-rename-flush_cache_user_range-to-flush_icache_user_range.patch
* m68k-implement-flush_icache_user_range.patch
* exec-only-build-read_code-when-needed.patch
* exec-use-flush_icache_user_range-in-read_code.patch
* binfmt_flat-use-flush_icache_user_range.patch
* nommu-use-flush_icache_user_range-in-brk-and-mmap.patch
* module-move-the-set_fs-hack-for-flush_icache_range-to-m68k.patch
* mmap-locking-api-initial-implementation-as-rwsem-wrappers.patch
* mmu-notifier-use-the-new-mmap-locking-api.patch
* dma-reservations-use-the-new-mmap-locking-api.patch
* mmap-locking-api-use-coccinelle-to-convert-mmap_sem-rwsem-call-sites.patch
* mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle.patch
* mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix.patch
* mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix-fix.patch
* mmap-locking-api-convert-mmap_sem-call-sites-missed-by-coccinelle-fix-fix-fix.patch
* mmap-locking-api-convert-nested-write-lock-sites.patch
* mmap-locking-api-add-mmap_read_trylock_non_owner.patch
* mmap-locking-api-add-mmap_lock_initializer.patch
* mmap-locking-api-add-mmap_assert_locked-and-mmap_assert_write_locked.patch
* mmap-locking-api-rename-mmap_sem-to-mmap_lock.patch
* mmap-locking-api-rename-mmap_sem-to-mmap_lock-fix.patch
* mmap-locking-api-convert-mmap_sem-api-comments.patch
* mmap-locking-api-convert-mmap_sem-comments.patch
* mmap-locking-api-convert-mmap_sem-comments-fix.patch
* mmap-locking-api-convert-mmap_sem-comments-fix-fix.patch
* mmap-locking-api-convert-mmap_sem-comments-fix-fix-fix.patch
* mm-pass-task-and-mm-to-do_madvise.patch
* mm-introduce-external-memory-hinting-api.patch
* mm-introduce-external-memory-hinting-api-fix.patch
* mm-introduce-external-memory-hinting-api-fix-2.patch
* mm-introduce-external-memory-hinting-api-fix-2-fix.patch
* mm-check-fatal-signal-pending-of-target-process.patch
* pid-move-pidfd_get_pid-function-to-pidc.patch
* mm-support-both-pid-and-pidfd-for-process_madvise.patch
* mm-madvise-allow-ksm-hints-for-remote-api.patch
* mm-support-vector-address-ranges-for-process_madvise.patch
* mm-support-vector-address-ranges-for-process_madvise-fix.patch
* mm-support-vector-address-ranges-for-process_madvise-fix-fix.patch
* mm-support-vector-address-ranges-for-process_madvise-fix-fix-fix.patch
* mm-support-vector-address-ranges-for-process_madvise-fix-fix-fix-fix.patch
* mm-support-vector-address-ranges-for-process_madvise-fix-fix-fix-fix-fix.patch
* mm-use-only-pidfd-for-process_madvise-syscall.patch
* mm-use-only-pidfd-for-process_madvise-syscall-fix.patch
* mm-remove-duplicated-include-from-madvisec.patch
* maccess-unexport-probe_kernel_write-and-probe_user_write.patch
* maccess-unexport-probe_kernel_write-and-probe_user_write-fix.patch
* maccess-remove-various-unused-weak-aliases.patch
* maccess-remove-duplicate-kerneldoc-comments.patch
* maccess-clarify-kerneldoc-comments.patch
* maccess-update-the-top-of-file-comment.patch
* maccess-rename-strncpy_from_unsafe_user-to-strncpy_from_user_nofault.patch
* maccess-rename-strncpy_from_unsafe_strict-to-strncpy_from_kernel_nofault.patch
* maccess-rename-strnlen_unsafe_user-to-strnlen_user_nofault.patch
* maccess-remove-probe_read_common-and-probe_write_common.patch
* maccess-unify-the-probe-kernel-arch-hooks.patch
* maccess-unify-the-probe-kernel-arch-hooks-fix.patch
* bpf-factor-out-a-bpf_trace_copy_string-helper.patch
* bpf-handle-the-compat-string-in-bpf_trace_copy_string-better.patch
* bpf-bpf_seq_printf-handle-potentially-unsafe-format-string-better.patch
* bpf-rework-the-compat-kernel-probe-handling.patch
* tracing-kprobes-handle-mixed-kernel-userspace-probes-better.patch
* maccess-remove-strncpy_from_unsafe.patch
* maccess-always-use-strict-semantics-for-probe_kernel_read.patch
* maccess-always-use-strict-semantics-for-probe_kernel_read-fix.patch
* maccess-move-user-access-routines-together.patch
* maccess-allow-architectures-to-provide-kernel-probing-directly.patch
* x86-use-non-set_fs-based-maccess-routines.patch
* x86-use-non-set_fs-based-maccess-routines-checkpatch-fixes.patch
* maccess-return-erange-when-copy_from_kernel_nofault_allowed-fails.patch
* mm-expand-documentation-over-__read_mostly.patch
* doc-cgroup-update-note-about-conditions-when-oom-killer-is-invoked.patch
* doc-cgroup-update-note-about-conditions-when-oom-killer-is-invoked-fix.patch
* sh-sh4a-bring-back-tmu3_device-early-device.patch
* arch-sh-vmlinuxscr-align-rodata.patch
* include-asm-generic-vmlinuxldsh-align-ro_after_init.patch
* sh-clkfwk-remove-r8-r16-r32.patch
* sh-remove-call-to-memset-after-dma_alloc_coherent.patch
* sh-use-generic-strncpy.patch
* sh-convert-ins-outs-macros-to-inline-functions.patch
* sh-convert-ins-outs-macros-to-inline-functions-checkpatch-fixes.patch
* sh-convert-iounmap-macros-to-inline-functions.patch
* sh-add-missing-export_symbol-for-__delay.patch make-sure-nobodys-leaking-resources.patch releasing-resources-with-children.patch mutex-subsystem-synchro-test-module.patch kernel-forkc-export-kernel_thread-to-modules.patch workaround-for-a-pci-restoring-bug.patch
+ mm-vmalloc-fix-a-typo-in-comment.patch added to -mm tree
In reply to Andrew MortonThe patch titled Subject: mm/vmalloc: fix a typo in comment has been added to the -mm tree. Its filename is mm-vmalloc-fix-a-typo-in-comment.patch
The patch titled
Subject: mm/vmalloc: fix a typo in comment
has been added to the -mm tree. Its filename is
mm-vmalloc-fix-a-typo-in-comment.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-vmalloc-fix-a-typo-in-comment.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-vmalloc-fix-a-typo-in-comment.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated there every 3-4 working days
------------------------------------------------------
From: Jeongtae Park <[email protected]>
Subject: mm/vmalloc: fix a typo in comment
There is a typo in comment, fix it.
"nother" -> "another"
---
mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-fix-a-typo-in-comment +1 -1
@@ -2317,7 +2317,7 @@ static inline void __vfree_deferred(cons* Use raw_cpu_ptr() because this can be called from preemptible * context. Preemption is absolutely fine here, because the llist_add() * implementation is lockless, so it works even if we are adding to -* nother cpu's list. schedule_work() should be fine with this too. +* another cpu's list. schedule_work() should be fine with this too. */ struct vfree_deferred *p =raw_cpu_ptr (&vfree_deferred);
_
Patches currently in -mm which might be from [email protected] are
mm-vmalloc-fix-a-typo-in-comment.patch
+ memory_hotplug-disable-the-functionality-for-32b.patch added to -mm tree
In reply to Andrew MortonThe patch titled Subject: mm/memory_hotplug: disable the functionality for 32b has been added to the -mm tree. Its filename is memory_hotplug-disable-the-functionality-for-32b.patch
The patch titled
Subject: mm/memory_hotplug: disable the functionality for 32b
has been added to the -mm tree. Its filename is
memory_hotplug-disable-the-functionality-for-32b.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/memory_hotplug-disable-the-functionality-for-32b.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/memory_hotplug-disable-the-functionality-for-32b.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated there every 3-4 working days
------------------------------------------------------
From: Michal Hocko <[email protected]>
Subject: mm/memory_hotplug: disable the functionality for 32b
Memory hotlug is broken for 32b systems at least since c6f03e2903c9 ("mm,memory_hotplug: remove zone restrictions") which has considerably reworked how can be memory associated with movable/kernel zones. The same is not really trivial to achieve in 32b where only lowmem is the kernel zone. While we can tweak this immediate problem around there are likely other land mines hidden at other places.
It is also quite dubious that there is a real usecase for the memory hotplug on 32b in the first place. Low memory is just too small to be hotplugable (for hot add) and generally unusable for hotremove. Adding more memory to highmem is also dubious because it would increase the low mem or vmalloc space pressure for memmaps.
Restrict the functionality to 64b systems. This will help future development to focus on usecases that have real life application. We can remove this restriction in future in presence of a real life usecase of course but until then make it explicit that hotplug on 32b is broken and requires a non trivial amount of work to fix.
Robin said:
: 32-bit Arm doesn't support memory hotplug, and as far as I'm aware there's : little likelihood of it ever wanting to. FWIW it looks like SuperH is the : only pure-32-bit architecture to have hotplug support at all.
---
mm/Kconfig | 1 + 1 file changed, 1 insertion(+)
mm/Kconfig renamed from mm/Kconfig~memory_hotplug-disable-the-functionality-for-32b +1 -0
@@ -155,6 +155,7 @@ config MEMORY_HOTPLUG bool "Allow for memory hot-add" depends on SPARSEMEM || X86_64_ACPI_NUMA depends on ARCH_ENABLE_MEMORY_HOTPLUG + depends on 64BIT || BROKEN select NUMA_KEEP_MEMINFO if NUMA config MEMORY_HOTPLUG_SPARSE
_
Patches currently in -mm which might be from [email protected] are
memory_hotplug-disable-the-functionality-for-32b.patch
+ mm-utilc-remove-the-vm_warn_once-for-vm_committed_as-underflow-check.patch added to -mm tree
In reply to Andrew MortonThe patch titled Subject: mm/util.c: remove the VM_WARN_ONCE for vm_committed_as underflow check has been added to the -mm tree. Its filename is…
The patch titled
Subject: mm/util.c: remove the VM_WARN_ONCE for vm_committed_as underflow check
has been added to the -mm tree. Its filename is
mm-utilc-remove-the-vm_warn_once-for-vm_committed_as-underflow-check.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-utilc-remove-the-vm_warn_once-for-vm_committed_as-underflow-check.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-utilc-remove-the-vm_warn_once-for-vm_committed_as-underflow-check.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated there every 3-4 working days
------------------------------------------------------
From: Feng Tang <[email protected]>
Subject: mm/util.c: remove the VM_WARN_ONCE for vm_committed_as underflow check
This check was added by 82f71ae4a2b8 ("mm: catch memory commitment underflow") in 2014 to have a safety check for issues which have been fixed. And there has been few report caught by it, as described in its commit log:
: This shouldn't happen any more - the previous two patches fixed : the committed_as underflow issues.
But it was really found by Qian Cai when he used the LTP memory stress suite to test a RFC patchset, which tries to improve scalability of per-cpu counter 'vm_committed_as', by chosing a bigger 'batch' number for loose overcommit policies (OVERCOMMIT_ALWAYS and OVERCOMMIT_GUESS), while keeping current number for OVERCOMMIT_NEVER.
With that patchset, when system firstly uses a loose policy, the 'vm_committed_as' count could be a big negative value, as its big 'batch' number allows a big deviation, then when the policy is changed to OVERCOMMIT_NEVER, the 'batch' will be decreased to a much smaller value, thus hits this WARN check.
To mitigate this, one proposed solution is to queue work on all online CPUs to do a local sync for 'vm_committed_as' when changing policy to OVERCOMMIT_NEVER, plus some global syncing to garante the case won't be hit.
But this solution is costy and slow, given this check hasn't shown real trouble or benefit, simply drop it from one hot path of MM. And perf stats does show some tiny saving for removing it.
---
mm/util.c | 4 ---- 1 file changed, 4 deletions(-)
mm/util.c renamed from mm/util.c~mm-utilc-remove-the-vm_warn_once-for-vm_committed_as-underflow-check +0 -3
@@ -796,10 +796,6 @@ int __vm_enough_memory(struct mm_struct {long allowed; -VM_WARN_ONCE (percpu_counter_read (&vm_committed_as) < - -(s64 )vm_committed_as_batch *num_online_cpus (), -"memory commitment underflow" );
[folded-merged] mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix.patch removed from -mm tree
In reply to Andrew MortonThe patch titled Subject: mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix has been removed from the -mm tree. Its filename was…
The patch titled
Subject: mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix
has been removed from the -mm tree. Its filename was
mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix.patch
This patch was dropped because it was folded into mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations.patch
------------------------------------------------------
From: Andrew Morton <[email protected]>
Subject: mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix
fix comment grammar, reflow comment
---
mm/page_alloc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations-fix +5 -5
@@ -3748,11 +3748,11 @@ retry: mark =wmark_pages (zone, alloc_flags &ALLOC_WMARK_MASK );/* * Allow GFP_ATOMIC order-0 allocations to exclude the -* zone->watermark_boost in its watermark calculations. -* We rely on the ALLOC_ flags set for GFP_ATOMIC -* requests in gfp_to_alloc_flags() for this. Reason not to -* use the GFP_ATOMIC directly is that we want to fall back -* to slow path thus wake up kswapd. +* zone->watermark_boost in their watermark calculations. +* We rely on the ALLOC_ flags set for GFP_ATOMIC requests in +* gfp_to_alloc_flags() for this. Reason not to use the +* GFP_ATOMIC directly is that we want to fall back to slow path +* thus wake up kswapd. */ if (unlikely (!order && !(alloc_flags &ALLOC_WMARK_MASK ) && (alloc_flags & (ALLOC_HARDER |ALLOC_HIGH )))) {
_
Patches currently in -mm which might be from [email protected] are
mm-page_alloc-skip-waternark_boost-for-atomic-order-0-allocations.patch arch-kunmap-remove-duplicate-kunmap-implementations-fix.patch arch-kmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch arch-kunmap_atomic-consolidate-duplicate-code-checkpatch-fixes.patch kmap-consolidate-kmap_prot-definitions-checkpatch-fixes.patch mm-replace-zero-length-array-with-flexible-array-member-fix.patch mm-hugetlb-fix-a-typo-in-comment-manitained-maintained-v2-checkpatch-fixes.patch