mm/madvise: enhance lazyfreeing with mTHP in madvise_free
Subsystems (3)
PATCH v3 1/1 mm/madvise: enhance lazyfreeing with mTHP in madvise_free
This patch optimizes lazyfreeing with PTE-mapped mTHP[1] (Inspired by David Hildenbrand[2]). We aim to avoid unnecessary folio splitting if the large folio is entirely within the given range.
On an Intel I5 CPU, lazyfreeing a 1GiB VMA backed by PTE-mapped folios of the same size results in the following runtimes for madvise(MADV_FREE) in seconds (shorter is better):
Folio Size | Old | New | Change
------------------------------------------
4KiB | 0.590251 | 0.590259 | 0%
16KiB | 2.990447 | 0.185655 | -94%
32KiB | 2.547831 | 0.104870 | -95%
64KiB | 2.457796 | 0.052812 | -97%
128KiB | 2.281034 | 0.032777 | -99%
256KiB | 2.230387 | 0.017496 | -99%
512KiB | 2.189106 | 0.010781 | -99%
1024KiB | 2.183949 | 0.007753 | -99%
2048KiB | 0.002799 | 0.002804 | 0%[1] https://lkml.kernel.org/r/[email protected]
[2] https://lore.kernel.org/linux-mm/[email protected]
---
This patch is rebased on Ryan's patchset[3] and follows a similar pattern to madvise_cold_or_pageout_pte_range() for madvise_free_pte_range().
[3] https://lore.kernel.org/linux-mm/[email protected]
The performance numbers are from v2. I did a quick benchmark run of v3 and nothing significantly changed.
v2 -> v3:
* Only skip all the PTEs for nr_pages when the number of batched PTEs matches nr_pages per Barry Song
* Change folio_pte_batch() to consume an optional *any_dirty and *any_young function, per David Hildenbrand
* Move the ptep_get_and_clear_full() loop into refresh_full_ptes(), per David Hildenbrand
* Follow a similar pattern for madvise_free_pte_range(), per Ryan Roberts
* https://lore.kernel.org/linux-mm/[email protected]
v1 -> v2:
* Update the performance numbers
* Update the changelog, suggested by Ryan Roberts
* Check the COW folio, suggested by Yin Fengwei
* Check if we are mapping all subpages, suggested by Barry Song, David Hildenbrand, Ryan Roberts
* https://lore.kernel.org/linux-mm/[email protected]
include/linux/pgtable.h | 48 ++++++++++++ mm/internal.h | 17 ++++- mm/madvise.c | 159 +++++++++++++++++++++------------------- mm/memory.c | 4 +- 4 files changed, 149 insertions(+), 79 deletions(-)
include/linux/pgtable.h +48 -0
@@ -736,6 +736,54 @@ static inline void clear_not_present_full_ptes(struct mm_struct *mm, }#endif +/* Flags for refresh_full_ptes(). */ +typedef int __bitwise rfp_t ; + +/* Refresh PTEs after pte_mkold(), droping the accessed bit. */ +#define RFP_CLEAR_YOUNG ((__force rfp_t)BIT(0)) + +/* Refresh PTEs after pte_mkclean(), droping the dirty bit. */ +#define RFP_CLEAR_DIRTY ((__force rfp_t)BIT(1)) + +/** +* refresh_full_ptes - refresh present PTEs that map consecutive pages of the +* same folio. +* @mm: Address space the pages are mapped into. +* @addr: Address the first page is mapped at. +* @ptep: Page table pointer for the first entry. +* @nr: Number of entries to refresh. +* @flags: Flags to indicate the PTE refresh semantics. +* @full: Whether we are clearing a full mm. +* +* Note that PTE bits in the PTE range besides the PFN can differ. For example, +* some PTEs might be write-protected. +* +* Context: The caller holds the page table lock. The PTEs map consecutive +* pages that belong to the same folio. The PTEs are all in the same PMD. +*/ +static inline void refresh_full_ptes (struct mm_struct *mm,unsigned long addr, +pte_t *ptep,unsigned int nr,rfp_t flags,int full) +{ +pte_t ptent; + +for (;;) { + ptent =ptep_get_and_clear_full (mm, addr, ptep, full); + +if (flags &RFP_CLEAR_YOUNG ) + ptent =pte_mkold (ptent); + +if (flags &RFP_CLEAR_DIRTY ) + ptent =pte_mkclean (ptent); + +set_pte_at (mm, addr, ptep, ptent); + +if (--nr ==0 ) +break ; + ptep++; + addr +=PAGE_SIZE ; + } +} +#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
mm/internal.h +15 -2
@@ -130,16 +130,20 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)*/ static inline int folio_pte_batch (struct folio *folio,unsigned long addr,pte_t *start_ptep,pte_t pte,int max_nr,fpb_t flags, -bool *any_writable) +bool *any_writable,bool *any_young,bool *any_dirty) {unsigned long folio_end_pfn =folio_pfn (folio) +folio_nr_pages (folio);const pte_t *end_ptep = start_ptep + max_nr;pte_t expected_pte, *ptep; -bool writable; +bool writable, young, dirty;int nr;if (any_writable) *any_writable = false; +if (any_young) + *any_young = false; +if (any_dirty) + *any_dirty = false;VM_WARN_ON_FOLIO (!pte_present (pte), folio);VM_WARN_ON_FOLIO (!folio_test_large (folio) || max_nr <1 , folio);
@@ -153,6 +157,11 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, pte =ptep_get (ptep);if (any_writable) writable = !!pte_write (pte); +if (any_young) + young = !!pte_young (pte); +if (any_dirty) + dirty = !!pte_dirty (pte); + pte =__pte_batch_clear_ignored (pte, flags);if (!pte_same (pte, expected_pte))
@@ -168,6 +177,10 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr,if (any_writable) *any_writable |= writable; +if (any_young) + *any_young |= young; +if (any_dirty) + *any_dirty |= dirty; nr =pte_batch_hint (ptep, pte); expected_pte =pte_advance_pfn (expected_pte, nr);
mm/madvise.c +84 -75
232 lines not on this page · raw message
mm/memory.c +2 -2
@@ -989,7 +989,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma flags |=FPB_IGNORE_SOFT_DIRTY ; nr =folio_pte_batch (folio, addr, src_pte, pte, max_nr, flags, - &any_writable); + &any_writable,NULL ,NULL );folio_ref_add (folio, nr);if (folio_test_anon (folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
@@ -1551,7 +1551,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb,*/ if (unlikely (folio_test_large (folio) && max_nr !=1 )) { nr =folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags, -NULL ); +NULL ,NULL ,NULL ); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush,
Signature · 1 line
2.33.1