mm: alloc_anon_folio: pass raw fault address to vma_alloc_folio
Subsystems (11)
A message that is not in the archive (sent off-list or not yet received): [email protected]
Michael S. Tsirkin
maintainerPATCH v6 8/30 mm: alloc_anon_folio: pass raw fault address to vma_alloc_folio
Pass vmf->address directly instead of ALIGN_DOWN(vmf->address, ...). vma_alloc_folio_noprof now aligns internally for NUMA interleave, and post_alloc_hook will use the raw address for cache-friendly zeroing via folio_zero_user().
--- mm/memory.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
mm/memory.c +1 -2
@@ -5252,8 +5252,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf)/* Try allocating the highest of the remaining orders. */ gfp =vma_thp_gfp_mask (vma);while (orders) { - addr =ALIGN_DOWN (vmf->address,PAGE_SIZE << order); - folio =vma_alloc_folio (gfp, order, vma, addr); + folio =vma_alloc_folio (gfp, order, vma, vmf->address);if (folio) {if (mem_cgroup_charge (folio, vma->vm_mm, gfp)) {count_mthp_stat (order,MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE );
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 9/30 mm: alloc_swap_folio: pass raw fault address to vma_alloc_folio
In reply to a message not in the archiveSame change as the previous patch but for alloc_swap_folio:
pass vmf->address directly instead of ALIGN_DOWN(vmf->address, ...).
--- mm/memory.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
mm/memory.c +1 -2
@@ -4734,8 +4734,7 @@ static struct folio *alloc_swap_folio(struct vm_fault *vmf)/* Try allocating the highest of the remaining orders. */ gfp =vma_thp_gfp_mask (vma);while (orders) { - addr =ALIGN_DOWN (vmf->address,PAGE_SIZE << order); - folio =vma_alloc_folio (gfp, order, vma, addr); + folio =vma_alloc_folio (gfp, order, vma, vmf->address);if (folio) {if (!mem_cgroup_swapin_charge_folio (folio, vma->vm_mm, gfp, entry))
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 10/30 mm: use __GFP_ZERO in alloc_anon_folio
In reply to a message not in the archiveConvert alloc_anon_folio() to pass __GFP_ZERO instead of zeroing at the callsite. post_alloc_hook uses the fault address passed through vma_alloc_folio for cache-friendly zeroing.
--- mm/memory.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-)
mm/memory.c +1 -10
@@ -5249,7 +5249,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) goto fallback;/* Try allocating the highest of the remaining orders. */ - gfp =vma_thp_gfp_mask (vma); + gfp =vma_thp_gfp_mask (vma) | __GFP_ZERO;while (orders) { folio =vma_alloc_folio (gfp, order, vma, vmf->address);if (folio) {
@@ -5259,15 +5259,6 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) goto next; }folio_throttle_swaprate (folio, gfp); -/* -* When a folio is not zeroed during allocation -* (__GFP_ZERO not used) or user folios require special -* handling, folio_zero_user() is used to make sure -* that the page corresponding to the faulting address -* will be hot in the cache after zeroing. -*/ -if (user_alloc_needs_zeroing ()) -folio_zero_user (folio, vmf->address);return folio; } next:
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 11/30 mm: vma_alloc_anon_folio_pmd: pass raw fault address to vma_alloc_folio
In reply to a message not in the archiveNow that vma_alloc_folio aligns the address internally, drop the redundant HPAGE_PMD_MASK alignment at the callsite.
--- mm/huge_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/huge_memory.c +1 -1
@@ -1337,7 +1337,7 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,const int order =HPAGE_PMD_ORDER ;struct folio *folio; - folio =vma_alloc_folio (gfp, order, vma, addr &HPAGE_PMD_MASK ); + folio =vma_alloc_folio (gfp, order, vma, addr);if (unlikely (!folio)) {count_vm_event (THP_FAULT_FALLBACK );
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 12/30 mm: use __GFP_ZERO in vma_alloc_anon_folio_pmd
In reply to a message not in the archiveConvert vma_alloc_anon_folio_pmd() to pass __GFP_ZERO instead of zeroing at the callsite. post_alloc_hook uses the fault address passed through vma_alloc_folio for cache-friendly zeroing.
--- mm/huge_memory.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-)
mm/huge_memory.c +1 -9
@@ -1333,7 +1333,7 @@ EXPORT_SYMBOL_GPL(thp_get_unmapped_area);static struct folio *vma_alloc_anon_folio_pmd (struct vm_area_struct *vma,unsigned long addr) { -gfp_t gfp =vma_thp_gfp_mask (vma); +gfp_t gfp =vma_thp_gfp_mask (vma) | __GFP_ZERO;const int order =HPAGE_PMD_ORDER ;struct folio *folio;
@@ -1356,14 +1356,6 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, }folio_throttle_swaprate (folio, gfp); -/* -* When a folio is not zeroed during allocation (__GFP_ZERO not used) -* or user folios require special handling, folio_zero_user() is used to -* make sure that the page corresponding to the faulting address will be -* hot in the cache after zeroing. -*/ -if (user_alloc_needs_zeroing ()) -folio_zero_user (folio, addr);/* * The memory barrier inside __folio_mark_uptodate makes sure that * folio_zero_user writes become visible before the set_pmd_at()
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 13/30 mm: hugetlb: use __GFP_ZERO and skip zeroing for zeroed pages
In reply to a message not in the archiveConvert the hugetlb fault and fallocate paths to use __GFP_ZERO. For pages allocated from the buddy allocator, post_alloc_hook() handles zeroing.
Hugetlb surplus pages need special handling because they can be pre-allocated into the pool during mmap (by hugetlb_acct_memory) before any page fault. Pool pages are kept around and may need zeroing long after buddy allocation, so a buddy-level zeroed hint (consumed at allocation time) cannot track their state.
Add a bool *zeroed output parameter to alloc_hugetlb_folio() so callers know whether the page needs zeroing. Buddy-allocated pages are always zeroed (zeroed by post_alloc_hook). Pool pages use a new HPG_zeroed flag to track whether the page is known-zero (freshly buddy-allocated, never mapped to userspace). The flag is set in alloc_surplus_hugetlb_folio() after buddy allocation and cleared in free_huge_folio() when a user-mapped page returns to the pool.
Callers that do not need zeroing (CoW, migration) pass NULL for zeroed and 0 for gfp.
--- fs/hugetlbfs/inode.c | 10 ++++++-- include/linux/hugetlb.h | 8 +++++-- mm/hugetlb.c | 52 ++++++++++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 17 deletions(-)
fs/hugetlbfs/inode.c +8 -2
@@ -810,14 +810,20 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,* folios in these areas, we need to consume the reserves * to keep reservation accounting consistent. */ - folio =alloc_hugetlb_folio (&pseudo_vma, addr, false); + { +bool zeroed; + + folio =alloc_hugetlb_folio (&pseudo_vma, addr, false, + __GFP_ZERO, &zeroed);if (IS_ERR (folio)) {mutex_unlock (&hugetlb_fault_mutex_table[hash]); error =PTR_ERR (folio); goto out; } -folio_zero_user (folio, addr); +if (!zeroed) +folio_zero_user (folio, addr);__folio_mark_uptodate (folio); + } error =hugetlb_add_to_page_cache (folio, mapping, index);if (unlikely (error)) {restore_reserve_on_error (h, &pseudo_vma, addr, folio);
include/linux/hugetlb.h +6 -2
@@ -599,6 +599,7 @@ enum hugetlb_page_flags { HPG_vmemmap_optimized, HPG_raw_hwp_unreliable, HPG_cma, + HPG_zeroed, __NR_HPAGEFLAGS, };
@@ -659,6 +660,7 @@ HPAGEFLAG(Freed, freed) HPAGEFLAG(VmemmapOptimized, vmemmap_optimized) HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable) HPAGEFLAG(Cma, cma) +HPAGEFLAG(Zeroed, zeroed)#ifdef CONFIG_HUGETLB_PAGE
@@ -706,7 +708,8 @@ int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);int replace_free_hugepage_folios (unsigned long start_pfn,unsigned long end_pfn);void wait_for_freed_hugetlb_folios (void );struct folio *alloc_hugetlb_folio (struct vm_area_struct *vma, -unsigned long addr,bool cow_from_owner); +unsigned long addr,bool cow_from_owner, +gfp_t gfp,bool *zeroed);struct folio *alloc_hugetlb_folio_nodemask (struct hstate *h,int preferred_nid,nodemask_t *nmask,gfp_t gfp_mask,bool allow_alloc_fallback);
@@ -1131,7 +1134,8 @@ static inline void wait_for_freed_hugetlb_folios(void)static inline struct folio *alloc_hugetlb_folio (struct vm_area_struct *vma,unsigned long addr, -bool cow_from_owner) +bool cow_from_owner, +gfp_t gfp,bool *zeroed) {return NULL ; }
mm/hugetlb.c +39 -13
144 lines not on this page · raw message
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 14/30 mm: memfd: skip zeroing for zeroed hugetlb pool pages
In reply to a message not in the archivegather_surplus_pages() pre-allocates hugetlb pages into the pool during mmap. Pass __GFP_ZERO so these pages are zeroed by the buddy allocator, and HPG_zeroed is set by alloc_surplus_hugetlb_folio.
Add bool *zeroed output to alloc_hugetlb_folio_reserve() so callers can check whether the pool page is known-zero. memfd's memfd_alloc_folio() uses this to skip the explicit folio_zero_user() when the page is already zero.
This avoids redundant zeroing for memfd hugetlb pages that were pre-allocated into the pool and never mapped to userspace.
--- include/linux/hugetlb.h | 6 ++++-- mm/hugetlb.c | 11 +++++++++-- mm/memfd.c | 14 ++++++++------ 3 files changed, 21 insertions(+), 10 deletions(-)
include/linux/hugetlb.h +4 -2
@@ -714,7 +714,8 @@ struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback);struct folio *alloc_hugetlb_folio_reserve (struct hstate *h,int preferred_nid, -nodemask_t *nmask,gfp_t gfp_mask); +nodemask_t *nmask,gfp_t gfp_mask, +bool *zeroed);int hugetlb_add_to_page_cache (struct folio *folio,struct address_space *mapping,pgoff_t idx);
@@ -1142,7 +1143,8 @@ static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,static inline struct folio *alloc_hugetlb_folio_reserve (struct hstate *h,int preferred_nid, -nodemask_t *nmask,gfp_t gfp_mask) +nodemask_t *nmask,gfp_t gfp_mask, +bool *zeroed) {return NULL ; }
mm/hugetlb.c +9 -2
@@ -2205,7 +2205,7 @@ struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, }struct folio *alloc_hugetlb_folio_reserve (struct hstate *h,int preferred_nid, -nodemask_t *nmask,gfp_t gfp_mask) +nodemask_t *nmask,gfp_t gfp_mask,bool *zeroed) {struct folio *folio;
@@ -2221,6 +2221,12 @@ struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, h->resv_huge_pages--;spin_unlock_irq (&hugetlb_lock); + +if (zeroed && folio) { + *zeroed =folio_test_hugetlb_zeroed (folio); +folio_clear_hugetlb_zeroed (folio); + } +return folio; }
@@ -2305,7 +2311,8 @@ static int gather_surplus_pages(struct hstate *h, long delta)* It is okay to use NUMA_NO_NODE because we use numa_mem_id() * down the road to pick the current node if that is the case. */ - folio =alloc_surplus_hugetlb_folio (h,htlb_alloc_mask (h), + folio =alloc_surplus_hugetlb_folio (h, +htlb_alloc_mask (h) | __GFP_ZERO,NUMA_NO_NODE , &alloc_nodemask,USER_ADDR_NONE );if (!folio) {
mm/memfd.c +8 -6
@@ -69,6 +69,7 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx)#ifdef CONFIG_HUGETLB_PAGE struct folio *folio;gfp_t gfp_mask; +bool zeroed;if (is_file_hugepages (memfd)) {/*
@@ -93,17 +94,18 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) folio =alloc_hugetlb_folio_reserve (h,numa_node_id (),NULL , - gfp_mask); + gfp_mask, + &zeroed);if (folio) {u32 hash;/* -* Zero the folio to prevent information leaks to userspace. -* Use folio_zero_user() which is optimized for huge/gigantic -* pages. Pass 0 as addr_hint since this is not a faulting path -* and we don't have a user virtual address yet. +* Zero the folio to prevent information leaks to +* userspace. Skip if the pool page is known-zero +* (HPG_zeroed set during pool pre-allocation). */ -folio_zero_user (folio,0 ); +if (!zeroed) +folio_zero_user (folio,0 );/* * Mark the folio uptodate before adding to page cache,
Signature · 1 line
MST
Michael S. Tsirkin
maintainerPATCH v6 15/30 mm: remove arch vma_alloc_zeroed_movable_folio overrides
In reply to a message not in the archiveNow that the generic vma_alloc_zeroed_movable_folio() uses __GFP_ZERO, the arch-specific macros on alpha, m68k, s390, and x86 that did the same thing are redundant. Remove them.
arm64 is not affected: it has a real function override that handles MTE tag zeroing, not just __GFP_ZERO.
--- arch/alpha/include/asm/page.h | 3 --- arch/m68k/include/asm/page_no.h | 3 --- arch/s390/include/asm/page.h | 3 --- arch/x86/include/asm/page.h | 3 --- 4 files changed, 12 deletions(-)
arch/alpha/include/asm/page.h +0 -3
@@ -12,9 +12,6 @@extern void clear_page (void *page); -#define vma_alloc_zeroed_movable_folio (vma, vaddr) \ - vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr) -extern void copy_page (void * _to,void * _from);#define copy_user_page (to, from, vaddr, pg) copy_page(to, from)
arch/m68k/include/asm/page_no.h +0 -3
@@ -12,9 +12,6 @@ extern unsigned long memory_end;#define copy_user_page (to, from, vaddr, pg) copy_page(to, from) -#define vma_alloc_zeroed_movable_folio (vma, vaddr) \ - vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr) -#define __pa (vaddr) ((unsigned long)(vaddr))#define __va (paddr) ((void *)((unsigned long)(paddr)))
arch/s390/include/asm/page.h +0 -3
@@ -67,9 +67,6 @@ static inline void copy_page(void *to, void *from)#define copy_user_page (to, from, vaddr, pg) copy_page(to, from) -#define vma_alloc_zeroed_movable_folio (vma, vaddr) \ - vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr) -#ifdef CONFIG_STRICT_MM_TYPECHECKS #define STRICT_MM_TYPECHECKS #endif
arch/x86/include/asm/page.h +0 -3
@@ -28,9 +28,6 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr,copy_page (to, from); } -#define vma_alloc_zeroed_movable_folio (vma, vaddr) \ - vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr) -#ifndef __pa#define __pa (x) __phys_addr((unsigned long)(x))#endif
Signature · 1 line
MST