Per-VMA lock support for swap and userfaults
Subsystems (12)
Suren Baghdasaryan
reviewerv6 cover Per-VMA lock support for swap and userfaults
When per-VMA locks were introduced in [1] several types of page faults would still fall back to mmap_lock to keep the patchset simple. Among them are swap and userfault pages. The main reason for skipping those cases was the fact that mmap_lock could be dropped while handling these faults and that required additional logic to be implemented.
Implement the mechanism to allow per-VMA locks to be dropped for these cases.
First, change handle_mm_fault to drop per-VMA locks when returning VM_FAULT_RETRY or VM_FAULT_COMPLETED to be consistent with the way mmap_lock is handled. Then change folio_lock_or_retry to accept vm_fault and return vm_fault_t which simplifies later patches. Finally allow swap and uffd page faults to be handled under per-VMA locks by dropping per-VMA and retrying, the same way it's done under mmap_lock. Naturally, once VMA lock is dropped that VMA should be assumed unstable and can't be used.
Changes since v5 posted at [2]
- 6/6 moved changes in sanitize_fault_flags into 3/6, per Peter Xu
- rebased over Linus' ToT
Note: patch 3/6 will cause a trivial merge conflict in arch/arm64/mm/fault.c when applied over mm-unstable branch due to a patch from ARM64 tree [3] which is missing in mm-unstable.
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/
[3] https://lore.kernel.org/all/[email protected]/
Suren Baghdasaryan (6):
swap: remove remnants of polling from read_swap_cache_async
mm: add missing VM_FAULT_RESULT_TRACE name for VM_FAULT_COMPLETED
mm: drop per-VMA lock when returning VM_FAULT_RETRY or
VM_FAULT_COMPLETED
mm: change folio_lock_or_retry to use vm_fault directly
mm: handle swap page faults under per-VMA lock
mm: handle userfaults under VMA lock
arch/arm64/mm/fault.c | 3 ++- arch/powerpc/mm/fault.c | 3 ++- arch/s390/mm/fault.c | 3 ++- arch/x86/mm/fault.c | 3 ++- fs/userfaultfd.c | 34 ++++++++++++---------------- include/linux/mm.h | 37 ++++++++++++++++++++++++++++++ include/linux/mm_types.h | 3 ++- include/linux/pagemap.h | 9 ++++---- mm/filemap.c | 37 +++++++++++++++--------------- mm/madvise.c | 4 ++-- mm/memory.c | 49 ++++++++++++++++++++++------------------ mm/swap.h | 1 - mm/swap_state.c | 12 ++++------ 13 files changed, 118 insertions(+), 80 deletions(-)
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Suren Baghdasaryan
reviewerPATCH v6 1/6 swap: remove remnants of polling from read_swap_cache_async
Commit [1] introduced IO polling support duding swapin to reduce swap read latency for block devices that can be polled. However later commit [2] removed polling support. Therefore it seems safe to remove do_poll parameter in read_swap_cache_async and always call swap_readpage with synchronous=false waiting for IO completion in folio_lock_or_retry.
[1] commit 23955622ff8d ("swap: add block io poll in swapin path")
[2] commit 9650b453a3d4 ("block: ignore RWF_HIPRI hint for sync dio")
--- mm/madvise.c | 4 ++-- mm/swap.h | 1 - mm/swap_state.c | 12 +++++------- 3 files changed, 7 insertions(+), 10 deletions(-)
mm/madvise.c +2 -2
@@ -218,7 +218,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, ptep =NULL ; page =read_swap_cache_async (entry,GFP_HIGHUSER_MOVABLE , - vma, addr, false, &splug); + vma, addr, &splug);if (page)put_page (page); }
@@ -262,7 +262,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,rcu_read_unlock (); page =read_swap_cache_async (entry,mapping_gfp_mask (mapping), - vma, addr, false, &splug); + vma, addr, &splug);if (page)put_page (page);
mm/swap.h +0 -1
@@ -46,7 +46,6 @@ struct folio *filemap_get_incore_folio(struct address_space *mapping,struct page *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr, -bool do_poll,struct swap_iocb **plug); struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma,
mm/swap_state.c +5 -7
@@ -527,15 +527,14 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,*/ struct page *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask,struct vm_area_struct *vma, -unsigned long addr,bool do_poll, -struct swap_iocb **plug) +unsigned long addr,struct swap_iocb **plug) {bool page_was_allocated;struct page *retpage =__read_swap_cache_async (entry, gfp_mask, vma, addr, &page_was_allocated);if (page_was_allocated) -swap_readpage (retpage, do_poll, plug); +swap_readpage (retpage, false, plug);return retpage; }
@@ -630,7 +629,7 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,struct swap_info_struct *si =swp_swap_info (entry);struct blk_plug plug;struct swap_iocb *splug =NULL ; -bool do_poll = true, page_allocated; +bool page_allocated;struct vm_area_struct *vma = vmf->vma;unsigned long addr = vmf->address;
@@ -638,7 +637,6 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,if (!mask) goto skip; - do_poll = false;/* Read a page_cluster sized and aligned cluster around offset. */ start_offset = offset & ~mask; end_offset = offset | mask;
@@ -670,7 +668,7 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,lru_add_drain ();/* Push any new pages onto the LRU now */ skip:/* The page was likely read above, so no need for plugging here */ -return read_swap_cache_async (entry, gfp_mask, vma, addr, do_poll,NULL ); +return read_swap_cache_async (entry, gfp_mask, vma, addr,NULL ); }int init_swap_address_space (unsigned int type,unsigned long nr_pages)
@@ -838,7 +836,7 @@ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, skip:/* The page was likely read above, so no need for plugging here */ return read_swap_cache_async (fentry, gfp_mask, vma, vmf->address, - ra_info.win ==1 ,NULL ); +NULL ); }/**
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Suren Baghdasaryan
reviewerPATCH v6 2/6 mm: add missing VM_FAULT_RESULT_TRACE name for VM_FAULT_COMPLETED
In reply to Suren BaghdasaryanVM_FAULT_RESULT_TRACE should contain an element for every vm_fault_reason to be used as flag_array inside trace_print_flags_seq(). The element for VM_FAULT_COMPLETED is missing, add it.
--- include/linux/mm_types.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/linux/mm_types.h +2 -1
@@ -1077,7 +1077,8 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" } struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Suren Baghdasaryan
reviewerPATCH v6 3/6 mm: drop per-VMA lock when returning VM_FAULT_RETRY or VM_FAULT_COMPLETED
In reply to Suren Baghdasaryanhandle_mm_fault returning VM_FAULT_RETRY or VM_FAULT_COMPLETED means mmap_lock has been released. However with per-VMA locks behavior is different and the caller should still release it. To make the rules consistent for the caller, drop the per-VMA lock when returning VM_FAULT_RETRY or VM_FAULT_COMPLETED. Currently the only path returning VM_FAULT_RETRY under per-VMA locks is do_swap_page and no path returns VM_FAULT_COMPLETED for now.
--- arch/arm64/mm/fault.c | 3 ++- arch/powerpc/mm/fault.c | 3 ++- arch/s390/mm/fault.c | 3 ++- arch/x86/mm/fault.c | 3 ++- mm/memory.c | 12 ++++++++++++ 5 files changed, 20 insertions(+), 4 deletions(-)
arch/arm64/mm/fault.c +2 -1
@@ -602,7 +602,8 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, goto lock_mmap; } fault =handle_mm_fault (vma, addr, mm_flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault & (VM_FAULT_RETRY |VM_FAULT_COMPLETED ))) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
arch/powerpc/mm/fault.c +2 -1
@@ -489,7 +489,8 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault & (VM_FAULT_RETRY |VM_FAULT_COMPLETED ))) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
arch/s390/mm/fault.c +2 -1
@@ -418,7 +418,8 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) goto lock_mmap; } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault & (VM_FAULT_RETRY |VM_FAULT_COMPLETED ))) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS ); goto out;
arch/x86/mm/fault.c +2 -1
@@ -1341,7 +1341,8 @@ void do_user_addr_fault(struct pt_regs *regs, goto lock_mmap; } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault & (VM_FAULT_RETRY |VM_FAULT_COMPLETED ))) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
mm/memory.c +12 -0
@@ -3730,6 +3730,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { ret =VM_FAULT_RETRY ; +vma_end_read (vma); goto out; }
@@ -5182,6 +5183,17 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, !is_cow_mapping(vma->vm_flags)))return VM_FAULT_SIGSEGV ; } +#ifdef CONFIG_PER_VMA_LOCK +/* +* Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of +* the assumption that lock is dropped on VM_FAULT_RETRY. +*/ +if (WARN_ON_ONCE ((*flags & + (FAULT_FLAG_VMA_LOCK |FAULT_FLAG_RETRY_NOWAIT )) == + (FAULT_FLAG_VMA_LOCK |FAULT_FLAG_RETRY_NOWAIT ))) +return VM_FAULT_SIGSEGV ; +#endif +return 0 ; }
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Suren Baghdasaryan
reviewerPATCH v6 4/6 mm: change folio_lock_or_retry to use vm_fault directly
In reply to Suren BaghdasaryanChange folio_lock_or_retry to accept vm_fault struct and return the vm_fault_t directly.
--- include/linux/pagemap.h | 9 ++++----- mm/filemap.c | 22 ++++++++++++---------- mm/memory.c | 14 ++++++-------- 3 files changed, 22 insertions(+), 23 deletions(-)
include/linux/pagemap.h +4 -5
@@ -900,8 +900,7 @@ static inline bool wake_page_match(struct wait_page_queue *wait_page,void __folio_lock (struct folio *folio);int __folio_lock_killable (struct folio *folio); -bool __folio_lock_or_retry (struct folio *folio,struct mm_struct *mm, -unsigned int flags); +vm_fault_t __folio_lock_or_retry (struct folio *folio,struct vm_fault *vmf);void unlock_page (struct page *page);void folio_unlock (struct folio *folio);
@@ -1005,11 +1004,11 @@ static inline int folio_lock_killable(struct folio *folio)* Return value and mmap_lock implications depend on flags; see * __folio_lock_or_retry(). */ -static inline bool folio_lock_or_retry (struct folio *folio, -struct mm_struct *mm,unsigned int flags) +static inline vm_fault_t folio_lock_or_retry (struct folio *folio, +struct vm_fault *vmf) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +return folio_trylock (folio) ?0 :__folio_lock_or_retry (folio, vmf); }/*
mm/filemap.c +12 -10
@@ -1669,32 +1669,34 @@ static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)/* * Return values: -* true - folio is locked; mmap_lock is still held. -* false - folio is not locked. +* 0 - folio is locked. +* VM_FAULT_RETRY - folio is not locked. * mmap_lock has been released (mmap_read_unlock(), unless flags had both * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in * which case mmap_lock is still held. * -* If neither ALLOW_RETRY nor KILLABLE are set, will always return true +* If neither ALLOW_RETRY nor KILLABLE are set, will always return 0 * with the folio locked and the mmap_lock unperturbed. */ -bool __folio_lock_or_retry (struct folio *folio,struct mm_struct *mm, -unsigned int flags) +vm_fault_t __folio_lock_or_retry (struct folio *folio,struct vm_fault *vmf) { +struct mm_struct *mm = vmf->vma->vm_mm; +unsigned int flags = vmf->flags; +if (fault_flag_allow_retry_first (flags)) {/* * CAUTION! In this case, mmap_lock is not released -* even though return 0. +* even though return VM_FAULT_RETRY. */ if (flags &FAULT_FLAG_RETRY_NOWAIT ) -return false; +return VM_FAULT_RETRY ;mmap_read_unlock (mm);if (flags &FAULT_FLAG_KILLABLE )folio_wait_locked_killable (folio);else folio_wait_locked (folio); -return false; +return VM_FAULT_RETRY ; }if (flags &FAULT_FLAG_KILLABLE ) {bool ret;
@@ -1702,13 +1704,13 @@ bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm, ret =__folio_lock_killable (folio);if (ret) {mmap_read_unlock (mm); -return false; +return VM_FAULT_RETRY ; } }else {__folio_lock (folio); } -return true; +return 0 ; }/**
mm/memory.c +6 -8
@@ -3582,6 +3582,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)struct folio *folio =page_folio (vmf->page);struct vm_area_struct *vma = vmf->vma;struct mmu_notifier_range range; +vm_fault_t ret;/* * We need a reference to lock the folio because we don't hold
@@ -3594,9 +3595,10 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)if (!folio_try_get (folio))return 0 ; -if (!folio_lock_or_retry (folio, vma->vm_mm, vmf->flags)) { + ret =folio_lock_or_retry (folio, vmf); +if (ret) {folio_put (folio); -return VM_FAULT_RETRY ; +return ret; } mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma->vm_mm, vmf->address & PAGE_MASK,
@@ -3721,7 +3723,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)bool exclusive = false;swp_entry_t entry;pte_t pte; -int locked;vm_fault_t ret =0 ;void *shadow =NULL ;
@@ -3844,12 +3845,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out_release; } - locked =folio_lock_or_retry (folio, vma->vm_mm, vmf->flags); - -if (!locked) { - ret |=VM_FAULT_RETRY ; + ret |=folio_lock_or_retry (folio, vmf); +if (ret &VM_FAULT_RETRY ) goto out_release; - }if (swapcache) {/*
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Matthew Wilcox
maintainerRe: [PATCH v6 4/6] mm: change folio_lock_or_retry to use vm_fault directly
Quoted 2 lines · Suren Baghdasaryan
Change
folio_lock_or_retryto acceptvm_faultstruct and return thevm_fault_tdirectly.
I thought we decided to call this folio_lock_fault()?
include/linux/pagemap.h static inline int folio_lock_killable(struct folio *folio) · line 1011
Quoted 6 lines
include/linux/pagemap.h +3
-1+static inline vm_fault_t folio_lock_or_retry (struct folio *folio, +struct vm_fault *vmf) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +return folio_trylock (folio) ?0 :__folio_lock_or_retry (folio, vmf);
No, don't use the awful ternary operator. The || form is used everywhere else.
Quoted 6 lines
/*
* Return values:
- * true - folio is locked;mmap_lockis still held.
- * false - folio is not locked.
+ * 0 - folio is locked.
+ *VM_FAULT_RETRY- folio is not locked.
I don't think we want to be so prescriptive here. It returns non-zero if the folio is not locked. The precise value is not something that callers should depend on.
Suren Baghdasaryan
reviewerRe: [PATCH v6 4/6] mm: change folio_lock_or_retry to use vm_fault directly
Quoted 14 lines · Matthew Wilcox
Quoted 2 lines · Suren Baghdasaryan
Change
folio_lock_or_retryto acceptvm_faultstruct and return thevm_fault_tdirectly.I thought we decided to call this
folio_lock_fault()?include/linux/pagemap.h static inline int folio_lock_killable(struct folio *folio) · line 1011
Quoted 6 lines
include/linux/pagemap.h +3
-1+static inline vm_fault_t folio_lock_or_retry (struct folio *folio, +struct vm_fault *vmf) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +return folio_trylock (folio) ?0 :__folio_lock_or_retry (folio, vmf);
Quoted 2 lines
No, don't use the awful ternary operator. The || form is used everywhere else.
Ok, but folio_trylock() returns a boolean while folio_lock_or_retry should return vm_fault_t. How exactly do you suggest changing this? Something like this perhaps:
static inline vm_fault_t folio_lock_or_retry(struct folio *folio,
struct vm_fault *vmf)
{
might_sleep();
if (folio_trylock(folio))
return 0;
return __folio_lock_or_retry(folio, mm, flags);
}?
Quoted 11 lines
Quoted 7 lines
/*
* Return values:
- * true - folio is locked;mmap_lockis still held.
- * false - folio is not locked.
+ * 0 - folio is locked.
+ *VM_FAULT_RETRY- folio is not locked.I don't think we want to be so prescriptive here. It returns non-zero if the folio is not locked. The precise value is not something that callers should depend on.
Ok, I'll change it to "non-zero" here.
Quoted 1 line
Matthew Wilcox
maintainerRe: [PATCH v6 4/6] mm: change folio_lock_or_retry to use vm_fault directly
Quoted 24 lines · Suren Baghdasaryan
Quoted 14 lines · Matthew Wilcox
Quoted 2 lines · Suren Baghdasaryan
Change
folio_lock_or_retryto acceptvm_faultstruct and return thevm_fault_tdirectly.I thought we decided to call this
folio_lock_fault()?include/linux/pagemap.h static inline int folio_lock_killable(struct folio *folio) · line 1011
Quoted 6 lines
include/linux/pagemap.h +3
-1+static inline vm_fault_t folio_lock_or_retry (struct folio *folio, +struct vm_fault *vmf) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +return folio_trylock (folio) ?0 :__folio_lock_or_retry (folio, vmf);Quoted 2 lines
No, don't use the awful ternary operator. The || form is used everywhere else.
Ok, but
folio_trylock()returns a boolean whilefolio_lock_or_retryshould returnvm_fault_t. How exactly do you suggest changing this? Something like this perhaps:static inline
vm_fault_tfolio_lock_or_retry(struct folio *folio,
structvm_fault*vmf)
Quoted 8 lines
{ might_sleep(); if (folio_trylock(folio)) return 0; return __folio_lock_or_retry(folio, mm, flags); }?
I think the automatic casting would work, but I prefer what you've written here.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
/*
* Return values:
- * true - folio is locked;mmap_lockis still held.
- * false - folio is not locked.
+ * 0 - folio is locked.
+ *VM_FAULT_RETRY- folio is not locked.
Quoted 5 lines
Quoted 3 lines
I don't think we want to be so prescriptive here. It returns non-zero if the folio is not locked. The precise value is not something that callers should depend on.
Ok, I'll change it to "non-zero" here.
Thanks!
Suren Baghdasaryan
reviewerRe: [PATCH v6 4/6] mm: change folio_lock_or_retry to use vm_fault directly
Quoted 45 lines · Matthew Wilcox
Quoted 24 lines · Suren Baghdasaryan
Quoted 14 lines · Matthew Wilcox
Quoted 2 lines · Suren Baghdasaryan
Change
folio_lock_or_retryto acceptvm_faultstruct and return thevm_fault_tdirectly.I thought we decided to call this
folio_lock_fault()?include/linux/pagemap.h static inline int folio_lock_killable(struct folio *folio) · line 1011
Quoted 6 lines
include/linux/pagemap.h +3
-1+static inline vm_fault_t folio_lock_or_retry (struct folio *folio, +struct vm_fault *vmf) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +return folio_trylock (folio) ?0 :__folio_lock_or_retry (folio, vmf);Quoted 2 lines
No, don't use the awful ternary operator. The || form is used everywhere else.
Ok, but
folio_trylock()returns a boolean whilefolio_lock_or_retryshould returnvm_fault_t. How exactly do you suggest changing this? Something like this perhaps:static inline
vm_fault_tfolio_lock_or_retry(struct folio *folio,
structvm_fault*vmf)Quoted 8 lines
{ might_sleep(); if (folio_trylock(folio)) return 0; return __folio_lock_or_retry(folio, mm, flags); }?
I think the automatic casting would work, but I prefer what you've written here.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
/*
* Return values:
- * true - folio is locked;mmap_lockis still held.
- * false - folio is not locked.
+ * 0 - folio is locked.
+ *VM_FAULT_RETRY- folio is not locked.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
I don't think we want to be so prescriptive here. It returns non-zero if the folio is not locked. The precise value is not something that callers should depend on.
Ok, I'll change it to "non-zero" here.
Thanks!
Posted v7 at https://lore.kernel.org/all/[email protected]/
Suren Baghdasaryan
reviewerPATCH v6 5/6 mm: handle swap page faults under per-VMA lock
In reply to Suren BaghdasaryanWhen page fault is handled under per-VMA lock protection, all swap page faults are retried with mmap_lock because folio_lock_or_retry has to drop and reacquire mmap_lock if folio could not be immediately locked. Follow the same pattern as mmap_lock to drop per-VMA lock when waiting for folio and retrying once folio is available.
With this obstacle removed, enable do_swap_page to operate under per-VMA lock protection. Drivers implementing ops->migrate_to_ram might still rely on mmap_lock, therefore we have to fall back to mmap_lock in that particular case.
Note that the only time do_swap_page calls synchronous swap_readpage is when SWP_SYNCHRONOUS_IO is set, which is only set for QUEUE_FLAG_SYNCHRONOUS devices: brd, zram and nvdimms (both btt and pmem). Therefore we don't sleep in this path, and there's no need to drop the mmap or per-VMA lock.
--- include/linux/mm.h | 13 +++++++++++++ mm/filemap.c | 17 ++++++++--------- mm/memory.c | 16 ++++++++++------ 3 files changed, 31 insertions(+), 15 deletions(-)
include/linux/mm.h +13 -0
@@ -720,6 +720,14 @@ static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) vma->detached = detached; } +static inline void release_fault_lock (struct vm_fault *vmf) +{ +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) +vma_end_read (vmf->vma); +else +mmap_read_unlock (vmf->vma->vm_mm); +} +struct vm_area_struct *lock_vma_under_rcu (struct mm_struct *mm,unsigned long address);
@@ -735,6 +743,11 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma) {}static inline void vma_mark_detached (struct vm_area_struct *vma,bool detached) {} +static inline void release_fault_lock (struct vm_fault *vmf) +{ +mmap_read_unlock (vmf->vma->vm_mm); +} +#endif /* CONFIG_PER_VMA_LOCK */ /*
mm/filemap.c +8 -9
@@ -1671,27 +1671,26 @@ static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)* Return values: * 0 - folio is locked. * VM_FAULT_RETRY - folio is not locked. -* mmap_lock has been released (mmap_read_unlock(), unless flags had both -* FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in -* which case mmap_lock is still held. +* mmap_lock or per-VMA lock has been released (mmap_read_unlock() or +* vma_end_read()), unless flags had both FAULT_FLAG_ALLOW_RETRY and +* FAULT_FLAG_RETRY_NOWAIT set, in which case the lock is still held. * * If neither ALLOW_RETRY nor KILLABLE are set, will always return 0 -* with the folio locked and the mmap_lock unperturbed. +* with the folio locked and the mmap_lock/per-VMA lock is left unperturbed. */ vm_fault_t __folio_lock_or_retry (struct folio *folio,struct vm_fault *vmf) { -struct mm_struct *mm = vmf->vma->vm_mm;unsigned int flags = vmf->flags;if (fault_flag_allow_retry_first (flags)) {/* -* CAUTION! In this case, mmap_lock is not released -* even though return VM_FAULT_RETRY. +* CAUTION! In this case, mmap_lock/per-VMA lock is not +* released even though returning VM_FAULT_RETRY. */ if (flags &FAULT_FLAG_RETRY_NOWAIT )return VM_FAULT_RETRY ; -mmap_read_unlock (mm); +release_fault_lock (vmf);if (flags &FAULT_FLAG_KILLABLE )folio_wait_locked_killable (folio);else
@@ -1703,7 +1702,7 @@ vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) ret =__folio_lock_killable (folio);if (ret) { -mmap_read_unlock (mm); +release_fault_lock (vmf);return VM_FAULT_RETRY ; } }else {
mm/memory.c +10 -6
@@ -3729,12 +3729,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; -vma_end_read (vma); - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {
@@ -3744,6 +3738,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf); }else if (is_device_private_entry (entry)) { +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* +* migrate_to_ram is not yet ready to operate +* under VMA lock. +*/ +vma_end_read (vma); + ret =VM_FAULT_RETRY ; + goto out; + } + vmf->page =pfn_swap_entry_to_page (entry); vmf->pte =pte_offset_map_lock (vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl);
Signature · 1 line
2.41.0.255.g8b1d071c50-goog
Suren Baghdasaryan
reviewerPATCH v6 6/6 mm: handle userfaults under VMA lock
In reply to Suren BaghdasaryanEnable handle_userfault to operate under VMA lock by releasing VMA lock instead of mmap_lock and retrying. Note that FAULT_FLAG_RETRY_NOWAIT should never be used when handling faults under per-VMA lock protection because that would break the assumption that lock is dropped on retry.
--- fs/userfaultfd.c | 34 ++++++++++++++-------------------- include/linux/mm.h | 24 ++++++++++++++++++++++++ mm/memory.c | 9 --------- 3 files changed, 38 insertions(+), 29 deletions(-)
fs/userfaultfd.c +14 -20
@@ -277,17 +277,16 @@ static inline struct uffd_msg userfault_msg(unsigned long address,* hugepmd ranges. */ static inline bool userfaultfd_huge_must_wait (struct userfaultfd_ctx *ctx, -struct vm_area_struct *vma, -unsigned long address, -unsigned long flags, -unsigned long reason) +struct vm_fault *vmf, +unsigned long reason) { +struct vm_area_struct *vma = vmf->vma;pte_t *ptep, pte;bool ret = true; -mmap_assert_locked (ctx->mm); +assert_fault_locked (vmf); - ptep =hugetlb_walk (vma, address,vma_mmu_pagesize (vma)); + ptep =hugetlb_walk (vma, vmf->address,vma_mmu_pagesize (vma));if (!ptep) goto out;
@@ -308,10 +307,8 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, }#else static inline bool userfaultfd_huge_must_wait (struct userfaultfd_ctx *ctx, -struct vm_area_struct *vma, -unsigned long address, -unsigned long flags, -unsigned long reason) +struct vm_fault *vmf, +unsigned long reason) {return false;/* should never get here */ }
@@ -325,11 +322,11 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,* threads. */ static inline bool userfaultfd_must_wait (struct userfaultfd_ctx *ctx, -unsigned long address, -unsigned long flags, +struct vm_fault *vmf,unsigned long reason) {struct mm_struct *mm = ctx->mm; +unsigned long address = vmf->address;pgd_t *pgd;p4d_t *p4d;pud_t *pud;
@@ -338,7 +335,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,pte_t ptent;bool ret = true; -mmap_assert_locked (mm); +assert_fault_locked (vmf); pgd =pgd_offset (mm, address);if (!pgd_present (*pgd))
@@ -440,7 +437,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)* Coredumping runs without mmap_lock so we can only check that * the mmap_lock is held, if PF_DUMPCORE was not set. */ -mmap_assert_locked (mm); +assert_fault_locked (vmf); ctx = vma->vm_userfaultfd_ctx.ctx;if (!ctx)
@@ -556,15 +553,12 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)spin_unlock_irq (&ctx->fault_pending_wqh.lock);if (!is_vm_hugetlb_page (vma)) - must_wait =userfaultfd_must_wait (ctx, vmf->address, vmf->flags, - reason); + must_wait =userfaultfd_must_wait (ctx, vmf, reason);else - must_wait =userfaultfd_huge_must_wait (ctx, vma, - vmf->address, - vmf->flags, reason); + must_wait =userfaultfd_huge_must_wait (ctx, vmf, reason);if (is_vm_hugetlb_page (vma))hugetlb_vma_unlock_read (vma); -mmap_read_unlock (mm); +release_fault_lock (vmf);if (likely (must_wait && !READ_ONCE (ctx->released))) {wake_up_poll (&ctx->fd_wqh,EPOLLIN );
include/linux/mm.h +24 -0
@@ -705,6 +705,17 @@ static inline bool vma_try_start_write(struct vm_area_struct *vma)return true; } +static inline void vma_assert_locked (struct vm_area_struct *vma) +{ +int mm_lock_seq; + +if (__is_vma_write_locked (vma, &mm_lock_seq)) +return ; + +lockdep_assert_held (&vma->vm_lock->lock); +VM_BUG_ON_VMA (!rwsem_is_locked (&vma->vm_lock->lock), vma); +} +static inline void vma_assert_write_locked (struct vm_area_struct *vma) {int mm_lock_seq;
@@ -728,6 +739,14 @@ static inline void release_fault_lock(struct vm_fault *vmf)mmap_read_unlock (vmf->vma->vm_mm); } +static inline void assert_fault_locked (struct vm_fault *vmf) +{ +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) +vma_assert_locked (vmf->vma); +else +mmap_assert_locked (vmf->vma->vm_mm); +} +struct vm_area_struct *lock_vma_under_rcu (struct mm_struct *mm,unsigned long address);
@@ -748,6 +767,11 @@ static inline void release_fault_lock(struct vm_fault *vmf) mmap_read_unlock(vmf->vma->vm_mm); } +static inline void assert_fault_locked (struct vm_fault *vmf) +{ +mmap_assert_locked (vmf->vma->vm_mm); +} +#endif /* CONFIG_PER_VMA_LOCK */ /*
mm/memory.c +0 -9
@@ -5407,15 +5407,6 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,if (!vma_start_read (vma)) goto inval; -/* -* Due to the possibility of userfault handler dropping mmap_lock, avoid -* it for now and fall back to page fault handling under mmap_lock. -*/ -if (userfaultfd_armed (vma)) { -vma_end_read (vma); - goto inval; - } -/* Check since vm_start/vm_end might change before we lock the VMA */ if (unlikely (address < vma->vm_start || address >= vma->vm_end)) {vma_end_read (vma);
Signature · 1 line
2.41.0.255.g8b1d071c50-goog