mm-handle-userfaults-under-vma-lock.patch removed from -mm tree
The quilt patch titled
Subject: mm: handle userfaults under VMA lock
has been removed from the -mm tree. Its filename was
mm-handle-userfaults-under-vma-lock.patch
This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Suren Baghdasaryan <[email protected]>
Subject: mm: handle userfaults under VMA lock
Date: Fri, 30 Jun 2023 14:19:57 -0700
Enable handle_userfault to operate under VMA lock by releasing VMA lock instead of mmap_lock and retrying. Note that FAULT_FLAG_RETRY_NOWAIT should never be used when handling faults under per-VMA lock protection because that would break the assumption that lock is dropped on retry.
---
fs/userfaultfd.c | 34 ++++++++++++++-------------------- include/linux/mm.h | 20 ++++++++++++++++++++ mm/memory.c | 9 +-------- 3 files changed, 35 insertions(+), 28 deletions(-)
fs/userfaultfd.c renamed from fs/userfaultfd.c~mm-handle-userfaults-under-vma-lock +14 -20
@@ -277,17 +277,16 @@ static inline struct uffd_msg userfault_* hugepmd ranges. */ static inline bool userfaultfd_huge_must_wait (struct userfaultfd_ctx *ctx, -struct vm_area_struct *vma, -unsigned long address, -unsigned long flags, -unsigned long reason) +struct vm_fault *vmf, +unsigned long reason) { +struct vm_area_struct *vma = vmf->vma;pte_t *ptep, pte;bool ret = true; -mmap_assert_locked (ctx->mm); +assert_fault_locked (vmf); - ptep =hugetlb_walk (vma, address,vma_mmu_pagesize (vma)); + ptep =hugetlb_walk (vma, vmf->address,vma_mmu_pagesize (vma));if (!ptep) goto out;
@@ -308,10 +307,8 @@ out: }#else static inline bool userfaultfd_huge_must_wait (struct userfaultfd_ctx *ctx, -struct vm_area_struct *vma, -unsigned long address, -unsigned long flags, -unsigned long reason) +struct vm_fault *vmf, +unsigned long reason) {return false;/* should never get here */ }
@@ -325,11 +322,11 @@ static inline bool userfaultfd_huge_must* threads. */ static inline bool userfaultfd_must_wait (struct userfaultfd_ctx *ctx, -unsigned long address, -unsigned long flags, +struct vm_fault *vmf,unsigned long reason) {struct mm_struct *mm = ctx->mm; +unsigned long address = vmf->address;pgd_t *pgd;p4d_t *p4d;pud_t *pud;
@@ -338,7 +335,7 @@ static inline bool userfaultfd_must_waitpte_t ptent;bool ret = true; -mmap_assert_locked (mm); +assert_fault_locked (vmf); pgd =pgd_offset (mm, address);if (!pgd_present (*pgd))
@@ -440,7 +437,7 @@ vm_fault_t handle_userfault(struct vm_fa* Coredumping runs without mmap_lock so we can only check that * the mmap_lock is held, if PF_DUMPCORE was not set. */ -mmap_assert_locked (mm); +assert_fault_locked (vmf); ctx = vma->vm_userfaultfd_ctx.ctx;if (!ctx)
@@ -556,15 +553,12 @@ vm_fault_t handle_userfault(struct vm_faspin_unlock_irq (&ctx->fault_pending_wqh.lock);if (!is_vm_hugetlb_page (vma)) - must_wait =userfaultfd_must_wait (ctx, vmf->address, vmf->flags, - reason); + must_wait =userfaultfd_must_wait (ctx, vmf, reason);else - must_wait =userfaultfd_huge_must_wait (ctx, vma, - vmf->address, - vmf->flags, reason); + must_wait =userfaultfd_huge_must_wait (ctx, vmf, reason);if (is_vm_hugetlb_page (vma))hugetlb_vma_unlock_read (vma); -mmap_read_unlock (mm); +release_fault_lock (vmf);if (likely (must_wait && !READ_ONCE (ctx->released))) {wake_up_poll (&ctx->fd_wqh,EPOLLIN );
include/linux/mm.h renamed from include/linux/mm.h~mm-handle-userfaults-under-vma-lock +20 -0
@@ -679,6 +679,7 @@ static inline void vma_end_read(struct vrcu_read_unlock (); } +/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);
@@ -721,6 +722,12 @@ static inline void vma_assert_write_lockVM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); } +static inline void vma_assert_locked (struct vm_area_struct *vma) +{ +if (!rwsem_is_locked (&vma->vm_lock->lock)) +vma_assert_write_locked (vma); +} +static inline void vma_mark_detached (struct vm_area_struct *vma,bool detached) {/* When detaching vma should be write-locked */
@@ -737,6 +744,14 @@ static inline void release_fault_lock(stmmap_read_unlock (vmf->vma->vm_mm); } +static inline void assert_fault_locked (struct vm_fault *vmf) +{ +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) +vma_assert_locked (vmf->vma); +else +mmap_assert_locked (vmf->vma->vm_mm); +} +struct vm_area_struct *lock_vma_under_rcu (struct mm_struct *mm,unsigned long address);
@@ -762,6 +777,11 @@ static inline void release_fault_lock(st mmap_read_unlock(vmf->vma->vm_mm); } +static inline void assert_fault_locked (struct vm_fault *vmf) +{ +mmap_assert_locked (vmf->vma->vm_mm); +} +#endif /* CONFIG_PER_VMA_LOCK */ extern const struct vm_operations_struct vma_dummy_vm_ops;
mm/memory.c renamed from mm/memory.c~mm-handle-userfaults-under-vma-lock +1 -8
@@ -5471,14 +5471,7 @@ retry:* concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA * from its anon_vma. */ -if (vma_is_anonymous (vma) && !vma->anon_vma) - goto inval_end_read; - -/* -* Due to the possibility of userfault handler dropping mmap_lock, avoid -* it for now and fall back to page fault handling under mmap_lock. -*/ -if (userfaultfd_armed (vma)) +if (unlikely (vma_is_anonymous (vma) && !vma->anon_vma)) goto inval_end_read;/* Check since vm_start/vm_end might change before we lock the VMA */
_
Patches currently in -mm which might be from [email protected] are