mm: handle swap page faults under VMA lock if page is uncontended
Subsystems (10)
Suren Baghdasaryan
reviewerPATCH 1/3 mm: handle swap page faults under VMA lock if page is uncontended
When page fault is handled under VMA lock protection, all swap page faults are retried with mmap_lock because folio_lock_or_retry implementation has to drop and reacquire mmap_lock if folio could not be immediately locked.
Instead of retrying all swapped page faults, retry only when folio locking fails.
Drivers implementing ops->migrate_to_ram might still rely on mmap_lock, therefore fall back to mmap_lock in this case.
--- mm/filemap.c | 6 ++++++ mm/memory.c | 14 +++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-)
mm/filemap.c +6 -0
@@ -1706,6 +1706,8 @@ static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)* mmap_lock has been released (mmap_read_unlock(), unless flags had both * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in * which case mmap_lock is still held. +* If flags had FAULT_FLAG_VMA_LOCK set, meaning the operation is performed +* with VMA lock only, the VMA lock is still held. * * If neither ALLOW_RETRY nor KILLABLE are set, will always return true * with the folio locked and the mmap_lock unperturbed.
@@ -1713,6 +1715,10 @@ static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)bool __folio_lock_or_retry (struct folio *folio,struct mm_struct *mm,unsigned int flags) { +/* Can't do this if not holding mmap_lock */ +if (flags &FAULT_FLAG_VMA_LOCK ) +return false; +if (fault_flag_allow_retry_first (flags)) {/* * CAUTION! In this case, mmap_lock is not released
mm/memory.c +9 -5
@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {
@@ -3725,6 +3720,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf); }else if (is_device_private_entry (entry)) { +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* +* migrate_to_ram is not yet ready to operate +* under VMA lock. +*/ + ret |=VM_FAULT_RETRY ; + goto out; + } + vmf->page =pfn_swap_entry_to_page (entry); vmf->pte =pte_offset_map_lock (vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl);
Signature · 1 line
2.40.1.495.gc816e09b53d-goog
Suren Baghdasaryan
reviewerPATCH 2/3 mm: drop VMA lock before waiting for migration
migration_entry_wait does not need VMA lock, therefore it can be dropped before waiting. Introduce VM_FAULT_VMA_UNLOCKED to indicate that VMA lock was dropped while in handle_mm_fault().
Note that once VMA lock is dropped, the VMA reference can't be used as there are no guarantees it was not freed.
--- arch/arm64/mm/fault.c | 3 ++- arch/powerpc/mm/fault.c | 3 ++- arch/s390/mm/fault.c | 3 ++- arch/x86/mm/fault.c | 3 ++- include/linux/mm_types.h | 6 +++++- mm/memory.c | 12 ++++++++++-- 6 files changed, 23 insertions(+), 7 deletions(-)
arch/arm64/mm/fault.c +2 -1
@@ -602,7 +602,8 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, } fault =handle_mm_fault (vma, addr &PAGE_MASK , mm_flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault &VM_FAULT_VMA_UNLOCKED )) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
arch/powerpc/mm/fault.c +2 -1
@@ -494,7 +494,8 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault &VM_FAULT_VMA_UNLOCKED )) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
arch/s390/mm/fault.c +2 -1
@@ -418,7 +418,8 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) goto lock_mmap; } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault &VM_FAULT_VMA_UNLOCKED )) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS ); goto out;
arch/x86/mm/fault.c +2 -1
@@ -1347,7 +1347,8 @@ void do_user_addr_fault(struct pt_regs *regs, goto lock_mmap; } fault =handle_mm_fault (vma, address, flags |FAULT_FLAG_VMA_LOCK , regs); -vma_end_read (vma); +if (!(fault &VM_FAULT_VMA_UNLOCKED )) +vma_end_read (vma);if (!(fault &VM_FAULT_RETRY )) {count_vm_vma_lock_event (VMA_LOCK_SUCCESS );
include/linux/mm_types.h +5 -1
@@ -1030,6 +1030,7 @@ typedef __bitwise unsigned int vm_fault_t;* fsync() to complete (for synchronous page faults * in DAX) * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released +* @VM_FAULT_VMA_UNLOCKED: VMA lock was released * @VM_FAULT_HINDEX_MASK: mask HINDEX value * */
@@ -1047,6 +1048,7 @@ enum vm_fault_reason {VM_FAULT_DONE_COW = (__force vm_fault_t )0x001000 ,VM_FAULT_NEEDDSYNC = (__force vm_fault_t )0x002000 ,VM_FAULT_COMPLETED = (__force vm_fault_t )0x004000 , +VM_FAULT_VMA_UNLOCKED = (__force vm_fault_t )0x008000 , VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, };
@@ -1070,7 +1072,9 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" }, \ + { VM_FAULT_VMA_UNLOCKED, "VMA_UNLOCKED" } struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */
mm/memory.c +10 -2
@@ -3714,8 +3714,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) { -migration_entry_wait (vma->vm_mm, vmf->pmd, - vmf->address); +/* Save mm in case VMA lock is dropped */ +struct mm_struct *mm = vma->vm_mm; + +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* No need to hold VMA lock for migration */ +vma_end_read (vma); +/* CAUTION! VMA can't be used after this */ + ret |=VM_FAULT_VMA_UNLOCKED ; + } +migration_entry_wait (mm, vmf->pmd, vmf->address); }else if (is_device_exclusive_entry (entry)) { vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf);
Signature · 1 line
2.40.1.495.gc816e09b53d-goog
Re: [PATCH 2/3] mm: drop VMA lock before waiting for migration
Suren Baghdasaryan <[email protected]> writes:
[...]
include/linux/mm_types.h typedef __bitwise unsigned int vm_fault_t; · line 1033
Quoted 9 lines
include/linux/mm_types.h +1
-0@@ -1030,6 +1030,7 @@ typedef __bitwise unsigned int vm_fault_t;* fsync() to complete (for synchronous page faults * in DAX) * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released +* @VM_FAULT_VMA_UNLOCKED: VMA lock was released
A note here saying vmf->vma should no longer be accessed would be nice.
Quoted 12 lines
include/linux/mm_types.h +1
-0* @VM_FAULT_HINDEX_MASK: mask HINDEX value * */ @@ -1047,6 +1048,7 @@ enum vm_fault_reason {VM_FAULT_DONE_COW = (__force vm_fault_t )0x001000 ,VM_FAULT_NEEDDSYNC = (__force vm_fault_t )0x002000 ,VM_FAULT_COMPLETED = (__force vm_fault_t )0x004000 , +VM_FAULT_VMA_UNLOCKED = (__force vm_fault_t )0x008000 , VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, };
include/linux/mm_types.h enum vm_fault_reason { · line 1076
Quoted 6 lines
include/linux/mm_types.h +2
-1@@ -1070,7 +1072,9 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" }, \
VM_FAULT_COMPLETED isn't used in this patch, guessing that's snuck in from one of the other patches in the series?
Quoted 27 lines
include/linux/mm_types.h +1
-0+ { VM_FAULT_VMA_UNLOCKED, "VMA_UNLOCKED" }struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */ mm/memory.c +10
-2@@ -3714,8 +3714,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) { -migration_entry_wait (vma->vm_mm, vmf->pmd, - vmf->address); +/* Save mm in case VMA lock is dropped */ +struct mm_struct *mm = vma->vm_mm; + +if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* No need to hold VMA lock for migration */ +vma_end_read (vma); +/* CAUTION! VMA can't be used after this */ + ret |=VM_FAULT_VMA_UNLOCKED ; + } +migration_entry_wait (mm, vmf->pmd, vmf->address); }else if (is_device_exclusive_entry (entry)) { vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf);
Suren Baghdasaryan
reviewerRe: [PATCH 2/3] mm: drop VMA lock before waiting for migration
Quoted 6 lines · Alistair Popple' via kernel-team
Suren Baghdasaryan <[email protected]> writes:
[...]
Quoted 11 lines
include/linux/mm_types.h typedef __bitwise unsigned int vm_fault_t; · line 1033
Quoted 9 lines
include/linux/mm_types.h +1
-0@@ -1030,6 +1030,7 @@ typedef __bitwise unsigned int vm_fault_t;* fsync() to complete (for synchronous page faults * in DAX) * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released +* @VM_FAULT_VMA_UNLOCKED: VMA lock was released A note here saying
vmf->vmashould no longer be accessed would be nice.
Good idea. Will add in the next version. Thanks!
Quoted 12 lines
Quoted 12 lines
include/linux/mm_types.h +1
-0* @VM_FAULT_HINDEX_MASK: mask HINDEX value * */ @@ -1047,6 +1048,7 @@ enum vm_fault_reason {VM_FAULT_DONE_COW = (__force vm_fault_t )0x001000 ,VM_FAULT_NEEDDSYNC = (__force vm_fault_t )0x002000 ,VM_FAULT_COMPLETED = (__force vm_fault_t )0x004000 , +VM_FAULT_VMA_UNLOCKED = (__force vm_fault_t )0x008000 , VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, };
Quoted 10 lines
include/linux/mm_types.h enum vm_fault_reason { · line 1076
Quoted 7 lines
include/linux/mm_types.h +2
-1@@ -1070,7 +1072,9 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" }, \
VM_FAULT_COMPLETEDisn't used in this patch, guessing that's snuck in from one of the other patches in the series?
I noticed that an entry for VM_FAULT_COMPLETED was missing and wanted to fix that... Should I drop that?
Quoted 32 lines
Quoted 18 lines
include/linux/mm_types.h +1
-0+ { VM_FAULT_VMA_UNLOCKED, "VMA_UNLOCKED" }struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */ mm/memory.c +3
-2@@ -3714,8 +3714,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) { -migration_entry_wait (vma->vm_mm, vmf->pmd, - vmf->address); +/* Save mm in case VMA lock is dropped */ +struct mm_struct *mm = vma->vm_mm; +mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3729
Quoted 10 lines
mm/memory.c +7
-0+if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* No need to hold VMA lock for migration */ +vma_end_read (vma); +/* CAUTION! VMA can't be used after this */ + ret |=VM_FAULT_VMA_UNLOCKED ; + } +migration_entry_wait (mm, vmf->pmd, vmf->address); }else if (is_device_exclusive_entry (entry)) { vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf);--
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
Re: [PATCH 2/3] mm: drop VMA lock before waiting for migration
Quoted 45 lines · Suren Baghdasaryan
Quoted 6 lines · Alistair Popple' via kernel-team
Suren Baghdasaryan <[email protected]> writes:
[...]
Quoted 11 lines
include/linux/mm_types.h typedef __bitwise unsigned int vm_fault_t; · line 1033
Quoted 9 lines
include/linux/mm_types.h +1
-0@@ -1030,6 +1030,7 @@ typedef __bitwise unsigned int vm_fault_t;* fsync() to complete (for synchronous page faults * in DAX) * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released +* @VM_FAULT_VMA_UNLOCKED: VMA lock was released A note here saying
vmf->vmashould no longer be accessed would be nice.Good idea. Will add in the next version. Thanks!
Quoted 12 lines
Quoted 12 lines
include/linux/mm_types.h +1
-0* @VM_FAULT_HINDEX_MASK: mask HINDEX value * */ @@ -1047,6 +1048,7 @@ enum vm_fault_reason {VM_FAULT_DONE_COW = (__force vm_fault_t )0x001000 ,VM_FAULT_NEEDDSYNC = (__force vm_fault_t )0x002000 ,VM_FAULT_COMPLETED = (__force vm_fault_t )0x004000 , +VM_FAULT_VMA_UNLOCKED = (__force vm_fault_t )0x008000 , VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, };Quoted 10 lines
include/linux/mm_types.h enum vm_fault_reason { · line 1076
Quoted 7 lines
include/linux/mm_types.h +2
-1@@ -1070,7 +1072,9 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" }, \
VM_FAULT_COMPLETEDisn't used in this patch, guessing that's snuck in from one of the other patches in the series?
Quoted 2 lines
I noticed that an entry for
VM_FAULT_COMPLETEDwas missing and wanted to fix that... Should I drop that?
Oh ok. It would certainly be good to add but really it should be it's own patch.
Quoted 32 lines
Quoted 32 lines
Quoted 18 lines
include/linux/mm_types.h +1
-0+ { VM_FAULT_VMA_UNLOCKED, "VMA_UNLOCKED" }struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */ mm/memory.c +3
-2@@ -3714,8 +3714,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) { -migration_entry_wait (vma->vm_mm, vmf->pmd, - vmf->address); +/* Save mm in case VMA lock is dropped */ +struct mm_struct *mm = vma->vm_mm; +mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3729
Quoted 10 lines
mm/memory.c +7
-0+if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* No need to hold VMA lock for migration */ +vma_end_read (vma); +/* CAUTION! VMA can't be used after this */ + ret |=VM_FAULT_VMA_UNLOCKED ; + } +migration_entry_wait (mm, vmf->pmd, vmf->address); }else if (is_device_exclusive_entry (entry)) { vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf);--
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
Suren Baghdasaryan
reviewerRe: [PATCH 2/3] mm: drop VMA lock before waiting for migration
Quoted 49 lines · Alistair Popple
Quoted 45 lines · Suren Baghdasaryan
Quoted 6 lines · Alistair Popple' via kernel-team
Suren Baghdasaryan <[email protected]> writes:
[...]
Quoted 11 lines
include/linux/mm_types.h typedef __bitwise unsigned int vm_fault_t; · line 1033
Quoted 9 lines
include/linux/mm_types.h +1
-0@@ -1030,6 +1030,7 @@ typedef __bitwise unsigned int vm_fault_t;* fsync() to complete (for synchronous page faults * in DAX) * @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released +* @VM_FAULT_VMA_UNLOCKED: VMA lock was released A note here saying
vmf->vmashould no longer be accessed would be nice.Good idea. Will add in the next version. Thanks!
Quoted 22 lines
Quoted 13 lines
include/linux/mm_types.h +1
-0* @VM_FAULT_HINDEX_MASK: mask HINDEX value * */ @@ -1047,6 +1048,7 @@ enum vm_fault_reason {VM_FAULT_DONE_COW = (__force vm_fault_t )0x001000 ,VM_FAULT_NEEDDSYNC = (__force vm_fault_t )0x002000 ,VM_FAULT_COMPLETED = (__force vm_fault_t )0x004000 , +VM_FAULT_VMA_UNLOCKED = (__force vm_fault_t )0x008000 , VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000, };include/linux/mm_types.h enum vm_fault_reason { · line 1076
Quoted 6 lines
include/linux/mm_types.h +2
-1@@ -1070,7 +1072,9 @@ enum vm_fault_reason { { VM_FAULT_RETRY, "RETRY" }, \ { VM_FAULT_FALLBACK, "FALLBACK" }, \ { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" } + { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { VM_FAULT_COMPLETED, "COMPLETED" }, \
VM_FAULT_COMPLETEDisn't used in this patch, guessing that's snuck in from one of the other patches in the series?
Quoted 5 lines
Quoted 2 lines
I noticed that an entry for
VM_FAULT_COMPLETEDwas missing and wanted to fix that... Should I drop that?Oh ok. It would certainly be good to add but really it should be it's own patch.
Ack. Will split in the next version. Thanks!
Quoted 34 lines
Quoted 33 lines
Quoted 33 lines
Quoted 19 lines
include/linux/mm_types.h +1
-0+ { VM_FAULT_VMA_UNLOCKED, "VMA_UNLOCKED" }struct vm_special_mapping {const char *name;/* The name, e.g. "[vdso]". */ mm/memory.c +3
-2@@ -3714,8 +3714,16 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) { -migration_entry_wait (vma->vm_mm, vmf->pmd, - vmf->address); +/* Save mm in case VMA lock is dropped */ +struct mm_struct *mm = vma->vm_mm; +mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3729
Quoted 10 lines
mm/memory.c +7
-0+if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { +/* No need to hold VMA lock for migration */ +vma_end_read (vma); +/* CAUTION! VMA can't be used after this */ + ret |=VM_FAULT_VMA_UNLOCKED ; + } +migration_entry_wait (mm, vmf->pmd, vmf->address); }else if (is_device_exclusive_entry (entry)) { vmf->page =pfn_swap_entry_to_page (entry); ret =remove_device_exclusive_entry (vmf);--
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].
Matthew Wilcox
maintainerRe: [PATCH 2/3] mm: drop VMA lock before waiting for migration
In reply to Suren BaghdasaryanQuoted 5 lines · Suren Baghdasaryan
migration_entry_waitdoes not need VMA lock, therefore it can be dropped before waiting. IntroduceVM_FAULT_VMA_UNLOCKEDto indicate that VMA lock was dropped while inhandle_mm_fault().
Note that once VMA lock is dropped, the VMA reference can't be used as there are no guarantees it was not freed.
How about we introduce:
void vmf_end_read(struct vm_fault *vmf)
{
if (!vmf->vma)
return;
vma_end_read(vmf->vma);
vmf->vma = NULL;
}Now we don't need a new flag, and calling vmf_end_read() is idempotent.
Oh, argh, we create the vmf too late. We really need to hoist the creation of vm_fault to the callers of handle_mm_fault().
Suren Baghdasaryan
reviewerRe: [PATCH 2/3] mm: drop VMA lock before waiting for migration
Quoted 18 lines · Matthew Wilcox
Quoted 5 lines · Suren Baghdasaryan
migration_entry_waitdoes not need VMA lock, therefore it can be dropped before waiting. IntroduceVM_FAULT_VMA_UNLOCKEDto indicate that VMA lock was dropped while inhandle_mm_fault().
Note that once VMA lock is dropped, the VMA reference can't be used as there are no guarantees it was not freed.How about we introduce:
void
vmf_end_read(struct vm_fault *vmf){ if (!vmf->vma) return; vma_end_read(vmf->vma); vmf->vma = NULL; }
Quoted 4 lines
Now we don't need a new flag, and calling
vmf_end_read()is idempotent.Oh, argh, we create the vmf too late. We really need to hoist the creation of
vm_faultto the callers ofhandle_mm_fault().
Yeah, unfortunately vmf does not propagate all the way up to do_user_addr_fault which needs to know that we dropped the lock.
Quoted 1 line
Suren Baghdasaryan
reviewerPATCH 3/3 mm: implement folio wait under VMA lock
In reply to Suren BaghdasaryanFollow the same pattern as mmap_lock when waiting for folio by dropping VMA lock before the wait and retrying once folio is available.
--- include/linux/pagemap.h | 14 ++++++++++---- mm/filemap.c | 43 ++++++++++++++++++++++------------------- mm/memory.c | 13 +++++++++---- 3 files changed, 42 insertions(+), 28 deletions(-)
include/linux/pagemap.h +10 -4
@@ -896,8 +896,8 @@ static inline bool wake_page_match(struct wait_page_queue *wait_page,void __folio_lock (struct folio *folio);int __folio_lock_killable (struct folio *folio); -bool __folio_lock_or_retry (struct folio *folio,struct mm_struct *mm, -unsigned int flags); +bool __folio_lock_or_retry (struct folio *folio,struct vm_area_struct *vma, +unsigned int flags,bool *lock_dropped);void unlock_page (struct page *page);void folio_unlock (struct folio *folio);
@@ -1002,10 +1002,16 @@ static inline int folio_lock_killable(struct folio *folio)* __folio_lock_or_retry(). */ static inline bool folio_lock_or_retry (struct folio *folio, -struct mm_struct *mm,unsigned int flags) +struct vm_area_struct *vma,unsigned int flags, +bool *lock_dropped) {might_sleep (); -return folio_trylock (folio) ||__folio_lock_or_retry (folio, mm, flags); +if (folio_trylock (folio)) { + *lock_dropped = false; +return true; + } + +return __folio_lock_or_retry (folio, vma, flags, lock_dropped); }/*
mm/filemap.c +23 -20
73 lines not on this page · raw message
mm/memory.c +9 -4
@@ -3568,6 +3568,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)struct folio *folio =page_folio (vmf->page);struct vm_area_struct *vma = vmf->vma;struct mmu_notifier_range range; +bool lock_dropped;/* * We need a reference to lock the folio because we don't hold
@@ -3580,8 +3581,10 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)if (!folio_try_get (folio))return 0 ; -if (!folio_lock_or_retry (folio, vma->vm_mm, vmf->flags)) { +if (!folio_lock_or_retry (folio, vma, vmf->flags, &lock_dropped)) {folio_put (folio); +if (lock_dropped && vmf->flags &FAULT_FLAG_VMA_LOCK ) +return VM_FAULT_VMA_UNLOCKED |VM_FAULT_RETRY ;return VM_FAULT_RETRY ; } mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
@@ -3704,7 +3707,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)bool exclusive = false;swp_entry_t entry;pte_t pte; -int locked; +bool locked; +bool lock_dropped;vm_fault_t ret =0 ;void *shadow =NULL ;
@@ -3837,9 +3841,10 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out_release; } - locked =folio_lock_or_retry (folio, vma->vm_mm, vmf->flags); - + locked =folio_lock_or_retry (folio, vma, vmf->flags, &lock_dropped);if (!locked) { +if (lock_dropped && vmf->flags &FAULT_FLAG_VMA_LOCK ) + ret |=VM_FAULT_VMA_UNLOCKED ; ret |=VM_FAULT_RETRY ; goto out_release; }
Signature · 1 line
2.40.1.495.gc816e09b53d-goog
Matthew Wilcox
maintainermm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {
You're missing the necessary fallback in the (!folio) case. swap_readpage() is synchronous and will sleep.
Suren Baghdasaryan
reviewerQuoted 7 lines · Matthew Wilcox
Quoted 5 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-0@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out;
Quoted 11 lines
mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 8 lines
mm/memory.c +0
-5-if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.
True, but is it unsafe to do that under VMA lock and has to be done under mmap_lock?
Matthew Wilcox
maintainerQuoted 20 lines · Suren Baghdasaryan
Quoted 7 lines · Matthew Wilcox
Quoted 5 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-0@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out;Quoted 11 lines
mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 8 lines
mm/memory.c +0
-5-if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?
... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Suren Baghdasaryan
reviewerQuoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.
Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in folio_lock_or_retry() with the lock being held. I argued against it because currently we drop mmap_lock lock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case of swap_readpage and swapin_readahead we already wait with mmap_lock held, so waiting with VMA lock held does not introduce new problems (unless there is a need to hold mmap_lock).
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?
Matthew Wilcox
maintainerQuoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).
Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?
OK, I looked at this path some more, and I think we're fine. This patch is only called for SWP_SYNCHRONOUS_IO which is only set for QUEUE_FLAG_SYNCHRONOUS devices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.
Suren Baghdasaryan
reviewerQuoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
mm/memory.c vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.cmm/memory.c +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)if (!pte_unmap_same (vmf)) goto out; -if (vmf->flags &FAULT_FLAG_VMA_LOCK ) { - ret =VM_FAULT_RETRY ; - goto out; - } - entry =pte_to_swp_entry (vmf->orig_pte);if (unlikely (non_swap_entry (entry))) {if (is_migration_entry (entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?
Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.
Yes but swapin_readahead does sleep, so I'll have to handle that case too after this.
Matthew Wilcox
maintainerQuoted 47 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 43 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?
Quoted 8 lines
Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.
Sleeping is OK, we do that in pXd_alloc()! Do we block on I/O anywhere in swapin_readahead()? It all looks like async I/O to me.
Suren Baghdasaryan
reviewerQuoted 55 lines · Matthew Wilcox
Quoted 53 lines · Suren Baghdasaryan
Quoted 51 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.
Quoted 5 lines
Quoted 2 lines
Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.
Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage()
swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases swap_readpage() is called with the synchronous parameter being false. So you are probably right here... Does that mean swapin_readahead() might return a page which does not have its content swapped-in yet?
Matthew Wilcox
maintainerQuoted 62 lines · Suren Baghdasaryan
Quoted 55 lines · Matthew Wilcox
Quoted 53 lines · Suren Baghdasaryan
Quoted 51 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Quoted 5 lines
Quoted 2 lines
Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.
Quoted 7 lines
Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here... Does that meanswapin_readahead()might return a page which does not have its content swapped-in yet?
That's my understanding. In that case it's !uptodate and still locked. The folio_lock_or_retry() will wait for the read to complete unless we've told it we'd rather retry.
Suren Baghdasaryan
reviewerQuoted 72 lines · Matthew Wilcox
Quoted 62 lines · Suren Baghdasaryan
Quoted 55 lines · Matthew Wilcox
Quoted 53 lines · Suren Baghdasaryan
Quoted 51 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Quoted 5 lines
Quoted 2 lines
Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Quoted 7 lines
Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here... Does that meanswapin_readahead()might return a page which does not have its content swapped-in yet?
Quoted 3 lines
That's my understanding. In that case it's !uptodate and still locked. The
folio_lock_or_retry()will wait for the read to complete unless we've told it we'd rather retry.
Ok, and we already drop the locks in folio_lock_or_retry() when needed. Sounds like we cover this case with this patchset?
Quoted 60 lines · Suren Baghdasaryan
Quoted 58 lines · Matthew Wilcox
Quoted 55 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.
Quoted 8 lines
Quoted 2 lines
Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here...
In both swap_cluster_readahead() and swap_vma_readahead() it looks like if the readahead window is 1 (aka we are not reading ahead), then we jump to directly calling read_swap_cache_async() passing do_poll = true, which means we may end up calling swap_readpage() passing synchronous = true.
I am not familiar with readahead heuristics, so I am not sure how common this is, but it's something to think about.
Quoted 3 lines
Does that mean
swapin_readahead()might return a page which does not have its content swapped-in yet?
Suren Baghdasaryan
reviewerQuoted 71 lines · Yosry Ahmed
Quoted 60 lines · Suren Baghdasaryan
Quoted 58 lines · Matthew Wilcox
Quoted 55 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Quoted 8 lines
Quoted 2 lines
Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here...
Quoted 8 lines
In both
swap_cluster_readahead()andswap_vma_readahead()it looks like if the readahead window is 1 (aka we are not reading ahead), then we jump to directly callingread_swap_cache_async()passingdo_poll= true, which means we may end up callingswap_readpage()passing synchronous = true.I am not familiar with readahead heuristics, so I am not sure how common this is, but it's something to think about.
Uh, you are correct. If this branch is common, we could use the same "drop the lock and retry" pattern inside read_swap_cache_async(). That would be quite easy to implement.
Thanks for checking on it!
Quoted 4 lines
Quoted 4 lines
Does that mean
swapin_readahead()might return a page which does not have its content swapped-in yet?
Quoted 79 lines · Suren Baghdasaryan
Quoted 77 lines · Yosry Ahmed
Quoted 60 lines · Suren Baghdasaryan
Quoted 58 lines · Matthew Wilcox
Quoted 55 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Quoted 8 lines
Quoted 2 lines
Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here...In both
swap_cluster_readahead()andswap_vma_readahead()it looks like if the readahead window is 1 (aka we are not reading ahead), then we jump to directly callingread_swap_cache_async()passingdo_poll= true, which means we may end up callingswap_readpage()passing synchronous = true.
Quoted 7 lines
Quoted 2 lines
I am not familiar with readahead heuristics, so I am not sure how common this is, but it's something to think about.
Uh, you are correct. If this branch is common, we could use the same "drop the lock and retry" pattern inside
read_swap_cache_async(). That would be quite easy to implement.
Thanks for checking on it!
I am honestly not sure how common this is.
+Ying who might have a better idea.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Does that mean
swapin_readahead()might return a page which does not have its content swapped-in yet?
Quoted 89 lines · Yosry Ahmed
Quoted 79 lines · Suren Baghdasaryan
Quoted 77 lines · Yosry Ahmed
Quoted 60 lines · Suren Baghdasaryan
Quoted 58 lines · Matthew Wilcox
Quoted 55 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Review vm_fault_t do_swap_page(struct vm_fault *vmf) · line 3716
Quoted 13 lines · Suren Baghdasaryan
+++
b/mm/memory.c(quoted diff) +0
-5@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (!pte_unmap_same(vmf)) goto out; - if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - ret = VM_FAULT_RETRY; - goto out; - } - entry = pte_to_swp_entry(vmf->orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Quoted 8 lines
Quoted 2 lines
Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here...In both
swap_cluster_readahead()andswap_vma_readahead()it looks like if the readahead window is 1 (aka we are not reading ahead), then we jump to directly callingread_swap_cache_async()passingdo_poll= true, which means we may end up callingswap_readpage()passing synchronous = true.Quoted 7 lines
Quoted 2 lines
I am not familiar with readahead heuristics, so I am not sure how common this is, but it's something to think about.
Uh, you are correct. If this branch is common, we could use the same "drop the lock and retry" pattern inside
read_swap_cache_async(). That would be quite easy to implement.
Thanks for checking on it!
Quoted 3 lines
I am honestly not sure how common this is.
+Ying who might have a better idea.
Checked the code and related history. It seems that we can just pass "synchronous = false" to swap_readpage() in read_swap_cache_async(). "synchronous = true" was introduced in commit 23955622ff8d ("swap: add block io poll in swapin path") to reduce swap read latency for block devices that can be polled. But in commit 9650b453a3d4 ("block: ignore RWF_HIPRI hint for sync dio"), the polling is deleted. So, we don't need to pass "synchronous = true" to swap_readpage() during swapin_readahead(), because we will wait the IO to complete in folio_lock_or_retry().
Best Regards,
Huang, Ying
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Does that mean
swapin_readahead()might return a page which does not have its content swapped-in yet?
Suren Baghdasaryan
reviewerQuoted 96 lines · Ying
Quoted 89 lines · Yosry Ahmed
Quoted 79 lines · Suren Baghdasaryan
Quoted 77 lines · Yosry Ahmed
Quoted 60 lines · Suren Baghdasaryan
Quoted 58 lines · Matthew Wilcox
Quoted 55 lines · Suren Baghdasaryan
Quoted 46 lines · Matthew Wilcox
Quoted 37 lines · Suren Baghdasaryan
Quoted 22 lines · Matthew Wilcox
Quoted 20 lines · Suren Baghdasaryan
Quoted 18 lines · Matthew Wilcox
Quoted 13 lines · Suren Baghdasaryan
+++ b/mm/memory.c
@@ -3711,11 +3711,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
if (!pte_unmap_same(vmf))
goto out;- if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
- ret = VM_FAULT_RETRY;
- goto out;
- }
-
entry = pte_to_swp_entry(vmf->orig_pte);
if (unlikely(non_swap_entry(entry))) {
if (is_migration_entry(entry)) {You're missing the necessary fallback in the (!folio) case.
swap_readpage()is synchronous and will sleep.Quoted 5 lines
Quoted 2 lines
True, but is it unsafe to do that under VMA lock and has to be done under
mmap_lock?... you were the one arguing that we didn't want to wait for I/O with the VMA lock held?
Well, that discussion was about waiting in
folio_lock_or_retry()with the lock being held. I argued against it because currently we dropmmap_locklock before waiting, so if we don't drop VMA lock we would be changing the current behavior which might introduce new regressions. In the case ofswap_readpageandswapin_readaheadwe already wait withmmap_lockheld, so waiting with VMA lock held does not introduce new problems (unless there is a need to holdmmap_lock).Quoted 6 lines
That said, you are absolutely correct that this situation can be improved by dropping the lock in these cases too. I just didn't want to attack everything at once. I believe after we agree on the approach implemented in https://lore.kernel.org/all/[email protected]
for dropping the VMA lock before waiting, these cases can be added easier. Does that make sense?Quoted 5 lines
OK, I looked at this path some more, and I think we're fine. This patch is only called for
SWP_SYNCHRONOUS_IOwhich is only set forQUEUE_FLAG_SYNCHRONOUSdevices, which are brd, zram and nvdimms (both btt and pmem). So the answer is that we don't sleep in this path, and there's no need to drop the lock.Yes but
swapin_readaheaddoes sleep, so I'll have to handle that case too after this.Quoted 8 lines
Quoted 2 lines
Sleeping is OK, we do that in
pXd_alloc()! Do we block on I/O anywhere inswapin_readahead()? It all looks like async I/O to me.Hmm. I thought that we have synchronous I/O in the following paths:
swapin_readahead()->swap_cluster_readahead()->swap_readpage() swapin_readahead()->swap_vma_readahead()->swap_readpage()but just noticed that in both cases
swap_readpage()is called with the synchronous parameter being false. So you are probably right here...In both
swap_cluster_readahead()andswap_vma_readahead()it looks like if the readahead window is 1 (aka we are not reading ahead), then we jump to directly callingread_swap_cache_async()passingdo_poll= true, which means we may end up callingswap_readpage()passing synchronous = true.Quoted 7 lines
Quoted 2 lines
I am not familiar with readahead heuristics, so I am not sure how common this is, but it's something to think about.
Uh, you are correct. If this branch is common, we could use the same "drop the lock and retry" pattern inside
read_swap_cache_async(). That would be quite easy to implement.
Thanks for checking on it!Quoted 3 lines
I am honestly not sure how common this is.
+Ying who might have a better idea.
Quoted 9 lines
Checked the code and related history. It seems that we can just pass "synchronous = false" to
swap_readpage()inread_swap_cache_async(). "synchronous = true" was introduced in commit23955622ff8d("swap: add block io poll in swapin path") to reduce swap read latency for block devices that can be polled. But in commit9650b453a3d4("block: ignoreRWF_HIPRIhint for sync dio"), the polling is deleted. So, we don't need to pass "synchronous = true" toswap_readpage()duringswapin_readahead(), because we will wait the IO to complete infolio_lock_or_retry().
Thanks for investigating, Ying! It sounds like we can make some simplifications here. I'll double-check and if I don't find anything else, will change to "synchronous = false" in the next version of the patchset.
Quoted 14 lines
Best Regards,
Huang, YingQuoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Does that mean
swapin_readahead()might return a page which does not have its content swapped-in yet?--
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected].