userfaultfd: handle zeropage moves by UFFDIO_MOVE
Subsystems (5)
PATCH v2 1/1 userfaultfd: handle zeropage moves by UFFDIO_MOVE
Current implementation of UFFDIO_MOVE fails to move zeropages and returns EBUSY when it encounters one. We can handle them by mapping a zeropage at the destination and clearing the mapping at the source. This is done both for ordinary and for huge zeropages.
Reported-by kernel test robot <[email protected]>
Reported-by Dan Carpenter <[email protected]>
Signed-off-by Suren Baghdasaryan <[email protected]>
---
Changes since v1 [1]
- Added missing double_pt_unlock in move_zeropage_pte, per Dan Carpenter
- Added Reported-by and Closes tags per bug report [2]
Applies cleanly over mm-unstable branch.
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/
mm/huge_memory.c | 105 +++++++++++++++++++++++++++-------------------- mm/userfaultfd.c | 44 ++++++++++++++++---- 2 files changed, 98 insertions(+), 51 deletions(-)
mm/huge_memory.c +61 -44
@@ -2200,13 +2200,18 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } src_page =pmd_page (src_pmdval); -if (unlikely (!PageAnonExclusive (src_page))) { -spin_unlock (src_ptl); -return -EBUSY ; - } - src_folio =page_folio (src_page); -folio_get (src_folio); +if (!is_huge_zero_pmd (src_pmdval)) { +if (unlikely (!PageAnonExclusive (src_page))) { +spin_unlock (src_ptl); +return -EBUSY ; + } + + src_folio =page_folio (src_page); +folio_get (src_folio); + }else + src_folio =NULL ; +spin_unlock (src_ptl);flush_cache_range (src_vma, src_addr, src_addr +HPAGE_PMD_SIZE );
@@ -2214,19 +2219,22 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm src_addr + HPAGE_PMD_SIZE);mmu_notifier_invalidate_range_start (&range); -folio_lock (src_folio); +if (src_folio) { +folio_lock (src_folio); -/* -* split_huge_page walks the anon_vma chain without the page -* lock. Serialize against it with the anon_vma lock, the page -* lock is not enough. -*/ - src_anon_vma =folio_get_anon_vma (src_folio); -if (!src_anon_vma) { - err = -EAGAIN ; - goto unlock_folio; - } -anon_vma_lock_write (src_anon_vma); +/* +* split_huge_page walks the anon_vma chain without the page +* lock. Serialize against it with the anon_vma lock, the page +* lock is not enough. +*/ + src_anon_vma =folio_get_anon_vma (src_folio); +if (!src_anon_vma) { + err = -EAGAIN ; + goto unlock_folio; + } +anon_vma_lock_write (src_anon_vma); + }else + src_anon_vma =NULL ; dst_ptl =pmd_lockptr (mm, dst_pmd);double_pt_lock (src_ptl, dst_ptl);
@@ -2235,45 +2243,54 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm err = -EAGAIN ; goto unlock_ptls; } -if (folio_maybe_dma_pinned (src_folio) || - !PageAnonExclusive (&src_folio->page)) { - err = -EBUSY ; - goto unlock_ptls; - } +if (src_folio) { +if (folio_maybe_dma_pinned (src_folio) || + !PageAnonExclusive (&src_folio->page)) { + err = -EBUSY ; + goto unlock_ptls; + } -if (WARN_ON_ONCE (!folio_test_head (src_folio)) || -WARN_ON_ONCE (!folio_test_anon (src_folio))) { - err = -EBUSY ; - goto unlock_ptls; - } +if (WARN_ON_ONCE (!folio_test_head (src_folio)) || +WARN_ON_ONCE (!folio_test_anon (src_folio))) { + err = -EBUSY ; + goto unlock_ptls; + } -folio_move_anon_rmap (src_folio, dst_vma); -WRITE_ONCE (src_folio->index,linear_page_index (dst_vma, dst_addr)); +folio_move_anon_rmap (src_folio, dst_vma); +WRITE_ONCE (src_folio->index,linear_page_index (dst_vma, dst_addr)); - src_pmdval =pmdp_huge_clear_flush (src_vma, src_addr, src_pmd); -/* Folio got pinned from under us. Put it back and fail the move. */ -if (folio_maybe_dma_pinned (src_folio)) { -set_pmd_at (mm, src_addr, src_pmd, src_pmdval); - err = -EBUSY ; - goto unlock_ptls; - } + src_pmdval =pmdp_huge_clear_flush (src_vma, src_addr, src_pmd); +/* Folio got pinned from under us. Put it back and fail the move. */ +if (folio_maybe_dma_pinned (src_folio)) { +set_pmd_at (mm, src_addr, src_pmd, src_pmdval); + err = -EBUSY ; + goto unlock_ptls; + } - _dst_pmd =mk_huge_pmd (&src_folio->page, dst_vma->vm_page_prot); -/* Follow mremap() behavior and treat the entry dirty after the move */ - _dst_pmd =pmd_mkwrite (pmd_mkdirty (_dst_pmd), dst_vma); + _dst_pmd =mk_huge_pmd (&src_folio->page, dst_vma->vm_page_prot); +/* Follow mremap() behavior and treat the entry dirty after the move */ + _dst_pmd =pmd_mkwrite (pmd_mkdirty (_dst_pmd), dst_vma); + }else { + src_pmdval =pmdp_huge_clear_flush (src_vma, src_addr, src_pmd); + _dst_pmd =mk_huge_pmd (src_page, dst_vma->vm_page_prot); + }set_pmd_at (mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable =pgtable_trans_huge_withdraw (mm, src_pmd);pgtable_trans_huge_deposit (mm, dst_pmd, src_pgtable); unlock_ptls:double_pt_unlock (src_ptl, dst_ptl); -anon_vma_unlock_write (src_anon_vma); -put_anon_vma (src_anon_vma); +if (src_anon_vma) { +anon_vma_unlock_write (src_anon_vma); +put_anon_vma (src_anon_vma); + } unlock_folio:/* unblock rmap walks */ -folio_unlock (src_folio); +if (src_folio) +folio_unlock (src_folio);mmu_notifier_invalidate_range_end (&range); -folio_put (src_folio); +if (src_folio) +folio_put (src_folio);return err; }#endif /* CONFIG_USERFAULTFD */
mm/userfaultfd.c +37 -7
68 lines not on this page · raw message
Signature · 1 line
2.43.0.429.g432eaa2c6b-goog