userfaultfd: enable zeropage support for shmem
Subsystems (5)
cover userfaultfd: enable zeropage support for shmem
Hi,
These patches enable support for UFFDIO_ZEROPAGE for shared memory.
The first two patches are not strictly related to userfaultfd, they are just minor refactoring to reduce amount of code duplication.
Mike Rapoport (7):
shmem: shmem_charge: verify max_block is not exceeded before inode update
shmem: introduce shmem_inode_acct_block
userfaultfd: shmem: add shmem_mfill_zeropage_pte for userfaultfd support
userfaultfd: mcopy_atomic: introduce mfill_atomic_pte helper
userfaultfd: shmem: wire up shmem_mfill_zeropage_pte
userfaultfd: report UFFDIO_ZEROPAGE as available for shmem VMAs
userfaultfd: selftest: enable testing of UFFDIO_ZEROPAGE for shmem
fs/userfaultfd.c | 10 +- include/linux/shmem_fs.h | 6 ++ mm/shmem.c | 167 +++++++++++++++++-------------- mm/userfaultfd.c | 48 ++++++--- tools/testing/selftests/vm/userfaultfd.c | 2 +- 5 files changed, 136 insertions(+), 97 deletions(-)
Signature · 1 line
2.7.4
PATCH 1/7 shmem: shmem_charge: verify max_block is not exceeded before inode update
Currently we update inode and shmem_inode_info before verifying that used_blocks will not exceed max_blocks. In case it will, we undo the update. Let's switch the order and move the verification of the blocks count before the inode and shmem_inode_info update.
--- mm/shmem.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)
mm/shmem.c +12 -13
@@ -265,6 +265,14 @@ bool shmem_charge(struct inode *inode, long pages)if (shmem_acct_block (info->flags, pages))return false; + +if (sbinfo->max_blocks) { +if (percpu_counter_compare (&sbinfo->used_blocks, + sbinfo->max_blocks - pages) >0 ) + goto unacct; +percpu_counter_add (&sbinfo->used_blocks, pages); + } +spin_lock_irqsave (&info->lock, flags); info->alloced += pages; inode->i_blocks += pages *BLOCKS_PER_PAGE ;
@@ -272,20 +280,11 @@ bool shmem_charge(struct inode *inode, long pages)spin_unlock_irqrestore (&info->lock, flags); inode->i_mapping->nrpages += pages; -if (!sbinfo->max_blocks) -return true; -if (percpu_counter_compare (&sbinfo->used_blocks, - sbinfo->max_blocks - pages) >0 ) { - inode->i_mapping->nrpages -= pages; -spin_lock_irqsave (&info->lock, flags); - info->alloced -= pages; -shmem_recalc_inode (inode); -spin_unlock_irqrestore (&info->lock, flags); -shmem_unacct_blocks (info->flags, pages); -return false; - } -percpu_counter_add (&sbinfo->used_blocks, pages);return true; + +unacct: +shmem_unacct_blocks (info->flags, pages); +return false; }void shmem_uncharge (struct inode *inode,long pages)
Signature · 1 line
2.7.4
PATCH 2/7 shmem: introduce shmem_inode_acct_block
In reply to Mike RapoportThe shmem_acct_block and the update of used_blocks are following one another in all the places they are used. Combine these two into a helper function.
--- mm/shmem.c | 102 ++++++++++++++++++++++++++++--------------------------------- 1 file changed, 46 insertions(+), 56 deletions(-)
mm/shmem.c +46 -56
@@ -187,6 +187,38 @@ static inline void shmem_unacct_blocks(unsigned long flags, long pages)vm_unacct_memory (pages *VM_ACCT (PAGE_SIZE )); } +static inline bool shmem_inode_acct_block (struct inode *inode,long pages) +{ +struct shmem_inode_info *info =SHMEM_I (inode); +struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb); + +if (shmem_acct_block (info->flags, pages)) +return false; + +if (sbinfo->max_blocks) { +if (percpu_counter_compare (&sbinfo->used_blocks, + sbinfo->max_blocks - pages) >0 ) + goto unacct; +percpu_counter_add (&sbinfo->used_blocks, pages); + } + +return true; + +unacct: +shmem_unacct_blocks (info->flags, pages); +return false; +} + +static inline void shmem_inode_unacct_blocks (struct inode *inode,long pages) +{ +struct shmem_inode_info *info =SHMEM_I (inode); +struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb); + +if (sbinfo->max_blocks) +percpu_counter_sub (&sbinfo->used_blocks, pages); +shmem_unacct_blocks (info->flags, pages); +} +static const struct super_operations shmem_ops;static const struct address_space_operations shmem_aops;static const struct file_operations shmem_file_operations;
@@ -248,31 +280,20 @@ static void shmem_recalc_inode(struct inode *inode) freed = info->alloced - info->swapped - inode->i_mapping->nrpages;if (freed >0 ) { -struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb); -if (sbinfo->max_blocks) -percpu_counter_add (&sbinfo->used_blocks, -freed); info->alloced -= freed; inode->i_blocks -= freed *BLOCKS_PER_PAGE ; -shmem_unacct_blocks (info->flags, freed); +shmem_inode_unacct_blocks (inode, freed); } }bool shmem_charge (struct inode *inode,long pages) {struct shmem_inode_info *info =SHMEM_I (inode); -struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb);unsigned long flags; -if (shmem_acct_block (info->flags, pages)) +if (!shmem_inode_acct_block (inode, pages))return false; -if (sbinfo->max_blocks) { -if (percpu_counter_compare (&sbinfo->used_blocks, - sbinfo->max_blocks - pages) >0 ) - goto unacct; -percpu_counter_add (&sbinfo->used_blocks, pages); - } -spin_lock_irqsave (&info->lock, flags); info->alloced += pages; inode->i_blocks += pages *BLOCKS_PER_PAGE ;
@@ -281,16 +302,11 @@ bool shmem_charge(struct inode *inode, long pages) inode->i_mapping->nrpages += pages;return true; - -unacct: -shmem_unacct_blocks (info->flags, pages); -return false; }void shmem_uncharge (struct inode *inode,long pages) {struct shmem_inode_info *info =SHMEM_I (inode); -struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb);unsigned long flags;spin_lock_irqsave (&info->lock, flags);
@@ -299,9 +315,7 @@ void shmem_uncharge(struct inode *inode, long pages)shmem_recalc_inode (inode);spin_unlock_irqrestore (&info->lock, flags); -if (sbinfo->max_blocks) -percpu_counter_sub (&sbinfo->used_blocks, pages); -shmem_unacct_blocks (info->flags, pages); +shmem_inode_unacct_blocks (inode, pages); }/*
@@ -1446,9 +1460,10 @@ static struct page *shmem_alloc_page(gfp_t gfp, }static struct page *shmem_alloc_and_acct_page (gfp_t gfp, -struct shmem_inode_info *info,struct shmem_sb_info *sbinfo, +struct inode *inode,pgoff_t index,bool huge) { +struct shmem_inode_info *info =SHMEM_I (inode);struct page *page;int nr;int err = -ENOSPC ;
@@ -1457,14 +1472,8 @@ static struct page *shmem_alloc_and_acct_page(gfp_t gfp, huge = false; nr = huge ?HPAGE_PMD_NR :1 ; -if (shmem_acct_block (info->flags, nr)) +if (!shmem_inode_acct_block (inode, nr)) goto failed; -if (sbinfo->max_blocks) { -if (percpu_counter_compare (&sbinfo->used_blocks, - sbinfo->max_blocks - nr) >0 ) - goto unacct; -percpu_counter_add (&sbinfo->used_blocks, nr); - }if (huge) page =shmem_alloc_hugepage (gfp, info, index);
@@ -1477,10 +1486,7 @@ static struct page *shmem_alloc_and_acct_page(gfp_t gfp, } err = -ENOMEM ; -if (sbinfo->max_blocks) -percpu_counter_add (&sbinfo->used_blocks, -nr); -unacct: -shmem_unacct_blocks (info->flags, nr); +shmem_inode_unacct_blocks (inode, nr); failed:return ERR_PTR (err); }
@@ -1746,10 +1752,9 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, } alloc_huge: - page =shmem_alloc_and_acct_page (gfp, info, sbinfo, - index, true); + page =shmem_alloc_and_acct_page (gfp, inode, index, true);if (IS_ERR (page)) { -alloc_nohuge: page =shmem_alloc_and_acct_page (gfp, info, sbinfo, +alloc_nohuge: page =shmem_alloc_and_acct_page (gfp, inode, index, false); }if (IS_ERR (page)) {
@@ -1867,10 +1872,7 @@ alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo,* Error recovery. */ unacct: -if (sbinfo->max_blocks) -percpu_counter_sub (&sbinfo->used_blocks, -1 <<compound_order (page)); -shmem_unacct_blocks (info->flags,1 <<compound_order (page)); +shmem_inode_unacct_blocks (inode,1 <<compound_order (page));if (PageTransHuge (page)) {unlock_page (page);
@@ -2204,7 +2206,6 @@ int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, {struct inode *inode =file_inode (dst_vma->vm_file);struct shmem_inode_info *info =SHMEM_I (inode); -struct shmem_sb_info *sbinfo =SHMEM_SB (inode->i_sb);struct address_space *mapping = inode->i_mapping;gfp_t gfp =mapping_gfp_mask (mapping);pgoff_t pgoff =linear_page_index (dst_vma, dst_addr);
@@ -2216,19 +2217,13 @@ int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,int ret; ret = -ENOMEM ; -if (shmem_acct_block (info->flags,1 )) +if (!shmem_inode_acct_block (inode,1 )) goto out; -if (sbinfo->max_blocks) { -if (percpu_counter_compare (&sbinfo->used_blocks, - sbinfo->max_blocks) >=0 ) - goto out_unacct_blocks; -percpu_counter_inc (&sbinfo->used_blocks); - }if (!*pagep) { page =shmem_alloc_page (gfp, info, pgoff);if (!page) - goto out_dec_used_blocks; + goto out_unacct_blocks; page_kaddr =kmap_atomic (page); ret = copy_from_user(page_kaddr, (const void __user *)src_addr,
@@ -2238,9 +2233,7 @@ int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,/* fallback to copy_from_user outside mmap_sem */ if (unlikely (ret)) { *pagep = page; -if (sbinfo->max_blocks) -percpu_counter_add (&sbinfo->used_blocks,-1 ); -shmem_unacct_blocks (info->flags,1 ); +shmem_inode_unacct_blocks (inode,1 );/* don't free the page */ return -EFAULT ; }
@@ -2303,11 +2296,8 @@ int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, out_release:unlock_page (page);put_page (page); -out_dec_used_blocks: -if (sbinfo->max_blocks) -percpu_counter_add (&sbinfo->used_blocks,-1 ); out_unacct_blocks: -shmem_unacct_blocks (info->flags,1 ); +shmem_inode_unacct_blocks (inode,1 ); goto out; }
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 3/7 userfaultfd: shmem: add shmem_mfill_zeropage_pte for userfaultfd support
In reply to Mike Rapoportshmem_mfill_zeropage_pte is the low level routine that implements the userfaultfd UFFDIO_ZEROPAGE command. Since for shmem mappings zero pages are always allocated and accounted, the new method is a slight extension of the existing shmem_mcopy_atomic_pte.
--- include/linux/shmem_fs.h | 6 +++++ mm/shmem.c | 62 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 17 deletions(-)
include/linux/shmem_fs.h +6 -0
@@ -137,9 +137,15 @@ extern int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,unsigned long dst_addr, unsigned long src_addr, struct page **pagep); +extern int shmem_mfill_zeropage_pte (struct mm_struct *dst_mm, +pmd_t *dst_pmd, +struct vm_area_struct *dst_vma, +unsigned long dst_addr);#else #define shmem_mcopy_atomic_pte (dst_mm, dst_pte, dst_vma, dst_addr, \ src_addr, pagep) ({ BUG(); 0; }) +#define shmem_mfill_zeropage_pte (dst_mm, dst_pmd, dst_vma, \ + dst_addr) ({ BUG(); 0; })#endif #endif
mm/shmem.c +45 -17
80 lines not on this page · raw message
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 4/7 userfaultfd: mcopy_atomic: introduce mfill_atomic_pte helper
In reply to Mike RapoportShuffle the code a bit to improve readability.
--- mm/userfaultfd.c | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-)
mm/userfaultfd.c +30 -16
@@ -371,6 +371,34 @@ extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, bool zeropage);#endif /* CONFIG_HUGETLB_PAGE */ +static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, +pmd_t *dst_pmd, +struct vm_area_struct *dst_vma, +unsigned long dst_addr, +unsigned long src_addr, +struct page **page, +bool zeropage) +{ +ssize_t err; + +if (vma_is_anonymous (dst_vma)) { +if (!zeropage) + err =mcopy_atomic_pte (dst_mm, dst_pmd, dst_vma, + dst_addr, src_addr, page); +else + err =mfill_zeropage_pte (dst_mm, dst_pmd, + dst_vma, dst_addr); + }else { + err = -EINVAL ;/* if zeropage is true return -EINVAL */ +if (likely (!zeropage)) + err =shmem_mcopy_atomic_pte (dst_mm, dst_pmd, + dst_vma, dst_addr, + src_addr, page); + } + +return err; +} + static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start,
@@ -487,22 +515,8 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,BUG_ON (pmd_none (*dst_pmd));BUG_ON (pmd_trans_huge (*dst_pmd)); -if (vma_is_anonymous (dst_vma)) { -if (!zeropage) - err =mcopy_atomic_pte (dst_mm, dst_pmd, dst_vma, - dst_addr, src_addr, - &page); -else - err =mfill_zeropage_pte (dst_mm, dst_pmd, - dst_vma, dst_addr); - }else { - err = -EINVAL ;/* if zeropage is true return -EINVAL */ -if (likely (!zeropage)) - err =shmem_mcopy_atomic_pte (dst_mm, dst_pmd, - dst_vma, dst_addr, - src_addr, &page); - } - + err =mfill_atomic_pte (dst_mm, dst_pmd, dst_vma, dst_addr, + src_addr, &page, zeropage);cond_resched ();if (unlikely (err == -EFAULT )) {
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 5/7 userfaultfd: shmem: wire up shmem_mfill_zeropage_pte
In reply to Mike RapoportFor shmem VMAs we can use shmem_mfill_zeropage_pte for UFFDIO_ZEROPAGE
--- mm/userfaultfd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
mm/userfaultfd.c +4 -2
@@ -389,11 +389,13 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, err =mfill_zeropage_pte (dst_mm, dst_pmd, dst_vma, dst_addr); }else { - err = -EINVAL ;/* if zeropage is true return -EINVAL */ -if (likely (!zeropage)) +if (!zeropage) err =shmem_mcopy_atomic_pte (dst_mm, dst_pmd, dst_vma, dst_addr, src_addr, page); +else + err =shmem_mfill_zeropage_pte (dst_mm, dst_pmd, + dst_vma, dst_addr); }return err;
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 6/7 userfaultfd: report UFFDIO_ZEROPAGE as available for shmem VMAs
In reply to Mike RapoportNow when shmem VMAs can be filled with zero page via userfaultfd we can report that UFFDIO_ZEROPAGE is available for those VMAs
--- fs/userfaultfd.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
fs/userfaultfd.c +5 -5
@@ -1183,7 +1183,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,struct uffdio_register __user *user_uffdio_register;unsigned long vm_flags, new_flags;bool found; -bool non_anon_pages; +bool basic_ioctls;unsigned long start, end, vma_end; user_uffdio_register = (struct uffdio_register __user *) arg;
@@ -1249,7 +1249,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,* Search for not compatible vmas. */ found = false; - non_anon_pages = false; + basic_ioctls = false;for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {cond_resched ();
@@ -1288,8 +1288,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,/* * Note vmas containing huge pages */ -if (is_vm_hugetlb_page (cur) ||vma_is_shmem (cur)) - non_anon_pages = true; +if (is_vm_hugetlb_page (cur)) + basic_ioctls = true; found = true; }
@@ -1360,7 +1360,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,* userland which ioctls methods are guaranteed to * succeed on this range. */ -if (put_user (non_anon_pages ?UFFD_API_RANGE_IOCTLS_BASIC : +if (put_user (basic_ioctls ?UFFD_API_RANGE_IOCTLS_BASIC :UFFD_API_RANGE_IOCTLS , &user_uffdio_register->ioctls)) ret = -EFAULT ;
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 7/7 userfaultfd: selftest: enable testing of UFFDIO_ZEROPAGE for shmem
In reply to Mike Rapoport--- tools/testing/selftests/vm/userfaultfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/testing/selftests/vm/userfaultfd.c +1 -1
@@ -198,7 +198,7 @@ static struct uffd_test_ops anon_uffd_test_ops = { };static struct uffd_test_ops shmem_uffd_test_ops = { - .expected_ioctls =UFFD_API_RANGE_IOCTLS_BASIC , + .expected_ioctls =ANON_EXPECTED_IOCTLS , .allocate_area = shmem_allocate_area, .release_pages = shmem_release_pages, };
Signature · 7 lines
2.7.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Hi,
Any updates/comments?
Quoted 25 lines · Mike Rapoport
Hi,
These patches enable support for
UFFDIO_ZEROPAGEfor shared memory.The first two patches are not strictly related to userfaultfd, they are just minor refactoring to reduce amount of code duplication.
Mike Rapoport (7):
shmem:shmem_charge: verifymax_blockis not exceeded before inode update
shmem: introduceshmem_inode_acct_block
userfaultfd: shmem: addshmem_mfill_zeropage_ptefor userfaultfd support
userfaultfd:mcopy_atomic: introducemfill_atomic_ptehelper
userfaultfd: shmem: wire upshmem_mfill_zeropage_pte
userfaultfd: reportUFFDIO_ZEROPAGEas available for shmem VMAs
userfaultfd: selftest: enable testing ofUFFDIO_ZEROPAGEfor shmemfs/userfaultfd.c | 10 +- include/linux/shmem_fs.h | 6 ++ mm/shmem.c | 167 +++++++++++++++++-------------- mm/userfaultfd.c | 48 ++++++--- tools/testing/selftests/vm/userfaultfd.c | 2 +- 5 files changed, 136 insertions(+), 97 deletions(-)
Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.