mm, hugepages: add mremap() support for hugepage backed vma
Subsystems (4)
PATCH v6 1/2 mm, hugepages: add mremap() support for hugepage backed vma
Support mremap() for hugepage backed vma segment by simply repositioning page table entries. The page table entries are repositioned to the new virtual address on mremap().
Hugetlb mremap() support is of course generic; my motivating use case is a library (hugepage_text), which reloads the ELF text of executables in hugepages. This significantly increases the execution performance of said executables.
Restricts the mremap operation on hugepages to up to the size of the original mapping as the underlying hugetlb reservation is not yet capable of handling remapping to a larger size.
During the mremap() operation we detect pmd_share'd mappings and we unshare those during the mremap(). On access and fault the sharing is established again.
---
Changes in v6:
- Converted hugetlb_vma_shareable() to static only (fixes build warning).
- Removed huge_pmd_shared(). It's not needed since I removed the BUG_ON(huge_pmd_shared) in v3.
- Removed *...* format for emphasis.
- Fixed mremap behavior to hugepage-align lengths but return error if addresses are not hugepage aligned.
Changes in v5:
- Remove hugetlb_vma_shareable and huge_pmd_shared dummy definitions for !CONFIG_HUGETLB_PAGE config, since they are not used and were causing added warning and build errors.
Changes in v4:
- Added addr, new_addr, old_len, and new_len hugepage alignment.
Changes in v3:
- Addressed review comments from Mike.
- Separated tests into their own patch.
Changes in v2:
- Re-wrote comment around clear_vma_resv_huge_pages() to make it clear that the resv_map has been moved to the new VMA and why we need to clear it from the current VMA.
- We detect huge_pmd_shared() pte's and unshare those rather than bug on hugetlb_vma_shareable().
- This case now returns EFAULT:
if (!vma || vma->vm_start > addr)
goto out;
- Added kselftests for mremap() support.
--- include/linux/hugetlb.h | 15 ++++++ mm/hugetlb.c | 116 +++++++++++++++++++++++++++++++++++++--- mm/mremap.c | 36 +++++++++++-- 3 files changed, 156 insertions(+), 11 deletions(-)
include/linux/hugetlb.h +15 -0
@@ -124,6 +124,7 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,void hugepage_put_subpool (struct hugepage_subpool *spool);void reset_vma_resv_huge_pages (struct vm_area_struct *vma); +void clear_vma_resv_huge_pages (struct vm_area_struct *vma);int hugetlb_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int hugetlb_overcommit_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);
@@ -132,6 +133,10 @@ int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,int hugetlb_mempolicy_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); +int move_hugetlb_page_tables (struct vm_area_struct *vma, +struct vm_area_struct *new_vma, +unsigned long old_addr,unsigned long new_addr, +unsigned long len);int copy_hugetlb_page_range (struct mm_struct *,struct mm_struct *,struct vm_area_struct *); long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, struct page **, struct vm_area_struct **,
@@ -215,6 +220,10 @@ static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) { } +static inline void clear_vma_resv_huge_pages (struct vm_area_struct *vma) +{ +} +static inline unsigned long hugetlb_total_pages (void ) {return 0 ;
@@ -262,6 +271,12 @@ static inline int copy_hugetlb_page_range(struct mm_struct *dst,return 0 ; } +#define move_hugetlb_page_tables (vma, new_vma, old_addr, new_addr, len) \ + ({ \ + BUG(); \ + 0; \ + }) +static inline void hugetlb_report_meminfo (struct seq_file *m) { }
mm/hugetlb.c +108 -8
146 lines not on this page · raw message
mm/mremap.c +33 -3
@@ -489,6 +489,10 @@ unsigned long move_page_tables(struct vm_area_struct *vma, old_end = old_addr + len;flush_cache_range (vma, old_addr, old_end); +if (is_vm_hugetlb_page (vma)) +return move_hugetlb_page_tables (vma, new_vma, old_addr, + new_addr, len); +mmu_notifier_range_init (&range,MMU_NOTIFY_UNMAP ,0 , vma, vma->vm_mm, old_addr, old_end);mmu_notifier_invalidate_range_start (&range);
@@ -646,6 +650,10 @@ static unsigned long move_vma(struct vm_area_struct *vma,mremap_userfaultfd_prep (new_vma, uf); } +if (is_vm_hugetlb_page (vma)) { +clear_vma_resv_huge_pages (vma); + } +/* Conceal VM_ACCOUNT so old reservation is not undone */ if (vm_flags &VM_ACCOUNT && !(flags &MREMAP_DONTUNMAP )) { vma->vm_flags &= ~VM_ACCOUNT ;
@@ -739,9 +747,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))return ERR_PTR (-EINVAL ); -if (is_vm_hugetlb_page (vma)) -return ERR_PTR (-EINVAL ); -/* We can't remap across vm area boundaries */ if (old_len > vma->vm_end - addr)return ERR_PTR (-EFAULT );
@@ -937,6 +942,31 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,if (mmap_write_lock_killable (current->mm))return -EINTR ; + vma =find_vma (mm, addr); +if (!vma || vma->vm_start > addr) { + ret =EFAULT ; + goto out; + } + +if (is_vm_hugetlb_page (vma)) { +struct hstate *h__maybe_unused =hstate_vma (vma); + + old_len =ALIGN (old_len,huge_page_size (h)); + new_len =ALIGN (new_len,huge_page_size (h)); + +/* addrs must be huge page aligned */ +if (addr & ~huge_page_mask (h)) + goto out; +if (new_addr & ~huge_page_mask (h)) + goto out; + +/* +* Don't allow remap expansion, because the underlying hugetlb +* reservation is not yet capable to handle split reservation. +*/ +if (new_len > old_len) + goto out; + }if (flags & (MREMAP_FIXED |MREMAP_DONTUNMAP )) { ret = mremap_to(addr, old_len, new_addr, new_len,
--
2.33.0.882.g93a45727a2-goog
PATCH v6 2/2 mm, hugepages: Add hugetlb vma mremap() test
---
Changes in v6:
- Reverted change in v4: test case now passes huge page aligned addrs to mmap/mremap.
Changes in v4:
- Added comments to make test output clearer.
- Modified test case slightly to test hugepage alignment of new_addr.
--- tools/testing/selftests/vm/.gitignore | 1 + tools/testing/selftests/vm/Makefile | 1 + tools/testing/selftests/vm/hugepage-mremap.c | 165 +++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 tools/testing/selftests/vm/hugepage-mremap.c
tools/testing/selftests/vm/.gitignore +1 -0
@@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only hugepage-mmap +hugepage-mremap hugepage-shm khugepaged map_hugetlb
tools/testing/selftests/vm/Makefile +1 -0
@@ -29,6 +29,7 @@ TEST_GEN_FILES = compaction_test TEST_GEN_FILES += gup_test TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugepage-mmap +TEST_GEN_FILES += hugepage-mremap TEST_GEN_FILES += hugepage-shm TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate
tools/testing/selftests/vm/hugepage-mremap.c new file +165 -0
165 lines not on this page · raw message
--
2.33.0.882.g93a45727a2-goog
Quoted 13 lines · Mina Almasry
Support
mremap()for hugepage backed vma segment by simply repositioning page table entries. The page table entries are repositioned to the new virtual address onmremap().Hugetlb
mremap()support is of course generic; my motivating use case is a library (hugepage_text), which reloads the ELF text of executables in hugepages. This significantly increases the execution performance of said executables.Restricts the mremap operation on hugepages to up to the size of the original mapping as the underlying hugetlb reservation is not yet capable of handling remapping to a larger size.
Quoted 5 lines
During the
mremap()operation we detect pmd_share'd mappings and we unshare those during themremap(). On access and fault the sharing is established again.Signed-off-by Mina Almasry <[email protected]>
Thanks!
Just some minor nits below. If you agree with the suggestions and make the changes, you can add:
Quoted 35 lines
include/linux/hugetlb.h +9
-0@@ -124,6 +124,7 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,void hugepage_put_subpool (struct hugepage_subpool *spool);void reset_vma_resv_huge_pages (struct vm_area_struct *vma); +void clear_vma_resv_huge_pages (struct vm_area_struct *vma);int hugetlb_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int hugetlb_overcommit_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);@@ -132,6 +133,10 @@ int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,int hugetlb_mempolicy_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); +int move_hugetlb_page_tables (struct vm_area_struct *vma, +struct vm_area_struct *new_vma, +unsigned long old_addr,unsigned long new_addr, +unsigned long len);int copy_hugetlb_page_range (struct mm_struct *,struct mm_struct *,struct vm_area_struct *); long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, struct page **, struct vm_area_struct **,@@ -215,6 +220,10 @@ static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) { } +static inline void clear_vma_resv_huge_pages (struct vm_area_struct *vma) +{ +} +static inline unsigned long hugetlb_total_pages (void ) {return 0 ;
include/linux/hugetlb.h static inline int copy_hugetlb_page_range(struct mm_struct *dst, · line 279
Quoted 9 lines
include/linux/hugetlb.h +6
-0@@ -262,6 +271,12 @@ static inline int copy_hugetlb_page_range(struct mm_struct *dst,return 0 ; } +#define move_hugetlb_page_tables (vma, new_vma, old_addr, new_addr, len) \ + ({ \ + BUG(); \ + 0; \ + }) +
Any reason why you did not make this a static inline? Trying to save code in the !CONFIG_HUGETLB case? macros seem to end up causing more issues down the line. I would suggest making this a static inline unless there is a good reason to keep as a macro.
Quoted 7 lines
include/linux/hugetlb.h +0
-0static inline void hugetlb_report_meminfo (struct seq_file *m) { }mm/hugetlb.c +0
-0
...
Quoted 20 lines
mm/hugetlb.c +20
-0+int move_hugetlb_page_tables (struct vm_area_struct *vma, +struct vm_area_struct *new_vma, +unsigned long old_addr,unsigned long new_addr, +unsigned long len) +{ +struct hstate *h =hstate_vma (vma); +struct address_space *mapping = vma->vm_file->f_mapping; +unsigned long sz =huge_page_size (h); +struct mm_struct *mm = vma->vm_mm; +unsigned long old_end = old_addr + len; +unsigned long old_addr_copy; +pte_t *src_pte, *dst_pte; +struct mmu_notifier_range range; + +mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, mm, old_addr, + old_end); +adjust_range_if_pmd_sharing_possible (vma, &range.start, &range.end); +mmu_notifier_invalidate_range_start (&range); +/* Prevent race with file truncation */ +i_mmap_lock_write (mapping);
mm/hugetlb.c int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · line 4888
Quoted 10 lines
mm/hugetlb.c +10
-0+for (; old_addr < old_end; old_addr += sz, new_addr += sz) { + src_pte =huge_pte_offset (mm, old_addr, sz); +if (!src_pte) +continue ; +if (huge_pte_none (huge_ptep_get (src_pte))) +continue ; + +/* old_addr arg to huge_pmd_unshare() is a pointer and so the +* arg may be modified. Pass a copy instead to preserve the +* value in old_arg.
value in old_addr.
Quoted 33 lines
mm/hugetlb.c +21
-1+*/ + old_addr_copy = old_addr; + +if (huge_pmd_unshare (mm, vma, &old_addr_copy, src_pte)) +continue ; + + dst_pte =huge_pte_alloc (mm, new_vma, new_addr, sz); +if (!dst_pte) +break ; + +move_huge_pte (vma, old_addr, new_addr, src_pte); + } +i_mmap_unlock_write (mapping); +flush_tlb_range (vma, old_end - len, old_end); +mmu_notifier_invalidate_range_end (&range); + +return len + old_addr - old_end; +} +static void __unmap_hugepage_range (struct mmu_gather *tlb,struct vm_area_struct *vma,unsigned long start,unsigned long end,struct page *ref_page)@@ -6280,7 +6385,8 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma,return saddr; } -static bool vma_shareable (struct vm_area_struct *vma,unsigned long addr) +static bool hugetlb_vma_shareable (struct vm_area_struct *vma, +unsigned long addr) {unsigned long base = addr &PUD_MASK ;unsigned long end = base +PUD_SIZE ;
mm/hugetlb.c bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) · line 6411
Quoted 8 lines
mm/hugetlb.c +1
-1@@ -6299,7 +6405,7 @@ bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)if (uffd_disable_huge_pmd_share (vma))return false;#endif -return vma_shareable (vma, addr); +return hugetlb_vma_shareable (vma, addr); }/*
In an earlier version of the patch, vma_shareable was renamed hugetlb_vma_shareable because it was going to be used outside hugetlb.c. Therefore the hugetlb_* name would provide some context. That is no longer the case. So, there really is no need to change the name. In fact, none of the remap code even calls this routine. Sugggest you drop the name change.
Signature · 1 line
Mike Kravetz
Quoted 19 lines · Mike Kravetz
Quoted 17 lines · Mina Almasry
Support
mremap()for hugepage backed vma segment by simply repositioning page table entries. The page table entries are repositioned to the new virtual address onmremap().Hugetlb
mremap()support is of course generic; my motivating use case is a library (hugepage_text), which reloads the ELF text of executables in hugepages. This significantly increases the execution performance of said executables.Restricts the mremap operation on hugepages to up to the size of the original mapping as the underlying hugetlb reservation is not yet capable of handling remapping to a larger size.
During the
mremap()operation we detect pmd_share'd mappings and we unshare those during themremap(). On access and fault the sharing is established again.
Quoted 9 lines
Quoted 1 line
Signed-off-by Mina Almasry <[email protected]>Thanks!
Just some minor nits below. If you agree with the suggestions and make the changes, you can add:
Reviewed-by Mike Kravetz <[email protected]>
Thank you as always for your review and patience. I've applied the changes and uploaded v7 with the reviewed-by. Any chance I can get an Ack at least on the associated test patch? Or should I send these somewhere else for review?
Quoted 38 lines
Quoted 38 lines
include/linux/hugetlb.h +9
-0@@ -124,6 +124,7 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,void hugepage_put_subpool (struct hugepage_subpool *spool);void reset_vma_resv_huge_pages (struct vm_area_struct *vma); +void clear_vma_resv_huge_pages (struct vm_area_struct *vma);int hugetlb_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int hugetlb_overcommit_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);@@ -132,6 +133,10 @@ int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,int hugetlb_mempolicy_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); +int move_hugetlb_page_tables (struct vm_area_struct *vma, +struct vm_area_struct *new_vma, +unsigned long old_addr,unsigned long new_addr, +unsigned long len);int copy_hugetlb_page_range (struct mm_struct *,struct mm_struct *,struct vm_area_struct *); long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, struct page **, struct vm_area_struct **,@@ -215,6 +220,10 @@ static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) { } +static inline void clear_vma_resv_huge_pages (struct vm_area_struct *vma) +{ +} +static inline unsigned long hugetlb_total_pages (void ) {return 0 ;@@ -262,6 +271,12 @@ static inline int copy_hugetlb_page_range(struct mm_struct *dst,return 0 ; }
Quoted 12 lines
include/linux/hugetlb.h static inline int copy_hugetlb_page_range(struct mm_struct *dst, · line 279
Quoted 6 lines
include/linux/hugetlb.h +6
-0+#define move_hugetlb_page_tables (vma, new_vma, old_addr, new_addr, len) \ + ({ \ + BUG(); \ + 0; \ + }) +Any reason why you did not make this a static inline? Trying to save code in the !CONFIG_HUGETLB case? macros seem to end up causing more issues down the line. I would suggest making this a static inline unless there is a good reason to keep as a macro.
No good reason. I converted to static inline.
Quoted 32 lines
Quoted 7 lines
include/linux/hugetlb.h +0
-0static inline void hugetlb_report_meminfo (struct seq_file *m) { }mm/hugetlb.c +0
-0...
Quoted 24 lines
mm/hugetlb.c +24
-0+int move_hugetlb_page_tables (struct vm_area_struct *vma, +struct vm_area_struct *new_vma, +unsigned long old_addr,unsigned long new_addr, +unsigned long len) +{ +struct hstate *h =hstate_vma (vma); +struct address_space *mapping = vma->vm_file->f_mapping; +unsigned long sz =huge_page_size (h); +struct mm_struct *mm = vma->vm_mm; +unsigned long old_end = old_addr + len; +unsigned long old_addr_copy; +pte_t *src_pte, *dst_pte; +struct mmu_notifier_range range; + +mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, mm, old_addr, + old_end); +adjust_range_if_pmd_sharing_possible (vma, &range.start, &range.end); +mmu_notifier_invalidate_range_start (&range); +/* Prevent race with file truncation */ +i_mmap_lock_write (mapping); +for (; old_addr < old_end; old_addr += sz, new_addr += sz) { + src_pte =huge_pte_offset (mm, old_addr, sz); +if (!src_pte) +continue ;
Quoted 9 lines
mm/hugetlb.c int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, · line 4888
Quoted 6 lines
mm/hugetlb.c +6
-0+if (huge_pte_none (huge_ptep_get (src_pte))) +continue ; + +/* old_addr arg to huge_pmd_unshare() is a pointer and so the +* arg may be modified. Pass a copy instead to preserve the +* value in old_arg. value in
old_addr.
Fixed.
Quoted 40 lines
Quoted 40 lines
mm/hugetlb.c +22
-2+*/ + old_addr_copy = old_addr; + +if (huge_pmd_unshare (mm, vma, &old_addr_copy, src_pte)) +continue ; + + dst_pte =huge_pte_alloc (mm, new_vma, new_addr, sz); +if (!dst_pte) +break ; + +move_huge_pte (vma, old_addr, new_addr, src_pte); + } +i_mmap_unlock_write (mapping); +flush_tlb_range (vma, old_end - len, old_end); +mmu_notifier_invalidate_range_end (&range); + +return len + old_addr - old_end; +} +static void __unmap_hugepage_range (struct mmu_gather *tlb,struct vm_area_struct *vma,unsigned long start,unsigned long end,struct page *ref_page)@@ -6280,7 +6385,8 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma,return saddr; } -static bool vma_shareable (struct vm_area_struct *vma,unsigned long addr) +static bool hugetlb_vma_shareable (struct vm_area_struct *vma, +unsigned long addr) {unsigned long base = addr &PUD_MASK ;unsigned long end = base +PUD_SIZE ;@@ -6299,7 +6405,7 @@ bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)if (uffd_disable_huge_pmd_share (vma))return false;#endif -return vma_shareable (vma, addr); +return hugetlb_vma_shareable (vma, addr); }
Quoted 8 lines
mm/hugetlb.c bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) · line 6411
Quoted 1 line
mm/hugetlb.c +0
-0/* In an earlier version of the patch,
vma_shareablewas renamedhugetlb_vma_shareablebecause it was going to be used outsidehugetlb.c. Therefore the hugetlb_* name would provide some context. That is no longer the case. So, there really is no need to change the name. In fact, none of the remap code even calls this routine. Sugggest you drop the name change.
Fixed.
Quoted 2 lines
--
Mike Kravetz