Skip to contentSkip to conversation

mm, hugepages: add mremap() support for hugepage backed vma

PATCH New Mina Almasry 1 participant 2 messages Oct 14, 2021

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH v8 1/2 mm, hugepages: add mremap() support for hugepage backed vma

Support mremap() for hugepage backed vma segment by simply repositioning page table entries. The page table entries are repositioned to the new virtual address on mremap().

Hugetlb mremap() support is of course generic; my motivating use case is a library (hugepage_text), which reloads the ELF text of executables in hugepages. This significantly increases the execution performance of said executables.

Restricts the mremap operation on hugepages to up to the size of the original mapping as the underlying hugetlb reservation is not yet capable of handling remapping to a larger size.

During the mremap() operation we detect pmd_share'd mappings and we unshare those during the mremap(). On access and fault the sharing is established again.

Cc Mike Kravetz <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Ken Chen <[email protected]>
Cc Chris Kennelly <[email protected]>
Cc Michal Hocko <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Kirill Shutemov <[email protected]>

---

Changes in v6:
- Converted hugetlb_vma_shareable() to static only (fixes build warning).
- Removed huge_pmd_shared(). It's not needed since I removed the BUG_ON(huge_pmd_shared) in v3.
- Removed *...* format for emphasis.
- Fixed mremap behavior to hugepage-align lengths but return error if addresses are not hugepage aligned.

Changes in v5:
- Remove hugetlb_vma_shareable and huge_pmd_shared dummy definitions for !CONFIG_HUGETLB_PAGE config, since they are not used and were causing added warning and build errors.

Changes in v4:
- Added addr, new_addr, old_len, and new_len hugepage alignment.

Changes in v3:
- Addressed review comments from Mike.
- Separated tests into their own patch.

Changes in v2:
- Re-wrote comment around clear_vma_resv_huge_pages() to make it clear that the resv_map has been moved to the new VMA and why we need to clear it from the current VMA.
- We detect huge_pmd_shared() pte's and unshare those rather than bug on hugetlb_vma_shareable().
- This case now returns EFAULT:
if (!vma || vma->vm_start > addr)

	 goto out;

- Added kselftests for mremap() support.

---
 include/linux/hugetlb.h |  19 +++++++
 mm/hugetlb.c            | 111 +++++++++++++++++++++++++++++++++++++---
 mm/mremap.c             |  36 +++++++++++--
 3 files changed, 157 insertions(+), 9 deletions(-)
include/linux/hugetlb.h +19 -0
index ebaba02706c87..44c2ab0dfa591 100644
@@ -124,6 +124,7 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 void hugepage_put_subpool(struct hugepage_subpool *spool);
 
 void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
+void clear_vma_resv_huge_pages(struct vm_area_struct *vma);
 int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *);
 int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *,
 		loff_t *);
@@ -132,6 +133,10 @@ int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *,
 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, void *, size_t *,
 		loff_t *);
 
+int move_hugetlb_page_tables(struct vm_area_struct *vma,
+			     struct vm_area_struct *new_vma,
+			     unsigned long old_addr, unsigned long new_addr,
+			     unsigned long len);
 int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
 long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
 			 struct page **, struct vm_area_struct **,
@@ -215,6 +220,10 @@ static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 {
 }
 
+static inline void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
+{
+}
+
 static inline unsigned long hugetlb_total_pages(void)
 {
 	return 0;
@@ -262,6 +271,16 @@ static inline int copy_hugetlb_page_range(struct mm_struct *dst,
 	return 0;
 }
 
+static inline int move_hugetlb_page_tables(struct vm_area_struct *vma,
+					   struct vm_area_struct *new_vma,
+					   unsigned long old_addr,
+					   unsigned long new_addr,
+					   unsigned long len)
+{
+	BUG();
+	return 0;
+}
+
 static inline void hugetlb_report_meminfo(struct seq_file *m)
 {
 }
mm/hugetlb.c +105 -6

129 lines not on this page · raw message

mm/mremap.c +33 -3
index c0b6c41b7b78f..002eec83e91e5 100644
@@ -489,6 +489,10 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
 	old_end = old_addr + len;
 	flush_cache_range(vma, old_addr, old_end);
 
+	if (is_vm_hugetlb_page(vma))
+		return move_hugetlb_page_tables(vma, new_vma, old_addr,
+						new_addr, len);
+
 	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
 				old_addr, old_end);
 	mmu_notifier_invalidate_range_start(&range);
@@ -646,6 +650,10 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 		mremap_userfaultfd_prep(new_vma, uf);
 	}
 
+	if (is_vm_hugetlb_page(vma)) {
+		clear_vma_resv_huge_pages(vma);
+	}
+
 	/* Conceal VM_ACCOUNT so old reservation is not undone */
 	if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP)) {
 		vma->vm_flags &= ~VM_ACCOUNT;
@@ -739,9 +747,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
 			(vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
 		return ERR_PTR(-EINVAL);
 
-	if (is_vm_hugetlb_page(vma))
-		return ERR_PTR(-EINVAL);
-
 	/* We can't remap across vm area boundaries */
 	if (old_len > vma->vm_end - addr)
 		return ERR_PTR(-EFAULT);
@@ -937,6 +942,31 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 
 	if (mmap_write_lock_killable(current->mm))
 		return -EINTR;
+	vma = find_vma(mm, addr);
+	if (!vma || vma->vm_start > addr) {
+		ret = EFAULT;
+		goto out;
+	}
+
+	if (is_vm_hugetlb_page(vma)) {
+		struct hstate *h __maybe_unused = hstate_vma(vma);
+
+		old_len = ALIGN(old_len, huge_page_size(h));
+		new_len = ALIGN(new_len, huge_page_size(h));
+
+		/* addrs must be huge page aligned */
+		if (addr & ~huge_page_mask(h))
+			goto out;
+		if (new_addr & ~huge_page_mask(h))
+			goto out;
+
+		/*
+		 * Don't allow remap expansion, because the underlying hugetlb
+		 * reservation is not yet capable to handle split reservation.
+		 */
+		if (new_len > old_len)
+			goto out;
+	}
 
 	if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) {
 		ret = mremap_to(addr, old_len, new_addr, new_len,

--
2.33.0.1079.g6e70778dc9-goog

#1 lore raw

PATCH v8 2/2 mm, hugepages: Add hugetlb vma mremap() test

Cc Mike Kravetz <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Ken Chen <[email protected]>
Cc Chris Kennelly <[email protected]>
Cc Michal Hocko <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Kirill Shutemov <[email protected]>

---

Changes in v8:
- Integrated hugepage-mremap into run_vmtests.sh
- Added Ack from Mike Kravetz

Changes in v6:
- Reverted change in v4: test case now passes huge page aligned addrs to mmap/mremap.

Changes in v4:
- Added comments to make test output clearer.
- Modified test case slightly to test hugepage alignment of new_addr.

---
 tools/testing/selftests/vm/.gitignore        |   1 +
 tools/testing/selftests/vm/Makefile          |   1 +
 tools/testing/selftests/vm/hugepage-mremap.c | 161 +++++++++++++++++++
 tools/testing/selftests/vm/run_vmtests.sh    |  11 ++
 4 files changed, 174 insertions(+)
 create mode 100644 tools/testing/selftests/vm/hugepage-mremap.c
tools/testing/selftests/vm/.gitignore +1 -0
index b02eac613fdda..2e7e86e852828 100644
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 hugepage-mmap
+hugepage-mremap
 hugepage-shm
 khugepaged
 map_hugetlb
tools/testing/selftests/vm/Makefile +1 -0
index d9605bd10f2de..1607322a112c9 100644
@@ -29,6 +29,7 @@ TEST_GEN_FILES = compaction_test
 TEST_GEN_FILES += gup_test
 TEST_GEN_FILES += hmm-tests
 TEST_GEN_FILES += hugepage-mmap
+TEST_GEN_FILES += hugepage-mremap
 TEST_GEN_FILES += hugepage-shm
 TEST_GEN_FILES += khugepaged
 TEST_GEN_FILES += madv_populate
tools/testing/selftests/vm/hugepage-mremap.c new file +161 -0

161 lines not on this page · raw message

tools/testing/selftests/vm/run_vmtests.sh +11 -0
index 45e803af7c775..a24d30af30949 100755
@@ -108,6 +108,17 @@ else
 	echo "[PASS]"
 fi
 
+echo "-----------------------"
+echo "running hugepage-mremap"
+echo "-----------------------"
+./hugepage-mremap
+if [ $? -ne 0 ]; then
+	echo "[FAIL]"
+	exitcode=1
+else
+	echo "[PASS]"
+fi
+
 echo "NOTE: The above hugetlb tests provide minimal coverage.  Use"
 echo "      https://github.com/libhugetlbfs/libhugetlbfs.git for"
 echo "      hugetlb regression testing."

--
2.33.0.1079.g6e70778dc9-goog