mm: add mremap flag for preserving the old mapping
Subsystems (5)
PATCH v4 1/1 mm: add mremap flag for preserving the old mapping
This introduces the MREMAP_RETAIN flag for preserving the source mapping when MREMAP_MAYMOVE moves the pages to a new destination. Accesses to the source mapping will fault and map in fresh zeroed pages.
It is currently limited to writable MAP_PRIVATE|MAP_ANONYMOUS mappings and will return EFAULT when used on anything else. This covers the intended use case in general purpose allocators.
For consistency, the old_len >= new_len case could decommit the pages instead of unmapping. However, userspace can accomplish the same thing via madvise and the flag is coherent without the additional complexity.
Motivation:
TCMalloc and jemalloc avoid releasing virtual memory in order to reduce virtual memory fragmentation. A call to munmap or mremap would leave a hole in the address space. Instead, unused pages are lazily returned to the operating system via MADV_DONTNEED.
Since mremap cannot be used to elide copies, TCMalloc and jemalloc end up being significantly slower for patterns like repeated vector / hash table reallocations. Consider the typical vector building pattern:
#include <string.h>
#include <stdlib.h>
int main(void) {
for (size_t i = 0; i < 100; i++) {
void *ptr = NULL;
size_t old_size = 0;
for (size_t size = 4; size < (1 << 30); size *= 2) {
ptr = realloc(ptr, size);
if (!ptr) return 1;
memset(ptr + old_size, 0xff, size - old_size);
old_size = size;
}
free(ptr);
}
}Transparent huge pages disabled:
glibc (baseline, uses mremap already): 15.051s
jemalloc without MREMAP_RETAIN: 38.540s
jemalloc with MREMAP_RETAIN: 15.086s
Transparent huge pages enabled:
glibc (baseline, uses mremap already): 8.464s
jemalloc without MREMAP_RETAIN: 18.230s
jemalloc with MREMAP_RETAIN: 6.696s
In practice, in-place growth never occurs for huge allocations because the heap grows in the downwards direction for all 3 allocators. TCMalloc and jemalloc pay for enormous copies while glibc is only spending time writing new elements to the vector. Even if it was grown in the other direction, real-world applications would end up blocking in-place growth with new allocations.
The allocators could attempt to map the source location again after an mremap call, but there is no guarantee of success in a multi-threaded program and fragmentating memory over time is considered unacceptable.
--- include/linux/huge_mm.h | 2 +- include/linux/mm.h | 6 ++++++ include/uapi/linux/mman.h | 1 + mm/huge_memory.c | 4 ++-- mm/memory.c | 2 +- mm/mmap.c | 12 +++++++++++ mm/mremap.c | 52 +++++++++++++++++++++++++++++++---------------- 7 files changed, 57 insertions(+), 22 deletions(-)
include/linux/huge_mm.h +1 -1
@@ -143,7 +143,7 @@ static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long end, long adjust_next) { -if (!vma->anon_vma || vma->vm_ops) +if (!vma->anon_vma || (vma->vm_ops && !vma->vm_ops->allow_huge_pages))return ;__vma_adjust_trans_huge (vma, start, end, adjust_next); }
include/linux/mm.h +6 -0
@@ -273,6 +273,12 @@ struct vm_operations_struct {/* called by sys_remap_file_pages() to populate non-linear mapping */ int (*remap_pages)(struct vm_area_struct *vma,unsigned long addr,unsigned long size,pgoff_t pgoff); + +/* Check if the mapping may be duplicated by MREMAP_RETAIN */ +bool (*may_duplicate)(struct vm_area_struct *vma); + +/* if there is no vm_ops table, this is considered true */ +bool allow_huge_pages; };struct mmu_gather ;
include/uapi/linux/mman.h +1 -0
@@ -5,6 +5,7 @@#define MREMAP_MAYMOVE 1#define MREMAP_FIXED 2 +#define MREMAP_RETAIN 4#define OVERCOMMIT_GUESS 0#define OVERCOMMIT_ALWAYS 1
mm/huge_memory.c +2 -2
@@ -2077,7 +2077,7 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma)* page fault if needed. */ return 0 ; -if (vma->vm_ops) +if ((vma->vm_ops && !vma->vm_ops->allow_huge_pages))/* khugepaged not yet working on file or special mappings */ return 0 ;VM_BUG_ON (vma->vm_flags &VM_NO_THP );
@@ -2405,7 +2405,7 @@ static bool hugepage_vma_check(struct vm_area_struct *vma) (vma->vm_flags & VM_NOHUGEPAGE))return false; -if (!vma->anon_vma || vma->vm_ops) +if (!vma->anon_vma || (vma->vm_ops && !vma->vm_ops->allow_huge_pages))return false;if (is_vma_temporary_stack (vma))return false;
mm/memory.c +1 -1
@@ -3275,7 +3275,7 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,return VM_FAULT_OOM ;if (pmd_none (*pmd) &&transparent_hugepage_enabled (vma)) {int ret =VM_FAULT_FALLBACK ; -if (!vma->vm_ops) +if (!vma->vm_ops || vma->vm_ops->allow_huge_pages) ret =do_huge_pmd_anonymous_page (mm, vma, address, pmd, flags);if (!(ret &VM_FAULT_FALLBACK ))
mm/mmap.c +12 -0
@@ -1500,6 +1500,16 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)return (vm_flags & (VM_NORESERVE |VM_SHARED |VM_WRITE )) ==VM_WRITE ; } +static bool anon_may_duplicate (struct vm_area_struct *vma) +{ +return vma->vm_flags &VM_WRITE && !(vma->vm_flags &VM_SHARED ); +} + +static const struct vm_operations_struct anon_vmops = { + .may_duplicate = anon_may_duplicate, + .allow_huge_pages = true +}; +unsigned long mmap_region (struct file *file,unsigned long addr,unsigned long len,vm_flags_t vm_flags,unsigned long pgoff) {
@@ -1569,6 +1579,8 @@ munmap_back: vma->vm_flags = vm_flags; vma->vm_page_prot =vm_get_page_prot (vm_flags); vma->vm_pgoff = pgoff; +if (!file) + vma->vm_ops = &anon_vmops;INIT_LIST_HEAD (&vma->anon_vma_chain);if (file) {
mm/mremap.c +34 -18
130 lines not on this page · raw message
Signature · 7 lines
2.1.2 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>