mm-mremap-refactor-mremap-system-call-implementation.patch removed from -mm tree
The quilt patch titled
Subject: mm/mremap: refactor mremap() system call implementation
has been removed from the -mm tree. Its filename was
mm-mremap-refactor-mremap-system-call-implementation.patch
This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Lorenzo Stoakes <[email protected]>
Subject: mm/mremap: refactor mremap() system call implementation
Date: Mon, 10 Mar 2025 20:50:35 +0000
Place checks into a separate function so the mremap() system call is less egregiously long, remove unnecessary mremap_to() offset_in_page() check and just check that earlier so we keep all such basic checks together.
Separate out the VMA in-place expansion, hugetlb and expand/move logic into separate, readable functions.
De-duplicate code where possible, add comments and ensure that all error handling explicitly specifies the error at the point of it occurring rather than setting a prefixed error value and implicitly setting (which is bug prone).
This lays the groundwork for subsequent patches further simplifying and extending the mremap() implementation.
---
mm/mremap.c | 405 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 251 insertions(+), 154 deletions(-)
mm/mremap.c renamed from mm/mremap.c~mm-mremap-refactor-mremap-system-call-implementation +251 -154
@@ -942,33 +942,14 @@ static unsigned long mremap_to(unsignedunsigned long ret;unsigned long map_flags =0 ; -if (offset_in_page (new_addr)) -return -EINVAL ; - +/* Is the new length or address silly? */ if (new_len >TASK_SIZE || new_addr >TASK_SIZE - new_len)return -EINVAL ; -/* Ensure the old/new locations do not overlap */ +/* Ensure the old/new locations do not overlap. */ if (addr + old_len > new_addr && new_addr + new_len > addr)return -EINVAL ; -/* -* move_vma() need us to stay 4 maps below the threshold, otherwise -* it will bail out at the very beginning. -* That is a problem if we have already unmaped the regions here -* (new_addr, and old_addr), because userspace will not know the -* state of the vma's after it gets -ENOMEM. -* So, to avoid such scenario we can pre-compute if the whole -* operation has high chances to success map-wise. -* Worst-scenario case is when both vma's (new_addr and old_addr) get -* split in 3 before unmapping it. -* That means 2 more maps (1 for each) to the ones we already hold. -* Check whether current map count plus 2 still leads us to 4 maps below -* the threshold, otherwise return -ENOMEM here to be more safe. -*/ -if ((mm->map_count +2 ) >= sysctl_max_map_count -3 ) -return -ENOMEM ; -if (flags &MREMAP_FIXED ) {/* * In mremap_to().
@@ -1035,6 +1016,218 @@ static int vma_expandable(struct vm_areareturn 1 ; } +/* Do the mremap() flags require that the new_addr parameter be specified? */ +static bool implies_new_addr (unsigned long flags) +{ +return flags & (MREMAP_FIXED |MREMAP_DONTUNMAP ); +} + +/* +* Are the parameters passed to mremap() valid? If so return 0, otherwise return +* error. +*/ +static unsigned long check_mremap_params (unsigned long addr, +unsigned long flags, +unsigned long old_len, +unsigned long new_len, +unsigned long new_addr) +{ +/* Ensure no unexpected flag values. */ +if (flags & ~(MREMAP_FIXED |MREMAP_MAYMOVE |MREMAP_DONTUNMAP )) +return -EINVAL ; + +/* Start address must be page-aligned. */ +if (offset_in_page (addr)) +return -EINVAL ; + +/* +* We allow a zero old-len as a special case +* for DOS-emu "duplicate shm area" thing. But +* a zero new-len is nonsensical. +*/ +if (!PAGE_ALIGN (new_len)) +return -EINVAL ; + +/* Remainder of checks are for cases with specific new_addr. */ +if (!implies_new_addr (flags)) +return 0 ; + +/* The new address must be page-aligned. */ +if (offset_in_page (new_addr)) +return -EINVAL ; + +/* A fixed address implies a move. */ +if (!(flags &MREMAP_MAYMOVE )) +return -EINVAL ; + +/* MREMAP_DONTUNMAP does not allow resizing in the process. */ +if (flags &MREMAP_DONTUNMAP && old_len != new_len) +return -EINVAL ; + +/* +* move_vma() need us to stay 4 maps below the threshold, otherwise +* it will bail out at the very beginning. +* That is a problem if we have already unmaped the regions here +* (new_addr, and old_addr), because userspace will not know the +* state of the vma's after it gets -ENOMEM. +* So, to avoid such scenario we can pre-compute if the whole +* operation has high chances to success map-wise. +* Worst-scenario case is when both vma's (new_addr and old_addr) get +* split in 3 before unmapping it. +* That means 2 more maps (1 for each) to the ones we already hold. +* Check whether current map count plus 2 still leads us to 4 maps below +* the threshold, otherwise return -ENOMEM here to be more safe. +*/ +if ((current->mm->map_count +2 ) >= sysctl_max_map_count -3 ) +return -ENOMEM ; + +return 0 ; +} + +/* +* We know we can expand the VMA in-place by delta pages, so do so. +* +* If we discover the VMA is locked, update mm_struct statistics accordingly and +* indicate so to the caller. +*/ +static unsigned long expand_vma_inplace (struct vm_area_struct *vma, +unsigned long delta,bool *locked) +{ +struct mm_struct *mm = current->mm; +long pages = delta >>PAGE_SHIFT ; +VMA_ITERATOR (vmi, mm, vma->vm_end); +long charged =0 ; + +if (vma->vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, pages)) +return -ENOMEM ; + + charged = pages; + } + +/* +* Function vma_merge_extend() is called on the +* extension we are adding to the already existing vma, +* vma_merge_extend() will merge this extension with the +* already existing vma (expand operation itself) and +* possibly also with the next vma if it becomes +* adjacent to the expanded vma and otherwise +* compatible. +*/ + vma =vma_merge_extend (&vmi, vma, delta); +if (!vma) { +vm_unacct_memory (charged); +return -ENOMEM ; + } + +vm_stat_account (mm, vma->vm_flags, pages); +if (vma->vm_flags &VM_LOCKED ) { + mm->locked_vm += pages; + *locked = true; + } + +return 0 ; +} + +static bool align_hugetlb (struct vm_area_struct *vma, +unsigned long addr, +unsigned long new_addr, +unsigned long *old_len_ptr, +unsigned long *new_len_ptr, +unsigned long *delta_ptr) +{ +unsigned long old_len = *old_len_ptr; +unsigned long new_len = *new_len_ptr; +struct hstate *h__maybe_unused =hstate_vma (vma); + + old_len =ALIGN (old_len,huge_page_size (h)); + new_len =ALIGN (new_len,huge_page_size (h)); + +/* addrs must be huge page aligned */ +if (addr & ~huge_page_mask (h)) +return false; +if (new_addr & ~huge_page_mask (h)) +return false; + +/* +* Don't allow remap expansion, because the underlying hugetlb +* reservation is not yet capable to handle split reservation. +*/ +if (new_len > old_len) +return false; + + *old_len_ptr = old_len; + *new_len_ptr = new_len; + *delta_ptr =abs_diff (old_len, new_len); +return true; +} + +/* +* We are mremap()'ing without specifying a fixed address to move to, but are +* requesting that the VMA's size be increased. +* +* Try to do so in-place, if this fails, then move the VMA to a new location to +* action the change. +*/ +static unsigned long expand_vma (struct vm_area_struct *vma, +unsigned long addr,unsigned long old_len, +unsigned long new_len,unsigned long flags, +bool *locked_ptr,unsigned long *new_addr_ptr, +struct vm_userfaultfd_ctx *uf_ptr, +struct list_head *uf_unmap_ptr) +{ +unsigned long err; +unsigned long map_flags; +unsigned long new_addr;/* We ignore any user-supplied one. */ +pgoff_t pgoff; + + err =resize_is_valid (vma, addr, old_len, new_len, flags); +if (err) +return err; + +/* +* [addr, old_len) spans precisely to the end of the VMA, so try to +* expand it in-place. +*/ +if (old_len == vma->vm_end - addr && +vma_expandable (vma, new_len - old_len)) { + err =expand_vma_inplace (vma, new_len - old_len, locked_ptr); +if (IS_ERR_VALUE (err)) +return err; + +/* +* We want to populate the newly expanded portion of the VMA to +* satisfy the expectation that mlock()'ing a VMA maintains all +* of its pages in memory. +*/ +if (*locked_ptr) + *new_addr_ptr = addr; + +/* OK we're done! */ +return addr; + } + +/* +* We weren't able to just expand or shrink the area, +* we need to create a new one and move it. +*/ + +/* We're not allowed to move the VMA, so error out. */ +if (!(flags &MREMAP_MAYMOVE )) +return -ENOMEM ; + +/* Find a new location to move the VMA to. */ + map_flags = (vma->vm_flags &VM_MAYSHARE ) ?MAP_SHARED :0 ; + pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >>PAGE_SHIFT ); + new_addr =get_unmapped_area (vma->vm_file,0 , new_len, pgoff, map_flags); +if (IS_ERR_VALUE (new_addr)) +return new_addr; + *new_addr_ptr = new_addr; + +return move_vma (vma, addr, old_len, new_len, new_addr, + locked_ptr, flags, uf_ptr, uf_unmap_ptr); +} +/* * Expand (or shrink) an existing mapping, potentially moving it at the * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
@@ -1048,7 +1241,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, a {struct mm_struct *mm = current->mm;struct vm_area_struct *vma; -unsigned long ret = -EINVAL ; +unsigned long ret; +unsigned long delta;bool locked = false;struct vm_userfaultfd_ctx uf =NULL_VM_UFFD_CTX ;LIST_HEAD (uf_unmap_early);
@@ -1067,70 +1261,38 @@ SYSCALL_DEFINE5(mremap, unsigned long, a*/ addr =untagged_addr (addr); -if (flags & ~(MREMAP_FIXED |MREMAP_MAYMOVE |MREMAP_DONTUNMAP )) -return ret; - -if (flags &MREMAP_FIXED && !(flags &MREMAP_MAYMOVE )) -return ret; - -/* -* MREMAP_DONTUNMAP is always a move and it does not allow resizing -* in the process. -*/ -if (flags &MREMAP_DONTUNMAP && - (!(flags &MREMAP_MAYMOVE ) || old_len != new_len)) -return ret; - - -if (offset_in_page (addr)) + ret =check_mremap_params (addr, flags, old_len, new_len, new_addr); +if (ret)return ret; old_len =PAGE_ALIGN (old_len); new_len =PAGE_ALIGN (new_len); + delta =abs_diff (old_len, new_len); -/* -* We allow a zero old-len as a special case -* for DOS-emu "duplicate shm area" thing. But -* a zero new-len is nonsensical. -*/ -if (!new_len) -return ret; - -if (mmap_write_lock_killable (current->mm)) +if (mmap_write_lock_killable (mm))return -EINTR ; + vma =vma_lookup (mm, addr);if (!vma) { ret = -EFAULT ; goto out; } -/* Don't allow remapping vmas when they have already been sealed */ +/* If mseal()'d, mremap() is prohibited. */ if (!can_modify_vma (vma)) { ret = -EPERM ; goto out; } -if (is_vm_hugetlb_page (vma)) { -struct hstate *h__maybe_unused =hstate_vma (vma); - - old_len =ALIGN (old_len,huge_page_size (h)); - new_len =ALIGN (new_len,huge_page_size (h)); - -/* addrs must be huge page aligned */ -if (addr & ~huge_page_mask (h)) - goto out; -if (new_addr & ~huge_page_mask (h)) - goto out; - -/* -* Don't allow remap expansion, because the underlying hugetlb -* reservation is not yet capable to handle split reservation. -*/ -if (new_len > old_len) - goto out; +/* Align to hugetlb page size, if required. */ +if (is_vm_hugetlb_page (vma) && + !align_hugetlb (vma, addr, new_addr, &old_len, &new_len, &delta)) { + ret = -EINVAL ; + goto out; } -if (flags & (MREMAP_FIXED |MREMAP_DONTUNMAP )) { +/* Are we RELOCATING the VMA to a SPECIFIC address? */ +if (implies_new_addr (flags)) { ret =mremap_to (addr, old_len, new_addr, new_len, &locked, flags, &uf, &uf_unmap_early, &uf_unmap);
@@ -1138,109 +1300,44 @@ SYSCALL_DEFINE5(mremap, unsigned long, a }/* -* Always allow a shrinking remap: that just unmaps -* the unnecessary pages.. -* do_vmi_munmap does all the needed commit accounting, and -* unlocks the mmap_lock if so directed. +* From here on in we are only RESIZING the VMA, attempting to do so +* in-place, moving the VMA if we cannot. */ -if (old_len >= new_len) { -VMA_ITERATOR (vmi, mm, addr + new_len); -if (old_len == new_len) { - ret = addr; - goto out; - } +/* NO-OP CASE - resizing to the same size. */ +if (new_len == old_len) { + ret = addr; + goto out; + } + +/* SHRINK CASE. Can always be done in-place. */ +if (new_len < old_len) { +VMA_ITERATOR (vmi, mm, addr + new_len); - ret =do_vmi_munmap (&vmi, mm, addr + new_len, old_len - new_len, +/* +* Simply unmap the shrunken portion of the VMA. This does all +* the needed commit accounting, unlocking the mmap lock. +*/ + ret =do_vmi_munmap (&vmi, mm, addr + new_len, delta, &uf_unmap, true);if (ret) goto out; +/* We succeeded, mmap lock released for us. */ ret = addr; goto out_unlocked; } -/* -* Ok, we need to grow.. -*/ - ret =resize_is_valid (vma, addr, old_len, new_len, flags); -if (ret) - goto out; +/* EXPAND case. We try to do in-place, if we can't, then we move it. */ + ret =expand_vma (vma, addr, old_len, new_len, flags, &locked, &new_addr, + &uf, &uf_unmap); -/* old_len exactly to the end of the area.. -*/ -if (old_len == vma->vm_end - addr) { -unsigned long delta = new_len - old_len; - -/* can we just expand the current mapping? */ -if (vma_expandable (vma, delta)) { -long pages = delta >>PAGE_SHIFT ; -VMA_ITERATOR (vmi, mm, vma->vm_end); -long charged =0 ; - -if (vma->vm_flags &VM_ACCOUNT ) { -if (security_vm_enough_memory_mm (mm, pages)) { - ret = -ENOMEM ; - goto out; - } - charged = pages; - } - -/* -* Function vma_merge_extend() is called on the -* extension we are adding to the already existing vma, -* vma_merge_extend() will merge this extension with the -* already existing vma (expand operation itself) and -* possibly also with the next vma if it becomes -* adjacent to the expanded vma and otherwise -* compatible. -*/ - vma =vma_merge_extend (&vmi, vma, delta); -if (!vma) { -vm_unacct_memory (charged); - ret = -ENOMEM ; - goto out; - } - -vm_stat_account (mm, vma->vm_flags, pages); -if (vma->vm_flags &VM_LOCKED ) { - mm->locked_vm += pages; - locked = true; - new_addr = addr; - } - ret = addr; - goto out; - } - } - -/* -* We weren't able to just expand or shrink the area, -* we need to create a new one and move it.. -*/ - ret = -ENOMEM ; -if (flags &MREMAP_MAYMOVE ) { -unsigned long map_flags =0 ; -if (vma->vm_flags &VM_MAYSHARE ) - map_flags |=MAP_SHARED ; - - new_addr =get_unmapped_area (vma->vm_file,0 , new_len, - vma->vm_pgoff + - ((addr - vma->vm_start) >>PAGE_SHIFT ), - map_flags); -if (IS_ERR_VALUE (new_addr)) { - ret = new_addr; - goto out; - } - - ret =move_vma (vma, addr, old_len, new_len, new_addr, - &locked, flags, &uf, &uf_unmap); - } out:if (offset_in_page (ret)) locked = false; -mmap_write_unlock (current->mm); +mmap_write_unlock (mm);if (locked && new_len > old_len) -mm_populate (new_addr + old_len, new_len - old_len); +mm_populate (new_addr + old_len, delta); out_unlocked:userfaultfd_unmap_complete (mm, &uf_unmap_early);mremap_userfaultfd_complete (&uf, addr, ret, old_len);
_
Patches currently in -mm which might be from [email protected] are