Skip to contentSkip to conversation

mm/mremap: Don't account pages in vma_to_resize()

PATCH Applied Dmitry Safonov 4 participants 7 messages Jul 21, 2021 – Jul 22, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 mm/mremap: Don't account pages in vma_to_resize()

All this vm_unacct_memory(charged) dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes in mremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growing MREMAP_DONTUNMAP mremap()s in move_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

Cc Alexander Viro <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Geffon <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Chen Wandun <[email protected]>
Cc Dan Carpenter <[email protected]>
Cc Dan Williams <[email protected]>
Cc Dave Jiang <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc John Hubbard <[email protected]>
Cc Kefeng Wang <[email protected]>
Cc "Kirill A. Shutemov" <[email protected]>
Cc Mike Kravetz <[email protected]>
Cc Minchan Kim <[email protected]>
Cc Ralph Campbell <[email protected]>
Cc Russell King <[email protected]>
Cc Thomas Bogendoerfer <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Vishal Verma <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Wei Yongjun <[email protected]>
Cc Will Deacon <[email protected]>
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")
---
 mm/mremap.c | 50 ++++++++++++++++++++++----------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)
mm/mremap.c +22 -28
index 5989d3990020..b90c8e732e29 100644
@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 		bool *locked, unsigned long flags,
 		struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
 {
+	long to_account = new_len - old_len;
 	struct mm_struct *mm = vma->vm_mm;
 	struct vm_area_struct *new_vma;
 	unsigned long vm_flags = vma->vm_flags;
@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	if (mm->map_count >= sysctl_max_map_count - 3)
 		return -ENOMEM;
 
+	if (unlikely(flags & MREMAP_DONTUNMAP))
+		to_account = new_len;
+
 	if (vma->vm_ops && vma->vm_ops->may_split) {
 		if (vma->vm_start != old_addr)
 			err = vma->vm_ops->may_split(vma, old_addr);
@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	if (err)
 		return err;
 
-	if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) {
-		if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
+	if (vm_flags & VM_ACCOUNT) {
+		if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
 			return -ENOMEM;
 	}
 
@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
 			   &need_rmap_locks);
 	if (!new_vma) {
-		if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT))
-			vm_unacct_memory(new_len >> PAGE_SHIFT);
+		if (vm_flags & VM_ACCOUNT)
+			vm_unacct_memory(to_account >> PAGE_SHIFT);
 		return -ENOMEM;
 	}
 
@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 }
 
 static struct vm_area_struct *vma_to_resize(unsigned long addr,
-	unsigned long old_len, unsigned long new_len, unsigned long flags,
-	unsigned long *p)
+	unsigned long old_len, unsigned long new_len, unsigned long flags)
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
 				(new_len - old_len) >> PAGE_SHIFT))
 		return ERR_PTR(-ENOMEM);
 
-	if (vma->vm_flags & VM_ACCOUNT) {
-		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-		if (security_vm_enough_memory_mm(mm, charged))
-			return ERR_PTR(-ENOMEM);
-		*p = charged;
-	}
-
 	return vma;
 }
 
@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long ret = -EINVAL;
-	unsigned long charged = 0;
 	unsigned long map_flags = 0;
 
 	if (offset_in_page(new_addr))
@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 		old_len = new_len;
 	}
 
-	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+	vma = vma_to_resize(addr, old_len, new_len, flags);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out;
@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 				((addr - vma->vm_start) >> PAGE_SHIFT),
 				map_flags);
 	if (IS_ERR_VALUE(ret))
-		goto out1;
+		goto out;
 
 	/* We got a new mapping */
 	if (!(flags & MREMAP_FIXED))
@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 	ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
 		       uf_unmap);
 
-	if (!(offset_in_page(ret)))
-		goto out;
-
-out1:
-	vm_unacct_memory(charged);
-
 out:
 	return ret;
 }
@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long ret = -EINVAL;
-	unsigned long charged = 0;
 	bool locked = false;
 	bool downgraded = false;
 	struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	/*
 	 * Ok, we need to grow..
 	 */
-	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+	vma = vma_to_resize(addr, old_len, new_len, flags);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out;
@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	if (old_len == vma->vm_end - addr) {
 		/* can we just expand the current mapping? */
 		if (vma_expandable(vma, new_len - old_len)) {
-			int pages = (new_len - old_len) >> PAGE_SHIFT;
+			long pages = (new_len - old_len) >> PAGE_SHIFT;
+
+			if (vma->vm_flags & VM_ACCOUNT) {
+				if (security_vm_enough_memory_mm(mm, pages)) {
+					ret = -ENOMEM;
+					goto out;
+				}
+			}
 
 			if (vma_adjust(vma, vma->vm_start, addr + new_len,
 				       vma->vm_pgoff, NULL)) {
+				vm_unacct_memory(pages);
 				ret = -ENOMEM;
 				goto out;
 			}
@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 			       &locked, flags, &uf, &uf_unmap);
 	}
 out:
-	if (offset_in_page(ret)) {
-		vm_unacct_memory(charged);
+	if (offset_in_page(ret))
 		locked = false;
-	}
 	if (downgraded)
 		mmap_read_unlock(current->mm);
 	else
Signature · 1 line
2.32.0
#1 lore raw
Quoted 7 lines · Dmitry Safonov

All this vm_unacct_memory(charged) dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes in mremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growing MREMAP_DONTUNMAP mremap()s in move_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))

Quoted 8 lines

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

The patch by Chen Wandun still makes sense with this v2. Heh :-)

Quoted 189 lines
Cc Alexander Viro <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Geffon <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Chen Wandun <[email protected]>
Cc Dan Carpenter <[email protected]>
Cc Dan Williams <[email protected]>
Cc Dave Jiang <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc John Hubbard <[email protected]>
Cc Kefeng Wang <[email protected]>
Cc "Kirill A. Shutemov" <[email protected]>
Cc Mike Kravetz <[email protected]>
Cc Minchan Kim <[email protected]>
Cc Ralph Campbell <[email protected]>
Cc Russell King <[email protected]>
Cc Thomas Bogendoerfer <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Vishal Verma <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Wei Yongjun <[email protected]>
Cc Will Deacon <[email protected]>
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")
---
 mm/mremap.c | 50 ++++++++++++++++++++++----------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)
mm/mremap.c +22 -28
index 5989d3990020..b90c8e732e29 100644
@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 		bool *locked, unsigned long flags,
 		struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
 {
+	long to_account = new_len - old_len;
 	struct mm_struct *mm = vma->vm_mm;
 	struct vm_area_struct *new_vma;
 	unsigned long vm_flags = vma->vm_flags;
@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	if (mm->map_count >= sysctl_max_map_count - 3)
 		return -ENOMEM;
 
+	if (unlikely(flags & MREMAP_DONTUNMAP))
+		to_account = new_len;
+
 	if (vma->vm_ops && vma->vm_ops->may_split) {
 		if (vma->vm_start != old_addr)
 			err = vma->vm_ops->may_split(vma, old_addr);
@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	if (err)
 		return err;
 
-	if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) {
-		if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
+	if (vm_flags & VM_ACCOUNT) {
+		if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
 			return -ENOMEM;
 	}
 
@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
 			   &need_rmap_locks);
 	if (!new_vma) {
-		if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT))
-			vm_unacct_memory(new_len >> PAGE_SHIFT);
+		if (vm_flags & VM_ACCOUNT)
+			vm_unacct_memory(to_account >> PAGE_SHIFT);
 		return -ENOMEM;
 	}
 
@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 }
 
 static struct vm_area_struct *vma_to_resize(unsigned long addr,
-	unsigned long old_len, unsigned long new_len, unsigned long flags,
-	unsigned long *p)
+	unsigned long old_len, unsigned long new_len, unsigned long flags)
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
 				(new_len - old_len) >> PAGE_SHIFT))
 		return ERR_PTR(-ENOMEM);
 
-	if (vma->vm_flags & VM_ACCOUNT) {
-		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-		if (security_vm_enough_memory_mm(mm, charged))
-			return ERR_PTR(-ENOMEM);
-		*p = charged;
-	}
-
 	return vma;
 }
 
@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long ret = -EINVAL;
-	unsigned long charged = 0;
 	unsigned long map_flags = 0;
 
 	if (offset_in_page(new_addr))
@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 		old_len = new_len;
 	}
 
-	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+	vma = vma_to_resize(addr, old_len, new_len, flags);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out;
@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 				((addr - vma->vm_start) >> PAGE_SHIFT),
 				map_flags);
 	if (IS_ERR_VALUE(ret))
-		goto out1;
+		goto out;
 
 	/* We got a new mapping */
 	if (!(flags & MREMAP_FIXED))
@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 	ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
 		       uf_unmap);
 
-	if (!(offset_in_page(ret)))
-		goto out;
-
-out1:
-	vm_unacct_memory(charged);
-
 out:
 	return ret;
 }
@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long ret = -EINVAL;
-	unsigned long charged = 0;
 	bool locked = false;
 	bool downgraded = false;
 	struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	/*
 	 * Ok, we need to grow..
 	 */
-	vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+	vma = vma_to_resize(addr, old_len, new_len, flags);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out;
@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 	if (old_len == vma->vm_end - addr) {
 		/* can we just expand the current mapping? */
 		if (vma_expandable(vma, new_len - old_len)) {
-			int pages = (new_len - old_len) >> PAGE_SHIFT;
+			long pages = (new_len - old_len) >> PAGE_SHIFT;
+
+			if (vma->vm_flags & VM_ACCOUNT) {
+				if (security_vm_enough_memory_mm(mm, pages)) {
+					ret = -ENOMEM;
+					goto out;
+				}
+			}
 
 			if (vma_adjust(vma, vma->vm_start, addr + new_len,
 				       vma->vm_pgoff, NULL)) {
+				vm_unacct_memory(pages);
 				ret = -ENOMEM;
 				goto out;
 			}
@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 			       &locked, flags, &uf, &uf_unmap);
 	}
 out:
-	if (offset_in_page(ret)) {
-		vm_unacct_memory(charged);
+	if (offset_in_page(ret))
 		locked = false;
-	}
 	if (downgraded)
 		mmap_read_unlock(current->mm);
 	else
Signature · 1 line
Dima
maintainer
#2 lore raw
Quoted 11 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

The patch by Chen Wandun still makes sense with this v2. Heh :-)

Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?

author Acked-by
#3 lore raw
Quoted 11 lines · Andrew Morton
Quoted 9 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

Quoted 7 lines
Quoted 2 lines

The patch by Chen Wandun still makes sense with this v2. Heh :-)

Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?

Yes, please keep that patch with

This one comes on the top to correct accounting for MREMAP_DONTUNMAP.

Thanks,
Dmitry

#4 lore raw
Quoted 20 lines · Dmitry Safonov
Quoted 11 lines · Andrew Morton
Quoted 9 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

Quoted 7 lines
Quoted 2 lines

The patch by Chen Wandun still makes sense with this v2. Heh :-)

Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?

Quoted 4 lines

Yes, please keep that patch with

This one comes on the top to correct accounting for MREMAP_DONTUNMAP.

Please, also wait for reviews for this one.
I think it's correct and simplifies the complex code, but now I look into the code again, I see that:
: /*
: * MREMAP_DONTUNMAP is always a move and it does not allow resizing
: * in the process.
: */
: if (flags & MREMAP_DONTUNMAP &&
: (!(flags & MREMAP_MAYMOVE) || old_len != new_len))
: return ret;

So, this part can be dropped:

Quoted 1 line
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")

But I think the patch still makes sense.

Brian, can I have your review on it?

Thanks,
Dmitry

#5 lore raw
In reply to Dmitry Safonov
Quoted 175 lines · Dmitry Safonov

All this vm_unacct_memory(charged) dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes in mremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growing MREMAP_DONTUNMAP mremap()s in move_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

Cc Alexander Viro <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Geffon <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Chen Wandun <[email protected]>
Cc Dan Carpenter <[email protected]>
Cc Dan Williams <[email protected]>
Cc Dave Jiang <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc John Hubbard <[email protected]>
Cc Kefeng Wang <[email protected]>
Cc "Kirill A. Shutemov" <[email protected]>
Cc Mike Kravetz <[email protected]>
Cc Minchan Kim <[email protected]>
Cc Ralph Campbell <[email protected]>
Cc Russell King <[email protected]>
Cc Thomas Bogendoerfer <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Vishal Verma <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Wei Yongjun <[email protected]>
Cc Will Deacon <[email protected]>
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")
---
 mm/mremap.c | 50 ++++++++++++++++++++++----------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)
mm/mremap.c +12 -24
index 5989d3990020..b90c8e732e29 100644
@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
                bool *locked, unsigned long flags,
                struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
 {
+       long to_account = new_len - old_len;
        struct mm_struct *mm = vma->vm_mm;
        struct vm_area_struct *new_vma;
        unsigned long vm_flags = vma->vm_flags;
@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        if (mm->map_count >= sysctl_max_map_count - 3)
                return -ENOMEM;
 
+       if (unlikely(flags & MREMAP_DONTUNMAP))
+               to_account = new_len;
+
        if (vma->vm_ops && vma->vm_ops->may_split) {
                if (vma->vm_start != old_addr)
                        err = vma->vm_ops->may_split(vma, old_addr);
@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        if (err)
                return err;
 
-       if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) {
-               if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
+       if (vm_flags & VM_ACCOUNT) {
+               if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
                        return -ENOMEM;
        }
 
@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
                           &need_rmap_locks);
        if (!new_vma) {
-               if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT))
-                       vm_unacct_memory(new_len >> PAGE_SHIFT);
+               if (vm_flags & VM_ACCOUNT)
+                       vm_unacct_memory(to_account >> PAGE_SHIFT);
                return -ENOMEM;
        }
 
@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 }
 
 static struct vm_area_struct *vma_to_resize(unsigned long addr,
-       unsigned long old_len, unsigned long new_len, unsigned long flags,
-       unsigned long *p)
+       unsigned long old_len, unsigned long new_len, unsigned long flags)
 {
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
                                (new_len - old_len) >> PAGE_SHIFT))
                return ERR_PTR(-ENOMEM);
 
-       if (vma->vm_flags & VM_ACCOUNT) {
-               unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-               if (security_vm_enough_memory_mm(mm, charged))
-                       return ERR_PTR(-ENOMEM);
-               *p = charged;
-       }
-
        return vma;
 }
 
@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long ret = -EINVAL;
-       unsigned long charged = 0;
        unsigned long map_flags = 0;
 
        if (offset_in_page(new_addr))
@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
                old_len = new_len;
        }
 
-       vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+       vma = vma_to_resize(addr, old_len, new_len, flags);
        if (IS_ERR(vma)) {
                ret = PTR_ERR(vma);
                goto out;
@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
                                ((addr - vma->vm_start) >> PAGE_SHIFT),
                                map_flags);
        if (IS_ERR_VALUE(ret))
-               goto out1;
+               goto out;
 
        /* We got a new mapping */
        if (!(flags & MREMAP_FIXED))
@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
        ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
                       uf_unmap);
 
-       if (!(offset_in_page(ret)))
-               goto out;
-
-out1:
-       vm_unacct_memory(charged);
-
 out:
        return ret;
 }
@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long ret = -EINVAL;
-       unsigned long charged = 0;
        bool locked = false;
        bool downgraded = false;
        struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        /*
         * Ok, we need to grow..
         */
-       vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+       vma = vma_to_resize(addr, old_len, new_len, flags);
        if (IS_ERR(vma)) {
                ret = PTR_ERR(vma);
                goto out;
@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        if (old_len == vma->vm_end - addr) {
                /* can we just expand the current mapping? */

mm/mremap.c SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · line 990

Quoted 10 lines
mm/mremap.c +8 -1
                if (vma_expandable(vma, new_len - old_len)) {
-                       int pages = (new_len - old_len) >> PAGE_SHIFT;
+                       long pages = (new_len - old_len) >> PAGE_SHIFT;
+
+                       if (vma->vm_flags & VM_ACCOUNT) {
+                               if (security_vm_enough_memory_mm(mm, pages)) {
+                                       ret = -ENOMEM;
+                                       goto out;
+                               }
+                       }

Wasn't this whole check already performed in vma_to_resize()?

Quoted 22 lines
mm/mremap.c +2 -4
                        if (vma_adjust(vma, vma->vm_start, addr + new_len,
                                       vma->vm_pgoff, NULL)) {
+                               vm_unacct_memory(pages);
                                ret = -ENOMEM;
                                goto out;
                        }
@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                               &locked, flags, &uf, &uf_unmap);
        }
 out:
-       if (offset_in_page(ret)) {
-               vm_unacct_memory(charged);
+       if (offset_in_page(ret))
                locked = false;
-       }
        if (downgraded)
                mmap_read_unlock(current->mm);
        else
--

2.32.0

Acked-by
#6 lore raw

Whoops I missed that you did remove it from vma_to_resize() so then it looks good to me.

Quoted 212 lines · Brian Geffon
Quoted 175 lines · Dmitry Safonov

All this vm_unacct_memory(charged) dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes in mremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growing MREMAP_DONTUNMAP mremap()s in move_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))

Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.

Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]

Cc Alexander Viro <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc Brian Geffon <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Chen Wandun <[email protected]>
Cc Dan Carpenter <[email protected]>
Cc Dan Williams <[email protected]>
Cc Dave Jiang <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc John Hubbard <[email protected]>
Cc Kefeng Wang <[email protected]>
Cc "Kirill A. Shutemov" <[email protected]>
Cc Mike Kravetz <[email protected]>
Cc Minchan Kim <[email protected]>
Cc Ralph Campbell <[email protected]>
Cc Russell King <[email protected]>
Cc Thomas Bogendoerfer <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Vishal Verma <[email protected]>
Cc Vlastimil Babka <[email protected]>
Cc Wei Yongjun <[email protected]>
Cc Will Deacon <[email protected]>
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")
---
 mm/mremap.c | 50 ++++++++++++++++++++++----------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)
mm/mremap.c +12 -24
index 5989d3990020..b90c8e732e29 100644
@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
                bool *locked, unsigned long flags,
                struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
 {
+       long to_account = new_len - old_len;
        struct mm_struct *mm = vma->vm_mm;
        struct vm_area_struct *new_vma;
        unsigned long vm_flags = vma->vm_flags;
@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        if (mm->map_count >= sysctl_max_map_count - 3)
                return -ENOMEM;
 
+       if (unlikely(flags & MREMAP_DONTUNMAP))
+               to_account = new_len;
+
        if (vma->vm_ops && vma->vm_ops->may_split) {
                if (vma->vm_start != old_addr)
                        err = vma->vm_ops->may_split(vma, old_addr);
@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        if (err)
                return err;
 
-       if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) {
-               if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
+       if (vm_flags & VM_ACCOUNT) {
+               if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
                        return -ENOMEM;
        }
 
@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
                           &need_rmap_locks);
        if (!new_vma) {
-               if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT))
-                       vm_unacct_memory(new_len >> PAGE_SHIFT);
+               if (vm_flags & VM_ACCOUNT)
+                       vm_unacct_memory(to_account >> PAGE_SHIFT);
                return -ENOMEM;
        }
 
@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 }
 
 static struct vm_area_struct *vma_to_resize(unsigned long addr,
-       unsigned long old_len, unsigned long new_len, unsigned long flags,
-       unsigned long *p)
+       unsigned long old_len, unsigned long new_len, unsigned long flags)
 {
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
                                (new_len - old_len) >> PAGE_SHIFT))
                return ERR_PTR(-ENOMEM);
 
-       if (vma->vm_flags & VM_ACCOUNT) {
-               unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-               if (security_vm_enough_memory_mm(mm, charged))
-                       return ERR_PTR(-ENOMEM);
-               *p = charged;
-       }
-
        return vma;
 }
 
@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long ret = -EINVAL;
-       unsigned long charged = 0;
        unsigned long map_flags = 0;
 
        if (offset_in_page(new_addr))
@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
                old_len = new_len;
        }
 
-       vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+       vma = vma_to_resize(addr, old_len, new_len, flags);
        if (IS_ERR(vma)) {
                ret = PTR_ERR(vma);
                goto out;
@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
                                ((addr - vma->vm_start) >> PAGE_SHIFT),
                                map_flags);
        if (IS_ERR_VALUE(ret))
-               goto out1;
+               goto out;
 
        /* We got a new mapping */
        if (!(flags & MREMAP_FIXED))
@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
        ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
                       uf_unmap);
 
-       if (!(offset_in_page(ret)))
-               goto out;
-
-out1:
-       vm_unacct_memory(charged);
-
 out:
        return ret;
 }
@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long ret = -EINVAL;
-       unsigned long charged = 0;
        bool locked = false;
        bool downgraded = false;
        struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        /*
         * Ok, we need to grow..
         */
-       vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
+       vma = vma_to_resize(addr, old_len, new_len, flags);
        if (IS_ERR(vma)) {
                ret = PTR_ERR(vma);
                goto out;
@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
        if (old_len == vma->vm_end - addr) {
                /* can we just expand the current mapping? */

mm/mremap.c SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · line 990

Quoted 10 lines
mm/mremap.c +8 -1
                if (vma_expandable(vma, new_len - old_len)) {
-                       int pages = (new_len - old_len) >> PAGE_SHIFT;
+                       long pages = (new_len - old_len) >> PAGE_SHIFT;
+
+                       if (vma->vm_flags & VM_ACCOUNT) {
+                               if (security_vm_enough_memory_mm(mm, pages)) {
+                                       ret = -ENOMEM;
+                                       goto out;
+                               }
+                       }

Wasn't this whole check already performed in vma_to_resize()?

Quoted 22 lines
mm/mremap.c +2 -4
                        if (vma_adjust(vma, vma->vm_start, addr + new_len,
                                       vma->vm_pgoff, NULL)) {
+                               vm_unacct_memory(pages);
                                ret = -ENOMEM;
                                goto out;
                        }
@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                               &locked, flags, &uf, &uf_unmap);
        }
 out:
-       if (offset_in_page(ret)) {
-               vm_unacct_memory(charged);
+       if (offset_in_page(ret))
                locked = false;
-       }
        if (downgraded)
                mmap_read_unlock(current->mm);
        else
--

2.32.0