mm/mremap: Don't account pages in vma_to_resize()
Subsystems (2)
PATCH v2 1/1 mm/mremap: Don't account pages in vma_to_resize()
All this vm_unacct_memory(charged) dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes in mremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growing MREMAP_DONTUNMAP mremap()s in move_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
Let's not do this.
Account memory in mremap() fast-path for growing VMAs or in move_vma() for actually moving things. The same simpler way as it's done by vm_stat_account(), but with a difference to call security_vm_enough_memory_mm() before copying/adjusting VMA.
Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]
--- mm/mremap.c | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-)
mm/mremap.c +22 -28
@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, bool *locked, unsigned long flags, struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) { + long to_account = new_len - old_len;struct mm_struct *mm = vma->vm_mm;struct vm_area_struct *new_vma;unsigned long vm_flags = vma->vm_flags;
@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (mm->map_count >= sysctl_max_map_count -3 )return -ENOMEM ; +if (unlikely (flags &MREMAP_DONTUNMAP )) + to_account = new_len; +if (vma->vm_ops && vma->vm_ops->may_split) {if (vma->vm_start != old_addr) err = vma->vm_ops->may_split (vma, old_addr);
@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (err)return err; -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) { -if (security_vm_enough_memory_mm (mm, new_len >>PAGE_SHIFT )) +if (vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, to_account >>PAGE_SHIFT ))return -ENOMEM ; }
@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, new_vma =copy_vma (&vma, new_addr, new_len, new_pgoff, &need_rmap_locks);if (!new_vma) { -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) -vm_unacct_memory (new_len >>PAGE_SHIFT ); +if (vm_flags &VM_ACCOUNT ) +vm_unacct_memory (to_account >>PAGE_SHIFT );return -ENOMEM ; }
@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, }static struct vm_area_struct *vma_to_resize (unsigned long addr, -unsigned long old_len,unsigned long new_len,unsigned long flags, -unsigned long *p) +unsigned long old_len,unsigned long new_len,unsigned long flags) {struct mm_struct *mm = current->mm;struct vm_area_struct *vma;
@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, (new_len - old_len) >> PAGE_SHIFT))return ERR_PTR (-ENOMEM ); -if (vma->vm_flags &VM_ACCOUNT ) { -unsigned long charged = (new_len - old_len) >>PAGE_SHIFT ; -if (security_vm_enough_memory_mm (mm, charged)) -return ERR_PTR (-ENOMEM ); - *p = charged; - } -return vma; }
@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;unsigned long map_flags =0 ;if (offset_in_page (new_addr))
@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, old_len = new_len; } - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;
@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ((addr - vma->vm_start) >>PAGE_SHIFT ), map_flags);if (IS_ERR_VALUE (ret)) - goto out1; + goto out;/* We got a new mapping */ if (!(flags &MREMAP_FIXED ))
@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ret =move_vma (vma, addr, old_len, new_len, new_addr, locked, flags, uf, uf_unmap); -if (!(offset_in_page (ret))) - goto out; - -out1: -vm_unacct_memory (charged); - out:return ret; }
@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;bool locked = false;bool downgraded = false;struct vm_userfaultfd_ctx uf =NULL_VM_UFFD_CTX ;
@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,/* * Ok, we need to grow.. */ - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;
@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,if (old_len == vma->vm_end - addr) {/* can we just expand the current mapping? */ if (vma_expandable (vma, new_len - old_len)) { -int pages = (new_len - old_len) >>PAGE_SHIFT ; +long pages = (new_len - old_len) >>PAGE_SHIFT ; + +if (vma->vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, pages)) { + ret = -ENOMEM ; + goto out; + } + }if (vma_adjust (vma, vma->vm_start, addr + new_len, vma->vm_pgoff,NULL )) { +vm_unacct_memory (pages); ret = -ENOMEM ; goto out; }
@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, &locked, flags, &uf, &uf_unmap); } out: -if (offset_in_page (ret)) { -vm_unacct_memory (charged); +if (offset_in_page (ret)) locked = false; - }if (downgraded)mmap_read_unlock (current->mm);else
Signature · 1 line
2.32.0
Quoted 7 lines · Dmitry Safonov
All this
vm_unacct_memory(charged)dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes inmremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growingMREMAP_DONTUNMAPmremap()s inmove_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))
Quoted 8 lines
Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]
The patch by Chen Wandun still makes sense with this v2. Heh :-)
Quoted 189 lines
Cc Alexander Viro <[email protected]>Cc Andrew Morton <[email protected]>Cc Andy Lutomirski <[email protected]>Cc Brian Geffon <[email protected]>Cc Catalin Marinas <[email protected]>Cc Chen Wandun <[email protected]>Cc Dan Carpenter <[email protected]>Cc Dan Williams <[email protected]>Cc Dave Jiang <[email protected]>Cc Hugh Dickins <[email protected]>Cc Ingo Molnar <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc John Hubbard <[email protected]>Cc Kefeng Wang <[email protected]>Cc "Kirill A. Shutemov" <[email protected]>Cc Mike Kravetz <[email protected]>Cc Minchan Kim <[email protected]>Cc Ralph Campbell <[email protected]>Cc Russell King <[email protected]>Cc Thomas Bogendoerfer <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Vishal Verma <[email protected]>Cc Vlastimil Babka <[email protected]>Cc Wei Yongjun <[email protected]>Cc Will Deacon <[email protected]>Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")Signed-off-by Dmitry Safonov <[email protected]>--- mm/mremap.c | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-)mm/mremap.c +22
-28@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, bool *locked, unsigned long flags, struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) { + long to_account = new_len - old_len;struct mm_struct *mm = vma->vm_mm;struct vm_area_struct *new_vma;unsigned long vm_flags = vma->vm_flags;@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (mm->map_count >= sysctl_max_map_count -3 )return -ENOMEM ; +if (unlikely (flags &MREMAP_DONTUNMAP )) + to_account = new_len; +if (vma->vm_ops && vma->vm_ops->may_split) {if (vma->vm_start != old_addr) err = vma->vm_ops->may_split (vma, old_addr);@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (err)return err; -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) { -if (security_vm_enough_memory_mm (mm, new_len >>PAGE_SHIFT )) +if (vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, to_account >>PAGE_SHIFT ))return -ENOMEM ; }@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, new_vma =copy_vma (&vma, new_addr, new_len, new_pgoff, &need_rmap_locks);if (!new_vma) { -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) -vm_unacct_memory (new_len >>PAGE_SHIFT ); +if (vm_flags &VM_ACCOUNT ) +vm_unacct_memory (to_account >>PAGE_SHIFT );return -ENOMEM ; }@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, }static struct vm_area_struct *vma_to_resize (unsigned long addr, -unsigned long old_len,unsigned long new_len,unsigned long flags, -unsigned long *p) +unsigned long old_len,unsigned long new_len,unsigned long flags) {struct mm_struct *mm = current->mm;struct vm_area_struct *vma;@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, (new_len - old_len) >> PAGE_SHIFT))return ERR_PTR (-ENOMEM ); -if (vma->vm_flags &VM_ACCOUNT ) { -unsigned long charged = (new_len - old_len) >>PAGE_SHIFT ; -if (security_vm_enough_memory_mm (mm, charged)) -return ERR_PTR (-ENOMEM ); - *p = charged; - } -return vma; }@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;unsigned long map_flags =0 ;if (offset_in_page (new_addr))@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, old_len = new_len; } - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ((addr - vma->vm_start) >>PAGE_SHIFT ), map_flags);if (IS_ERR_VALUE (ret)) - goto out1; + goto out;/* We got a new mapping */ if (!(flags &MREMAP_FIXED ))@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ret =move_vma (vma, addr, old_len, new_len, new_addr, locked, flags, uf, uf_unmap); -if (!(offset_in_page (ret))) - goto out; - -out1: -vm_unacct_memory (charged); - out:return ret; }@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;bool locked = false;bool downgraded = false;struct vm_userfaultfd_ctx uf =NULL_VM_UFFD_CTX ;@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,/* * Ok, we need to grow.. */ - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,if (old_len == vma->vm_end - addr) {/* can we just expand the current mapping? */ if (vma_expandable (vma, new_len - old_len)) { -int pages = (new_len - old_len) >>PAGE_SHIFT ; +long pages = (new_len - old_len) >>PAGE_SHIFT ; + +if (vma->vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, pages)) { + ret = -ENOMEM ; + goto out; + } + }if (vma_adjust (vma, vma->vm_start, addr + new_len, vma->vm_pgoff,NULL )) { +vm_unacct_memory (pages); ret = -ENOMEM ; goto out; }@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, &locked, flags, &uf, &uf_unmap); } out: -if (offset_in_page (ret)) { -vm_unacct_memory (charged); +if (offset_in_page (ret)) locked = false; - }if (downgraded)mmap_read_unlock (current->mm);else
Signature · 1 line
Dima
Andrew Morton
maintainerQuoted 11 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines
Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]The patch by Chen Wandun still makes sense with this v2. Heh :-)
Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?
Dmitry Safonov
Acked-byQuoted 11 lines · Andrew Morton
Quoted 9 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines
Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]
Quoted 7 lines
Quoted 2 lines
The patch by Chen Wandun still makes sense with this v2. Heh :-)
Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?
Yes, please keep that patch with
This one comes on the top to correct accounting for MREMAP_DONTUNMAP.
Thanks,
Dmitry
Quoted 20 lines · Dmitry Safonov
Quoted 11 lines · Andrew Morton
Quoted 9 lines · On Wed, 21 Jul 2021 14:21:54 +0100 Dmitry Safonov <[email protected]> wrote:
Quoted 8 lines
Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]Quoted 7 lines
Quoted 2 lines
The patch by Chen Wandun still makes sense with this v2. Heh :-)
Should
https://lkml.kernel.org/r/[email protected]
still be applied then? Did you ack it?
Quoted 4 lines
Yes, please keep that patch with
Acked-by Dmitry Safonov <[email protected]>This one comes on the top to correct accounting for
MREMAP_DONTUNMAP.
Please, also wait for reviews for this one.
I think it's correct and simplifies the complex code, but now I look into the code again, I see that:
: /*
: * MREMAP_DONTUNMAP is always a move and it does not allow resizing
: * in the process.
: */
: if (flags & MREMAP_DONTUNMAP &&
: (!(flags & MREMAP_MAYMOVE) || old_len != new_len))
: return ret;
So, this part can be dropped:
Quoted 1 line
Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")
But I think the patch still makes sense.
Brian, can I have your review on it?
Thanks,
Dmitry
Quoted 175 lines · Dmitry Safonov
All this
vm_unacct_memory(charged)dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes inmremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growingMREMAP_DONTUNMAPmremap()s inmove_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]Cc Alexander Viro <[email protected]>Cc Andrew Morton <[email protected]>Cc Andy Lutomirski <[email protected]>Cc Brian Geffon <[email protected]>Cc Catalin Marinas <[email protected]>Cc Chen Wandun <[email protected]>Cc Dan Carpenter <[email protected]>Cc Dan Williams <[email protected]>Cc Dave Jiang <[email protected]>Cc Hugh Dickins <[email protected]>Cc Ingo Molnar <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc John Hubbard <[email protected]>Cc Kefeng Wang <[email protected]>Cc "Kirill A. Shutemov" <[email protected]>Cc Mike Kravetz <[email protected]>Cc Minchan Kim <[email protected]>Cc Ralph Campbell <[email protected]>Cc Russell King <[email protected]>Cc Thomas Bogendoerfer <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Vishal Verma <[email protected]>Cc Vlastimil Babka <[email protected]>Cc Wei Yongjun <[email protected]>Cc Will Deacon <[email protected]>Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")Signed-off-by Dmitry Safonov <[email protected]>--- mm/mremap.c | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-)mm/mremap.c +12
-24@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, bool *locked, unsigned long flags, struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) { + long to_account = new_len - old_len;struct mm_struct *mm = vma->vm_mm;struct vm_area_struct *new_vma;unsigned long vm_flags = vma->vm_flags;@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (mm->map_count >= sysctl_max_map_count -3 )return -ENOMEM ; +if (unlikely (flags &MREMAP_DONTUNMAP )) + to_account = new_len; +if (vma->vm_ops && vma->vm_ops->may_split) {if (vma->vm_start != old_addr) err = vma->vm_ops->may_split (vma, old_addr);@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (err)return err; -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) { -if (security_vm_enough_memory_mm (mm, new_len >>PAGE_SHIFT )) +if (vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, to_account >>PAGE_SHIFT ))return -ENOMEM ; }@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, new_vma =copy_vma (&vma, new_addr, new_len, new_pgoff, &need_rmap_locks);if (!new_vma) { -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) -vm_unacct_memory (new_len >>PAGE_SHIFT ); +if (vm_flags &VM_ACCOUNT ) +vm_unacct_memory (to_account >>PAGE_SHIFT );return -ENOMEM ; }@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, }static struct vm_area_struct *vma_to_resize (unsigned long addr, -unsigned long old_len,unsigned long new_len,unsigned long flags, -unsigned long *p) +unsigned long old_len,unsigned long new_len,unsigned long flags) {struct mm_struct *mm = current->mm;struct vm_area_struct *vma;@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, (new_len - old_len) >> PAGE_SHIFT))return ERR_PTR (-ENOMEM ); -if (vma->vm_flags &VM_ACCOUNT ) { -unsigned long charged = (new_len - old_len) >>PAGE_SHIFT ; -if (security_vm_enough_memory_mm (mm, charged)) -return ERR_PTR (-ENOMEM ); - *p = charged; - } -return vma; }@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;unsigned long map_flags =0 ;if (offset_in_page (new_addr))@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, old_len = new_len; } - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ((addr - vma->vm_start) >>PAGE_SHIFT ), map_flags);if (IS_ERR_VALUE (ret)) - goto out1; + goto out;/* We got a new mapping */ if (!(flags &MREMAP_FIXED ))@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ret =move_vma (vma, addr, old_len, new_len, new_addr, locked, flags, uf, uf_unmap); -if (!(offset_in_page (ret))) - goto out; - -out1: -vm_unacct_memory (charged); - out:return ret; }@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;bool locked = false;bool downgraded = false;struct vm_userfaultfd_ctx uf =NULL_VM_UFFD_CTX ;@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,/* * Ok, we need to grow.. */ - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,if (old_len == vma->vm_end - addr) {/* can we just expand the current mapping? */
mm/mremap.c SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · line 990
Quoted 10 lines
mm/mremap.c +8
-1if (vma_expandable (vma, new_len - old_len)) { -int pages = (new_len - old_len) >>PAGE_SHIFT ; +long pages = (new_len - old_len) >>PAGE_SHIFT ; + +if (vma->vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, pages)) { + ret = -ENOMEM ; + goto out; + } + }
Wasn't this whole check already performed in vma_to_resize()?
Quoted 22 lines
mm/mremap.c +2
-4if (vma_adjust (vma, vma->vm_start, addr + new_len, vma->vm_pgoff,NULL )) { +vm_unacct_memory (pages); ret = -ENOMEM ; goto out; }@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, &locked, flags, &uf, &uf_unmap); } out: -if (offset_in_page (ret)) { -vm_unacct_memory (charged); +if (offset_in_page (ret)) locked = false; - }if (downgraded)mmap_read_unlock (current->mm);else --2.32.0
Brian Geffon
Acked-byWhoops I missed that you did remove it from vma_to_resize() so then it looks good to me.
Quoted 212 lines · Brian Geffon
Quoted 175 lines · Dmitry Safonov
All this
vm_unacct_memory(charged)dance seems to complicate the life without a good reason. Furthermore, it seems not always done right on error-pathes inmremap_to().
And worse than that: this `charged' difference is sometimes double-accounted for growingMREMAP_DONTUNMAPmremap()s inmove_vma():
: if (security_vm_enough_memory_mm(mm, new_len >> PAGE_SHIFT))Let's not do this.
Account memory inmremap()fast-path for growing VMAs or inmove_vma()for actually moving things. The same simpler way as it's done byvm_stat_account(), but with a difference to callsecurity_vm_enough_memory_mm()before copying/adjusting VMA.Originally noticed by Chen Wandun:
https://lkml.kernel.org/r/[email protected]Cc Alexander Viro <[email protected]>Cc Andrew Morton <[email protected]>Cc Andy Lutomirski <[email protected]>Cc Brian Geffon <[email protected]>Cc Catalin Marinas <[email protected]>Cc Chen Wandun <[email protected]>Cc Dan Carpenter <[email protected]>Cc Dan Williams <[email protected]>Cc Dave Jiang <[email protected]>Cc Hugh Dickins <[email protected]>Cc Ingo Molnar <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc John Hubbard <[email protected]>Cc Kefeng Wang <[email protected]>Cc "Kirill A. Shutemov" <[email protected]>Cc Mike Kravetz <[email protected]>Cc Minchan Kim <[email protected]>Cc Ralph Campbell <[email protected]>Cc Russell King <[email protected]>Cc Thomas Bogendoerfer <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Vishal Verma <[email protected]>Cc Vlastimil Babka <[email protected]>Cc Wei Yongjun <[email protected]>Cc Will Deacon <[email protected]>Fixes e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()")Signed-off-by Dmitry Safonov <[email protected]>--- mm/mremap.c | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-)mm/mremap.c +12
-24@@ -565,6 +565,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, bool *locked, unsigned long flags, struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) { + long to_account = new_len - old_len;struct mm_struct *mm = vma->vm_mm;struct vm_area_struct *new_vma;unsigned long vm_flags = vma->vm_flags;@@ -583,6 +584,9 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (mm->map_count >= sysctl_max_map_count -3 )return -ENOMEM ; +if (unlikely (flags &MREMAP_DONTUNMAP )) + to_account = new_len; +if (vma->vm_ops && vma->vm_ops->may_split) {if (vma->vm_start != old_addr) err = vma->vm_ops->may_split (vma, old_addr);@@ -604,8 +608,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,if (err)return err; -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) { -if (security_vm_enough_memory_mm (mm, new_len >>PAGE_SHIFT )) +if (vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, to_account >>PAGE_SHIFT ))return -ENOMEM ; }@@ -613,8 +617,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, new_vma =copy_vma (&vma, new_addr, new_len, new_pgoff, &need_rmap_locks);if (!new_vma) { -if (unlikely (flags &MREMAP_DONTUNMAP && vm_flags &VM_ACCOUNT )) -vm_unacct_memory (new_len >>PAGE_SHIFT ); +if (vm_flags &VM_ACCOUNT ) +vm_unacct_memory (to_account >>PAGE_SHIFT );return -ENOMEM ; }@@ -708,8 +712,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, }static struct vm_area_struct *vma_to_resize (unsigned long addr, -unsigned long old_len,unsigned long new_len,unsigned long flags, -unsigned long *p) +unsigned long old_len,unsigned long new_len,unsigned long flags) {struct mm_struct *mm = current->mm;struct vm_area_struct *vma;@@ -768,13 +771,6 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, (new_len - old_len) >> PAGE_SHIFT))return ERR_PTR (-ENOMEM ); -if (vma->vm_flags &VM_ACCOUNT ) { -unsigned long charged = (new_len - old_len) >>PAGE_SHIFT ; -if (security_vm_enough_memory_mm (mm, charged)) -return ERR_PTR (-ENOMEM ); - *p = charged; - } -return vma; }@@ -787,7 +783,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;unsigned long map_flags =0 ;if (offset_in_page (new_addr))@@ -830,7 +825,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, old_len = new_len; } - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -853,7 +848,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ((addr - vma->vm_start) >>PAGE_SHIFT ), map_flags);if (IS_ERR_VALUE (ret)) - goto out1; + goto out;/* We got a new mapping */ if (!(flags &MREMAP_FIXED ))@@ -862,12 +857,6 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, ret =move_vma (vma, addr, old_len, new_len, new_addr, locked, flags, uf, uf_unmap); -if (!(offset_in_page (ret))) - goto out; - -out1: -vm_unacct_memory (charged); - out:return ret; }@@ -899,7 +888,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,struct mm_struct *mm = current->mm;struct vm_area_struct *vma;unsigned long ret = -EINVAL ; -unsigned long charged =0 ;bool locked = false;bool downgraded = false;struct vm_userfaultfd_ctx uf =NULL_VM_UFFD_CTX ;@@ -981,7 +969,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,/* * Ok, we need to grow.. */ - vma =vma_to_resize (addr, old_len, new_len, flags, &charged); + vma =vma_to_resize (addr, old_len, new_len, flags);if (IS_ERR (vma)) { ret =PTR_ERR (vma); goto out;@@ -992,10 +980,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,if (old_len == vma->vm_end - addr) {/* can we just expand the current mapping? */ mm/mremap.c SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · line 990
Quoted 10 lines
mm/mremap.c +8
-1if (vma_expandable (vma, new_len - old_len)) { -int pages = (new_len - old_len) >>PAGE_SHIFT ; +long pages = (new_len - old_len) >>PAGE_SHIFT ; + +if (vma->vm_flags &VM_ACCOUNT ) { +if (security_vm_enough_memory_mm (mm, pages)) { + ret = -ENOMEM ; + goto out; + } + }Wasn't this whole check already performed in
vma_to_resize()?Quoted 22 lines
mm/mremap.c +2
-4if (vma_adjust (vma, vma->vm_start, addr + new_len, vma->vm_pgoff,NULL )) { +vm_unacct_memory (pages); ret = -ENOMEM ; goto out; }@@ -1034,10 +1030,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, &locked, flags, &uf, &uf_unmap); } out: -if (offset_in_page (ret)) { -vm_unacct_memory (charged); +if (offset_in_page (ret)) locked = false; - }if (downgraded)mmap_read_unlock (current->mm);else --2.32.0