move all VMA allocation, freeing and duplication logic to mm
Subsystems (6)
v2 cover move all VMA allocation, freeing and duplication logic to mm
Currently VMA allocation, freeing and duplication exist in kernel/fork.c, which is a violation of separation of concerns, and leaves these functions exposed to the rest of the kernel when they are in fact internal implementation details.
Resolve this by moving this logic to mm, and making it internal to vma.c, vma.h.
This also allows us, in future, to provide userland testing around this functionality.
We additionally abstract dup_mmap() to mm, being careful to ensure kernel/fork.c acceses this via the mm internal header so it is not exposed elsewhere in the kernel.
As part of this change, also abstract initial stack allocation performed in __bprm_mm_init() out of fs code into mm via the create_init_stack_vma(), as this code uses vm_area_alloc() and vm_area_free().
In order to allow for this, we must add code shared between nommu and mmu-enabled configurations in order to share VMA allocation, freeing and duplication code correctly while also keeping these functions available in userland VMA testing.
This is achieved by adding a vma_init.c file which is also compiled by the userland tests.
v2:
* Moved vma init, alloc, free, dup functions to newly created vma_init.c function as per Suren, Liam.
* Added MAINTAINERS entry for vma_init.c, added to Makefile.
* Updated mmap_init() comment.
* Propagated tags (thanks everyone!)
* Added detach_free_vma() helper and correctly detached vmas in userland VMA test code.
* Updated userland test code to also compile the vma_init.c file.
* Corrected create_init_stack_vma() comment as per Suren.
* Updated commit message as per Suren.
v1:
https://lore.kernel.org/all/[email protected]/
Lorenzo Stoakes (3):
mm: abstract initial stack setup to mm subsystem
mm: move dup_mmap() to mm
mm: perform VMA allocation, freeing, duplication in mm
MAINTAINERS | 1 + fs/exec.c | 51 +----- include/linux/mm.h | 2 + kernel/fork.c | 277 +------------------------------ mm/Makefile | 2 +- mm/internal.h | 2 + mm/mmap.c | 253 +++++++++++++++++++++++++++- mm/nommu.c | 12 +- mm/vma.h | 6 + mm/vma_init.c | 101 +++++++++++ tools/testing/vma/Makefile | 2 +- tools/testing/vma/vma.c | 26 ++- tools/testing/vma/vma_internal.h | 143 +++++++++++++--- 13 files changed, 511 insertions(+), 367 deletions(-) create mode 100644 mm/vma_init.c
--
2.49.0
PATCH v2 1/3 mm: abstract initial stack setup to mm subsystem
There are peculiarities within the kernel where what is very clearly mm code is performed elsewhere arbitrarily.
This violates separation of concerns and makes it harder to refactor code to make changes to how fundamental initialisation and operation of mm logic is performed.
One such case is the creation of the VMA containing the initial stack upon execve()'ing a new process. This is currently performed in __bprm_mm_init() in fs/exec.c.
Abstract this operation to create_init_stack_vma(). This allows us to limit use of vma allocation and free code to fork and mm only.
We previously did the same for the step at which we relocate the initial stack VMA downwards via relocate_vma_down(), now we move the initial VMA establishment too.
--- fs/exec.c | 51 +--------------------------------- include/linux/mm.h | 2 ++ mm/mmap.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 50 deletions(-)
fs/exec.c +1 -50
@@ -244,56 +244,7 @@ static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos,static int __bprm_mm_init (struct linux_binprm *bprm) { -int err; -struct vm_area_struct *vma =NULL ; -struct mm_struct *mm = bprm->mm; - - bprm->vma = vma =vm_area_alloc (mm); -if (!vma) -return -ENOMEM ; -vma_set_anonymous (vma); - -if (mmap_write_lock_killable (mm)) { - err = -EINTR ; - goto err_free; - } - -/* -* Need to be called with mmap write lock -* held, to avoid race with ksmd. -*/ - err =ksm_execve (mm); -if (err) - goto err_ksm; - -/* -* Place the stack at the largest stack address the architecture -* supports. Later, we'll move this to an appropriate place. We don't -* use STACK_TOP because that can depend on attributes which aren't -* configured yet. -*/ -BUILD_BUG_ON (VM_STACK_FLAGS &VM_STACK_INCOMPLETE_SETUP ); - vma->vm_end =STACK_TOP_MAX ; - vma->vm_start = vma->vm_end -PAGE_SIZE ; -vm_flags_init (vma,VM_SOFTDIRTY |VM_STACK_FLAGS |VM_STACK_INCOMPLETE_SETUP ); - vma->vm_page_prot =vm_get_page_prot (vma->vm_flags); - - err =insert_vm_struct (mm, vma); -if (err) - goto err; - - mm->stack_vm = mm->total_vm =1 ; -mmap_write_unlock (mm); - bprm->p = vma->vm_end -sizeof (void *); -return 0 ; -err: -ksm_exit (mm); -err_ksm: -mmap_write_unlock (mm); -err_free: - bprm->vma =NULL ; -vm_area_free (vma); -return err; +return create_init_stack_vma (bprm->mm, &bprm->vma, &bprm->p); }static bool valid_arg_len (struct linux_binprm *bprm,long len)
include/linux/mm.h +2 -0
@@ -3223,6 +3223,8 @@ void anon_vma_interval_tree_verify(struct anon_vma_chain *node);extern int __vm_enough_memory (struct mm_struct *mm,long pages,int cap_sys_admin);extern int insert_vm_struct (struct mm_struct *,struct vm_area_struct *);extern void exit_mmap (struct mm_struct *); +int create_init_stack_vma (struct mm_struct *mm,struct vm_area_struct **vmap, +unsigned long *top_mem_p);int relocate_vma_down (struct vm_area_struct *vma,unsigned long shift);bool mmap_read_lock_maybe_expand (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,bool write);
mm/mmap.c +69 -0
75 lines not on this page · raw message
Signature · 1 line
2.49.0
Kees Cook
maintainerRe: [PATCH v2 1/3] mm: abstract initial stack setup to mm subsystem
Quoted 18 lines · Lorenzo Stoakes
There are peculiarities within the kernel where what is very clearly mm code is performed elsewhere arbitrarily.
This violates separation of concerns and makes it harder to refactor code to make changes to how fundamental initialisation and operation of mm logic is performed.
One such case is the creation of the VMA containing the initial stack upon execve()'ing a new process. This is currently performed in
__bprm_mm_init()infs/exec.c.Abstract this operation to
create_init_stack_vma(). This allows us to limit use of vma allocation and free code to fork and mm only.We previously did the same for the step at which we relocate the initial stack VMA downwards via
relocate_vma_down(), now we move the initial VMA establishment too.
Quoted 5 lines
Signed-off-by Lorenzo Stoakes <[email protected]>Reviewed-by Suren Baghdasaryan <[email protected]>--- fs/exec.c | 51 +---------------------------------
I'm kind of on the fence about this. On the one hand, yes, it's all vma goo, and should live with the rest of vma code, as you suggest. On the other had, exec is the only consumer of this behavior, and moving it out of fs/exec.c means that changes to the code that specifically only impacts exec are now in a separate file, and will no longer get exec maintainer/reviewer CCs (based on MAINTAINERS file matching). Exec is notoriously fragile, so I'm kind of generally paranoid about changes to its behaviors going unnoticed.
In defense of moving it, yes, this routine has gotten updates over the many years, but it's relatively stable. But at least one thing has gone in without exec maintainer review recently (I would have Acked it, but the point is review): 9e567ca45f ("mm/ksm: fix ksm exec support for prctl") Everything else was before I took on the role officially (Nov 2022).
So I guess I'm asking, how do we make sure stuff pulled out of exec still gets exec maintainer review?
Review
Quoted 8 lines
[...]
static int__bprm_mm_init(struct linux_binprm *bprm){(quoted diff) +1
-2- int err; - return err; + return create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p); }
I'd prefer __bprm_mm_init() go away if it's just a 1:1 wrapper now. However, it doesn't really look like it makes too much sense for the NOMMU logic get moved as well, since it explicitly depends on exec-specific values (MAX_ARG_PAGES), so perhaps something like this:
fs/exec.c +5 -1
@@ -382,9 +382,13 @@ static int bprm_mm_init(struct linux_binprm *bprm) bprm->rlim_stack = current->signal->rlim[RLIMIT_STACK ];task_unlock (current->group_leader); - err =__bprm_mm_init (bprm); +#ifndef CONFIG_MMU + bprm->p =PAGE_SIZE *MAX_ARG_PAGES -sizeof (void *); +#else + err =create_init_stack_vma (bprm->mm, &bprm->vma, &bprm->p);if (err) goto err; +#endif return 0 ;
On a related note, I'd like to point out that my claim that exec is the only consumer here, is slightly a lie. Technically this is correct, but only because this is specifically setting up the _stack_.
The rest of the VMA setup actually surrounds this code (another reason I remain unhappy about moving it). Specifically the mm_alloc() before __bprm_mm_init (which is reached through alloc_brpm()). And then, following alloc_bprm() in do_execveat_common(), is the call to setup_new_exec(), which does the rest of the VMA setup, specifically arch_pick_mmap_layout() and related fiddling.
The "create userspace VMA" logic, mostly through mm_alloc(), is used in a few places (e.g. text poking), but the "bring up a _usable_ userspace VMA" logic (i.e. one also with functional mmap) is repeated in lib/kunit/alloc_user.c for allowing testing of code that touches userspace (see kunit_attach_mm() and the kunit_vm_mmap() users). (But these tests don't actually run userspace code, so no stack is set up.)
I guess what I'm trying to say is that I think we need a more clearly defined "create usable userspace VMA" API, as we've got at least 3 scattered approaches right now: exec ("everything"), non-mmap-non-stack users (text poking, et al), and mmap-but-not-stack users (kunit tests).
And the One True User of a full userspace VMA, exec, has the full setup scattered into several phases, mostly due to needing to separate those phases because it needs to progressively gather the information needed to correctly configure each piece:
- set up userspace VMA at all (mm_alloc)
- set up a stack because exec args need to go somewhere (__bprm_mm_init)
- move stack to the right place (depends on executable binary and task bits)
- set up mmap (arch_pick_mmap_layout) to actually load executable binary (depends on arch, binary, and task bits)
Hopefully this all explains why I'm uncomfortable to see __bprm_mm_init get relocated. It'll _probably_ be fine, but I get antsy about changes to code that only exec uses...
Signature · 1 line
Kees Cook
Re: [PATCH v2 1/3] mm: abstract initial stack setup to mm subsystem
Quoted 34 lines · Kees Cook
Quoted 18 lines · Lorenzo Stoakes
There are peculiarities within the kernel where what is very clearly mm code is performed elsewhere arbitrarily.
This violates separation of concerns and makes it harder to refactor code to make changes to how fundamental initialisation and operation of mm logic is performed.
One such case is the creation of the VMA containing the initial stack upon execve()'ing a new process. This is currently performed in
__bprm_mm_init()infs/exec.c.Abstract this operation to
create_init_stack_vma(). This allows us to limit use of vma allocation and free code to fork and mm only.We previously did the same for the step at which we relocate the initial stack VMA downwards via
relocate_vma_down(), now we move the initial VMA establishment too.Quoted 5 lines
Signed-off-by Lorenzo Stoakes <[email protected]>Reviewed-by Suren Baghdasaryan <[email protected]>--- fs/exec.c | 51 +---------------------------------I'm kind of on the fence about this. On the one hand, yes, it's all vma goo, and should live with the rest of vma code, as you suggest. On the other had, exec is the only consumer of this behavior, and moving it out of
fs/exec.cmeans that changes to the code that specifically only impacts exec are now in a separate file, and will no longer get exec maintainer/reviewer CCs (based onMAINTAINERSfile matching). Exec is notoriously fragile, so I'm kind of generally paranoid about changes to its behaviors going unnoticed.
Quoted 8 lines
In defense of moving it, yes, this routine has gotten updates over the many years, but it's relatively stable. But at least one thing has gone in without exec maintainer review recently (I would have Acked it, but the point is review):
9e567ca45f("mm/ksm: fix ksm exec support for prctl") Everything else was before I took on the role officially (Nov 2022).So I guess I'm asking, how do we make sure stuff pulled out of exec still gets exec maintainer review?
I think we have two options here:
1. Separate out this code into mm/vma_exec.c and treat it like mm/vma_init.c, then add you as a reviewer, so you have visibility on everything that happens there.
2. Add you as a reviewer to memory mapping in general.
I think 1 is preferable actually, as it'll reduce noise for you and then you'll _always_ get notified about change in this code.
Note that we have done this previously for similar reasons with relocate_vma_down() we could move this function into that file.
For the sake of saving time given time zone differences, let me explore option 1in a v3, and obviously if that doesn't work for you let me know! :)
We'll have to then have fs/exec.c include ../mm/vma_exec.h, so it is _only_ exec that gets access to this.
Quoted 23 lines
Review
Quoted 9 lines
[...]
static int__bprm_mm_init(struct linux_binprm *bprm){(quoted diff) +1
-2- int err; - return err; + return create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p); }I'd prefer
__bprm_mm_init()go away if it's just a 1:1 wrapper now. However, it doesn't really look like it makes too much sense for the NOMMU logic get moved as well, since it explicitly depends on exec-specific values (MAX_ARG_PAGES), so perhaps something like this:fs/exec.c +0
-0@@ -382,9 +382,13 @@ static int bprm_mm_init(struct linux_binprm *bprm) bprm->rlim_stack = current->signal->rlim[RLIMIT_STACK ];task_unlock (current->group_leader);
fs/exec.c static int bprm_mm_init(struct linux_binprm *bprm) · line 393
Quoted 10 lines
fs/exec.c +5
-1- err =__bprm_mm_init (bprm); +#ifndef CONFIG_MMU + bprm->p =PAGE_SIZE *MAX_ARG_PAGES -sizeof (void *); +#else + err =create_init_stack_vma (bprm->mm, &bprm->vma, &bprm->p);if (err) goto err; +#endif return 0 ;
Sure I can respin with this.
Quoted 7 lines
On a related note, I'd like to point out that my claim that exec is the only consumer here, is slightly a lie. Technically this is correct, but only because this is specifically setting up the _stack_.
Quoted 6 lines
The rest of the VMA setup actually surrounds this code (another reason I remain unhappy about moving it). Specifically the
mm_alloc()before__bprm_mm_init(which is reached throughalloc_brpm()). And then, followingalloc_bprm()indo_execveat_common(), is the call tosetup_new_exec(), which does the rest of the VMA setup, specificallyarch_pick_mmap_layout()and related fiddling.
No other callers try to allocate/free vmas, which is the issue here.
Quoted 7 lines
The "create userspace VMA" logic, mostly through
mm_alloc(), is used in a few places (e.g. text poking), but the "bring up a _usable_ userspace VMA" logic (i.e. one also with functional mmap) is repeated inlib/kunit/alloc_user.cfor allowing testing of code that touches userspace (seekunit_attach_mm()and thekunit_vm_mmap()users). (But these tests don't actually run userspace code, so no stack is set up.)
Hm but this seems something different? This is using the mm_alloc() interface?
Quoted 5 lines
I guess what I'm trying to say is that I think we need a more clearly defined "create usable userspace VMA" API, as we've got at least 3 scattered approaches right now: exec ("everything"), non-mmap-non-stack users (text poking, et al), and mmap-but-not-stack users (kunit tests).
But only exec is actually allocating and essentially 'forcing in' a stack vma like this?
I mean I'm all for having some nicely abstracted interface rather than scattering open-coded stuff around, but the one and only objective _here_ is to disallow the use of very clearly mm-specific internal APIs elsewhere.
exec is of course 'special' in this stack behaviour, but I feel we can express this 'specialness' better by having this kind of well-defined, well-described functions exposed by mm, rather than handing over absolutely fundamental API calls to any part of the kernel that wants them.
Quoted 11 lines
And the One True User of a full userspace VMA, exec, has the full setup scattered into several phases, mostly due to needing to separate those phases because it needs to progressively gather the information needed to correctly configure each piece:
- set up userspace VMA at all (mm_alloc)
- set up a stack because exec args need to go somewhere (__bprm_mm_init)
- move stack to the right place (depends on executable binary and task bits)
- set up mmap (arch_pick_mmap_layout) to actually load executable binary (depends on arch, binary, and task bits)
Quoted 3 lines
Hopefully this all explains why I'm uncomfortable to see
__bprm_mm_initget relocated. It'll _probably_ be fine, but I get antsy about changes to code that only exec uses...
Right, I understand and appreciate that, for sure. This is fiddly core code, you worry about things breaking - I mean I feel you (don't ask me about brk(), please please don't ask me about brk() :P)
So hopefully the proposed solution with vma_exec.c should cover this off nicely?
Quoted 3 lines
--
Kees Cook
Cheers, Lorenzo
Re: [PATCH v2 1/3] mm: abstract initial stack setup to mm subsystem
Quoted 41 lines · Lorenzo Stoakes
Quoted 40 lines · Kees Cook
Quoted 18 lines · Lorenzo Stoakes
There are peculiarities within the kernel where what is very clearly mm code is performed elsewhere arbitrarily.
This violates separation of concerns and makes it harder to refactor code to make changes to how fundamental initialisation and operation of mm logic is performed.
One such case is the creation of the VMA containing the initial stack upon execve()'ing a new process. This is currently performed in
__bprm_mm_init()infs/exec.c.Abstract this operation to
create_init_stack_vma(). This allows us to limit use of vma allocation and free code to fork and mm only.We previously did the same for the step at which we relocate the initial stack VMA downwards via
relocate_vma_down(), now we move the initial VMA establishment too.Quoted 5 lines
Signed-off-by Lorenzo Stoakes <[email protected]>Reviewed-by Suren Baghdasaryan <[email protected]>--- fs/exec.c | 51 +---------------------------------I'm kind of on the fence about this. On the one hand, yes, it's all vma goo, and should live with the rest of vma code, as you suggest. On the other had, exec is the only consumer of this behavior, and moving it out of
fs/exec.cmeans that changes to the code that specifically only impacts exec are now in a separate file, and will no longer get exec maintainer/reviewer CCs (based onMAINTAINERSfile matching). Exec is notoriously fragile, so I'm kind of generally paranoid about changes to its behaviors going unnoticed.In defense of moving it, yes, this routine has gotten updates over the many years, but it's relatively stable. But at least one thing has gone in without exec maintainer review recently (I would have Acked it, but the point is review):
9e567ca45f("mm/ksm: fix ksm exec support for prctl") Everything else was before I took on the role officially (Nov 2022).
Quoted 9 lines
Quoted 2 lines
So I guess I'm asking, how do we make sure stuff pulled out of exec still gets exec maintainer review?
I think we have two options here:
1. Separate out this code into
mm/vma_exec.cand treat it likemm/vma_init.c, then add you as a reviewer, so you have visibility on everything that happens there.
Actually, (off-list) Vlastimil made the very good suggestion that we can just add this new file to both exec and memory mapping sections, have tested it and it works!
So I think this should cover off your concerns?
[snip]
Kees Cook
maintainerRe: [PATCH v2 1/3] mm: abstract initial stack setup to mm subsystem
Quoted 52 lines · Lorenzo Stoakes
Quoted 41 lines · Lorenzo Stoakes
Quoted 40 lines · Kees Cook
Quoted 18 lines · Lorenzo Stoakes
There are peculiarities within the kernel where what is very clearly mm code is performed elsewhere arbitrarily.
This violates separation of concerns and makes it harder to refactor code to make changes to how fundamental initialisation and operation of mm logic is performed.
One such case is the creation of the VMA containing the initial stack upon execve()'ing a new process. This is currently performed in
__bprm_mm_init()infs/exec.c.Abstract this operation to
create_init_stack_vma(). This allows us to limit use of vma allocation and free code to fork and mm only.We previously did the same for the step at which we relocate the initial stack VMA downwards via
relocate_vma_down(), now we move the initial VMA establishment too.Quoted 5 lines
Signed-off-by Lorenzo Stoakes <[email protected]>Reviewed-by Suren Baghdasaryan <[email protected]>--- fs/exec.c | 51 +---------------------------------I'm kind of on the fence about this. On the one hand, yes, it's all vma goo, and should live with the rest of vma code, as you suggest. On the other had, exec is the only consumer of this behavior, and moving it out of
fs/exec.cmeans that changes to the code that specifically only impacts exec are now in a separate file, and will no longer get exec maintainer/reviewer CCs (based onMAINTAINERSfile matching). Exec is notoriously fragile, so I'm kind of generally paranoid about changes to its behaviors going unnoticed.In defense of moving it, yes, this routine has gotten updates over the many years, but it's relatively stable. But at least one thing has gone in without exec maintainer review recently (I would have Acked it, but the point is review):
9e567ca45f("mm/ksm: fix ksm exec support for prctl") Everything else was before I took on the role officially (Nov 2022).Quoted 9 lines
Quoted 2 lines
So I guess I'm asking, how do we make sure stuff pulled out of exec still gets exec maintainer review?
I think we have two options here:
1. Separate out this code into
mm/vma_exec.cand treat it likemm/vma_init.c, then add you as a reviewer, so you have visibility on everything that happens there.
Quoted 7 lines
Actually, (off-list) Vlastimil made the very good suggestion that we can just add this new file to both exec and memory mapping sections, have tested it and it works!
So I think this should cover off your concerns?
[snip]
Yes, this is brilliant! Totally works for me; thank you (and Vlastimil) for finding a good way to do it.
-Kees
Signature · 1 line
Kees Cook
PATCH v2 2/3 mm: move dup_mmap() to mm
In reply to Lorenzo StoakesThis is a key step in our being able to abstract and isolate VMA allocation and destruction logic.
This function is the last one where vm_area_free() and vm_area_dup() are directly referenced outside of mmap, so having this in mm allows us to isolate these.
We do the same for the nommu version which is substantially simpler.
We place the declaration for dup_mmap() in mm/internal.h and have kernel/fork.c import this in order to prevent improper use of this functionality elsewhere in the kernel.
While we're here, we remove the useless #ifdef CONFIG_MMU check around mmap_read_lock_maybe_expand() in mmap.c, mmap.c is compiled only if CONFIG_MMU is set.
--- kernel/fork.c | 189 ++------------------------------------------------ mm/internal.h | 2 + mm/mmap.c | 181 +++++++++++++++++++++++++++++++++++++++++++++-- mm/nommu.c | 8 +++ 4 files changed, 189 insertions(+), 191 deletions(-)
kernel/fork.c +4 -185
@@ -112,6 +112,9 @@#include <asm/cacheflush.h> #include <asm/tlbflush.h> +/* For dup_mmap(). */ +#include "../mm/internal.h" +#include <trace/events/sched.h> #define CREATE_TRACE_POINTS
@@ -589,7 +592,7 @@ void free_task(struct task_struct *tsk) }EXPORT_SYMBOL (free_task); -static void dup_mm_exe_file (struct mm_struct *mm,struct mm_struct *oldmm) +void dup_mm_exe_file (struct mm_struct *mm,struct mm_struct *oldmm) {struct file *exe_file;
@@ -604,183 +607,6 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) }#ifdef CONFIG_MMU -static __latent_entropy int dup_mmap(struct mm_struct *mm, -struct mm_struct *oldmm) -{ -struct vm_area_struct *mpnt, *tmp; -int retval; -unsigned long charge =0 ; -LIST_HEAD (uf); -VMA_ITERATOR (vmi, mm,0 ); - -if (mmap_write_lock_killable (oldmm)) -return -EINTR ; -flush_cache_dup_mm (oldmm); -uprobe_dup_mmap (oldmm, mm); -/* -* Not linked in yet - no deadlock potential: -*/ -mmap_write_lock_nested (mm,SINGLE_DEPTH_NESTING ); - -/* No ordering required: file already has been exposed. */ -dup_mm_exe_file (mm, oldmm); - - mm->total_vm = oldmm->total_vm; - mm->data_vm = oldmm->data_vm; - mm->exec_vm = oldmm->exec_vm; - mm->stack_vm = oldmm->stack_vm; - -/* Use __mt_dup() to efficiently build an identical maple tree. */ - retval =__mt_dup (&oldmm->mm_mt, &mm->mm_mt,GFP_KERNEL ); -if (unlikely (retval)) - goto out; - -mt_clear_in_rcu (vmi.mas.tree); -for_each_vma (vmi, mpnt) { -struct file *file; - -vma_start_write (mpnt); -if (mpnt->vm_flags &VM_DONTCOPY ) { - retval =vma_iter_clear_gfp (&vmi, mpnt->vm_start, - mpnt->vm_end,GFP_KERNEL ); -if (retval) - goto loop_out; - -vm_stat_account (mm, mpnt->vm_flags, -vma_pages (mpnt)); -continue ; - } - charge =0 ; -/* -* Don't duplicate many vmas if we've been oom-killed (for -* example) -*/ -if (fatal_signal_pending (current)) { - retval = -EINTR ; - goto loop_out; - } -if (mpnt->vm_flags &VM_ACCOUNT ) { -unsigned long len =vma_pages (mpnt); - -if (security_vm_enough_memory_mm (oldmm, len))/* sic */ - goto fail_nomem; - charge = len; - } - tmp =vm_area_dup (mpnt); -if (!tmp) - goto fail_nomem; - -/* track_pfn_copy() will later take care of copying internal state. */ -if (unlikely (tmp->vm_flags &VM_PFNMAP )) -untrack_pfn_clear (tmp); - - retval =vma_dup_policy (mpnt, tmp); -if (retval) - goto fail_nomem_policy; - tmp->vm_mm = mm; - retval =dup_userfaultfd (tmp, &uf); -if (retval) - goto fail_nomem_anon_vma_fork; -if (tmp->vm_flags &VM_WIPEONFORK ) { -/* -* VM_WIPEONFORK gets a clean slate in the child. -* Don't prepare anon_vma until fault since we don't -* copy page for current vma. -*/ - tmp->anon_vma =NULL ; - }else if (anon_vma_fork (tmp, mpnt)) - goto fail_nomem_anon_vma_fork; -vm_flags_clear (tmp,VM_LOCKED_MASK ); -/* -* Copy/update hugetlb private vma information. -*/ -if (is_vm_hugetlb_page (tmp)) -hugetlb_dup_vma_private (tmp); - -/* -* Link the vma into the MT. After using __mt_dup(), memory -* allocation is not necessary here, so it cannot fail. -*/ -vma_iter_bulk_store (&vmi, tmp); - - mm->map_count++; - -if (tmp->vm_ops && tmp->vm_ops->open) - tmp->vm_ops->open (tmp); - - file = tmp->vm_file; -if (file) { -struct address_space *mapping = file->f_mapping; - -get_file (file); -i_mmap_lock_write (mapping); -if (vma_is_shared_maywrite (tmp)) -mapping_allow_writable (mapping); -flush_dcache_mmap_lock (mapping); -/* insert tmp into the share list, just after mpnt */ -vma_interval_tree_insert_after (tmp, mpnt, - &mapping->i_mmap); -flush_dcache_mmap_unlock (mapping); -i_mmap_unlock_write (mapping); - } - -if (!(tmp->vm_flags &VM_WIPEONFORK )) - retval =copy_page_range (tmp, mpnt); - -if (retval) { - mpnt =vma_next (&vmi); - goto loop_out; - } - } -/* a new mm has just been created */ - retval =arch_dup_mmap (oldmm, mm); -loop_out: -vma_iter_free (&vmi); -if (!retval) { -mt_set_in_rcu (vmi.mas.tree); -ksm_fork (mm, oldmm); -khugepaged_fork (mm, oldmm); - }else { - -/* -* The entire maple tree has already been duplicated. If the -* mmap duplication fails, mark the failure point with -* XA_ZERO_ENTRY. In exit_mmap(), if this marker is encountered, -* stop releasing VMAs that have not been duplicated after this -* point. -*/ -if (mpnt) { -mas_set_range (&vmi.mas, mpnt->vm_start, mpnt->vm_end -1 ); -mas_store (&vmi.mas,XA_ZERO_ENTRY ); -/* Avoid OOM iterating a broken tree */ -set_bit (MMF_OOM_SKIP , &mm->flags); - } -/* -* The mm_struct is going to exit, but the locks will be dropped -* first. Set the mm_struct as unstable is advisable as it is -* not fully initialised. -*/ -set_bit (MMF_UNSTABLE , &mm->flags); - } -out: -mmap_write_unlock (mm); -flush_tlb_mm (oldmm); -mmap_write_unlock (oldmm); -if (!retval) -dup_userfaultfd_complete (&uf); -else -dup_userfaultfd_fail (&uf); -return retval; - -fail_nomem_anon_vma_fork: -mpol_put (vma_policy (tmp)); -fail_nomem_policy: -vm_area_free (tmp); -fail_nomem: - retval = -ENOMEM ; -vm_unacct_memory (charge); - goto loop_out; -} -static inline int mm_alloc_pgd (struct mm_struct *mm) { mm->pgd =pgd_alloc (mm);
@@ -794,13 +620,6 @@ static inline void mm_free_pgd(struct mm_struct *mm) pgd_free(mm, mm->pgd); }#else -static int dup_mmap (struct mm_struct *mm,struct mm_struct *oldmm) -{ -mmap_write_lock (oldmm); -dup_mm_exe_file (mm, oldmm); -mmap_write_unlock (oldmm); -return 0 ; -}#define mm_alloc_pgd (mm) (0)#define mm_free_pgd (mm)#endif /* CONFIG_MMU */
mm/internal.h +2 -0
@@ -1630,5 +1630,7 @@ static inline bool reclaim_pt_is_enabled(unsigned long start, unsigned long end, }#endif /* CONFIG_PT_RECLAIM */ +void dup_mm_exe_file (struct mm_struct *mm,struct mm_struct *oldmm); +int dup_mmap (struct mm_struct *mm,struct mm_struct *oldmm);#endif /* __MM_INTERNAL_H */
mm/mmap.c +175 -6
192 lines not on this page · raw message
mm/nommu.c +8 -0
@@ -1874,3 +1874,11 @@ static int __meminit init_admin_reserve(void)return 0 ; }subsys_initcall (init_admin_reserve); + +int dup_mmap (struct mm_struct *mm,struct mm_struct *oldmm) +{ +mmap_write_lock (oldmm); +dup_mm_exe_file (mm, oldmm); +mmap_write_unlock (oldmm); +return 0 ; +}
Signature · 1 line
2.49.0
PATCH v2 3/3 mm: perform VMA allocation, freeing, duplication in mm
In reply to Lorenzo StoakesRight now these are performed in kernel/fork.c which is odd and a violation of separation of concerns, as well as preventing us from integrating this and related logic into userland VMA testing going forward, and perhaps more importantly - enabling us to, in a subsequent commit, make VMA allocation/freeing a purely internal mm operation.
There is a fly in the ointment - nommu - mmap.c is not compiled if CONFIG_MMU not set, and neither is vma.c.
To square the circle, let's add a new file - vma_init.c. This will be compiled for both CONFIG_MMU and nommu builds, and will also form part of the VMA userland testing.
This allows us to de-duplicate code, while maintaining separation of concerns and the ability for us to userland test this logic.
Update the VMA userland tests accordingly, additionally adding a detach_free_vma() helper function to correctly detach VMAs before freeing them in test code, as this change was triggering the assert for this.
--- MAINTAINERS | 1 + kernel/fork.c | 88 ------------------- mm/Makefile | 2 +- mm/mmap.c | 3 +- mm/nommu.c | 4 +- mm/vma.h | 6 ++ mm/vma_init.c | 101 ++++++++++++++++++++++ tools/testing/vma/Makefile | 2 +- tools/testing/vma/vma.c | 26 ++++-- tools/testing/vma/vma_internal.h | 143 +++++++++++++++++++++++++------ 10 files changed, 250 insertions(+), 126 deletions(-) create mode 100644 mm/vma_init.c
MAINTAINERS +1 -0
@@ -15608,6 +15608,7 @@ F: mm/mremap.c F: mm/mseal.c F: mm/vma.c F: mm/vma.h +F: mm/vma_init.c F: mm/vma_internal.h F: tools/testing/selftests/mm/merge.c F: tools/testing/vma/
kernel/fork.c +0 -88
109 lines not on this page · raw message
mm/Makefile +1 -1
@@ -55,7 +55,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o $(mmu-y) + debug.o gup.o mmap_lock.o vma_init.o $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o
mm/mmap.c +2 -1
@@ -1596,7 +1596,7 @@ static const struct ctl_table mmap_table[] = {#endif /* CONFIG_SYSCTL */ /* -* initialise the percpu counter for VM +* initialise the percpu counter for VM, initialise VMA state. */ void __init mmap_init (void ) {
@@ -1607,6 +1607,7 @@ void __init mmap_init(void)#ifdef CONFIG_SYSCTL register_sysctl_init ("vm" , mmap_table);#endif +vma_state_init (); }/*
mm/nommu.c +3 -1
@@ -399,7 +399,8 @@ static const struct ctl_table nommu_table[] = { };/* -* initialise the percpu counter for VM and region record slabs +* initialise the percpu counter for VM and region record slabs, initialise VMA +* state. */ void __init mmap_init (void ) {
@@ -409,6 +410,7 @@ void __init mmap_init(void)VM_BUG_ON (ret); vm_region_jar =KMEM_CACHE (vm_region,SLAB_PANIC |SLAB_ACCOUNT );register_sysctl_init ("vm" , nommu_table); +vma_state_init (); }/*
mm/vma.h +6 -0
@@ -548,4 +548,10 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address);int __vm_munmap (unsigned long start,size_t len,bool unlock); +/* vma_init.h, shared between CONFIG_MMU and nommu. */ +void __init vma_state_init (void ); +struct vm_area_struct *vm_area_alloc (struct mm_struct *mm); +struct vm_area_struct *vm_area_dup (struct vm_area_struct *orig); +void vm_area_free (struct vm_area_struct *vma); +#endif /* __MM_VMA_H */
mm/vma_init.c new file +101 -0
101 lines not on this page · raw message
tools/testing/vma/Makefile +1 -1
@@ -9,7 +9,7 @@ include ../shared/shared.mk OFILES = $(SHARED_OFILES) vma.o maple-shim.o TARGETS = vma -vma.o: vma.c vma_internal.h ../../../mm/vma.c ../../../mm/vma.h +vma.o: vma.c vma_internal.h ../../../mm/vma.c ../../../mm/vma_init.c ../../../mm/vma.h vma: $(OFILES) $(CC) $(CFLAGS) -o $@ $(OFILES) $(LDLIBS)
tools/testing/vma/vma.c +17 -9
@@ -28,6 +28,7 @@ unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;* Directly import the VMA implementation here. Our vma_internal.h wrapper * provides userland-equivalent functionality for everything vma.c uses. */ +#include "../../../mm/vma_init.c" #include "../../../mm/vma.c" const struct vm_operations_struct vma_dummy_vm_ops;
@@ -90,6 +91,12 @@ static int attach_vma(struct mm_struct *mm, struct vm_area_struct *vma)return res; } +static void detach_free_vma (struct vm_area_struct *vma) +{ +vma_mark_detached (vma); +vm_area_free (vma); +} +/* Helper function to allocate a VMA and link it to the tree. */ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm, unsigned long start,
@@ -103,7 +110,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,return NULL ;if (attach_vma (mm, vma)) { -vm_area_free (vma); +detach_free_vma (vma);return NULL ; }
@@ -248,7 +255,7 @@ static int cleanup_mm(struct mm_struct *mm, struct vma_iterator *vmi)vma_iter_set (vmi,0 );for_each_vma (*vmi, vma) { -vm_area_free (vma); +detach_free_vma (vma); count++; }
@@ -318,7 +325,7 @@ static bool test_simple_merge(void)ASSERT_EQ (vma->vm_pgoff,0 );ASSERT_EQ (vma->vm_flags, flags); -vm_area_free (vma); +detach_free_vma (vma);mtree_destroy (&mm.mm_mt);return true;
@@ -360,7 +367,7 @@ static bool test_simple_modify(void)ASSERT_EQ (vma->vm_end,0x1000 );ASSERT_EQ (vma->vm_pgoff,0 ); -vm_area_free (vma); +detach_free_vma (vma);vma_iter_clear (&vmi); vma =vma_next (&vmi);
@@ -369,7 +376,7 @@ static bool test_simple_modify(void)ASSERT_EQ (vma->vm_end,0x2000 );ASSERT_EQ (vma->vm_pgoff,1 ); -vm_area_free (vma); +detach_free_vma (vma);vma_iter_clear (&vmi); vma =vma_next (&vmi);
@@ -378,7 +385,7 @@ static bool test_simple_modify(void)ASSERT_EQ (vma->vm_end,0x3000 );ASSERT_EQ (vma->vm_pgoff,2 ); -vm_area_free (vma); +detach_free_vma (vma);mtree_destroy (&mm.mm_mt);return true;
@@ -406,7 +413,7 @@ static bool test_simple_expand(void)ASSERT_EQ (vma->vm_end,0x3000 );ASSERT_EQ (vma->vm_pgoff,0 ); -vm_area_free (vma); +detach_free_vma (vma);mtree_destroy (&mm.mm_mt);return true;
@@ -427,7 +434,7 @@ static bool test_simple_shrink(void)ASSERT_EQ (vma->vm_end,0x1000 );ASSERT_EQ (vma->vm_pgoff,0 ); -vm_area_free (vma); +detach_free_vma (vma);mtree_destroy (&mm.mm_mt);return true;
@@ -618,7 +625,7 @@ static bool test_merge_new(void)ASSERT_EQ (vma->vm_pgoff,0 );ASSERT_EQ (vma->anon_vma, &dummy_anon_vma); -vm_area_free (vma); +detach_free_vma (vma); count++; }
@@ -1667,6 +1674,7 @@ int main(void)int num_tests =0 , num_fail =0 ;maple_tree_init (); +vma_state_init ();#define TEST (name) \ do { \
tools/testing/vma/vma_internal.h +118 -25
191 lines not on this page · raw message
Signature · 1 line
2.49.0