mm: vmalloc: Avoid calling __find_vmap_area() twice in __vunmap()
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH v3 1/3 mm: vmalloc: Avoid calling __find_vmap_area() twice in __vunmap()
Currently the __vunmap() path calls __find_vmap_area() twice. Once on entry to check that the area exists, then inside the remove_vm_area() function which also performs a new search for the VA.
In order to improvie it from a performance point of view we split remove_vm_area() into two new parts:
- find_unlink_vmap_area() that does a search and unlink from tree;
- __remove_vm_area() that removes without searching.
In this case there is no any functional change for remove_vm_area() whereas vm_remove_mappings(), where a second search happens, switches to the __remove_vm_area() variant where the already detached VA is passed as a parameter, so there is no need to find it again.
Performance wise, i use test_vmalloc.sh with 32 threads doing alloc free on a 64-CPUs-x86_64-box:
perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k] __vunmap
- 30.92% __vunmap
- 17.67% _raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33% remove_vm_area
- 11.79% free_vmap_area_noflush
- 11.18% _raw_spin_lock
native_queued_spin_lock_slowpath
0.76% free_unref_page
perf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k] __vunmap
- 11.23% __vunmap
- 8.28% find_unlink_vmap_area
- 7.95% _raw_spin_lock
7.44% native_queued_spin_lock_slowpath
- 1.93% free_vmap_area_noflush
- 0.56% _raw_spin_lock
0.53% native_queued_spin_lock_slowpath
0.60% __vunmap_range_noflush
__vunmap() consumes around ~20% less CPU cycles on this test.
v2 -> v3:
- update commit message;
- rename the vm_remove_mappings() to the va_remove_mappings();
- move va-unlinking to the callers so the free_vmap_area_noflush() now expects a VA that has been disconnected;
- eliminate a local variable in the remove_vm_area().
--- mm/vmalloc.c | 77 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 30 deletions(-)
mm/vmalloc.c +47 -30
@@ -1815,9 +1815,9 @@ static void drain_vmap_area_work(struct work_struct *work) }/* -* Free a vmap area, caller ensuring that the area has been unmapped -* and flush_cache_vunmap had been called for the correct range -* previously. +* Free a vmap area, caller ensuring that the area has been unmapped, +* unlinked and flush_cache_vunmap had been called for the correct +* range previously. */ static void free_vmap_area_noflush (struct vmap_area *va) {
@@ -1825,9 +1825,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (WARN_ON_ONCE (!list_empty (&va->list))) +return ; nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);
@@ -1871,6 +1870,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /*
@@ -2015,6 +2027,10 @@ static void free_vmap_block(struct vmap_block *vb) tmp =xa_erase (&vmap_blocks,addr_to_vb_idx (vb->va->va_start));BUG_ON (tmp != vb); +spin_lock (&vmap_area_lock); +unlink_va (vb->va, &vmap_area_root); +spin_unlock (&vmap_area_lock); +free_vmap_area_noflush (vb->va);kfree_rcu (vb, rcu_head); }
@@ -2591,6 +2607,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address
@@ -2603,26 +2633,10 @@ struct vm_struct *find_vm_area(const void *addr)*/ struct vm_struct *remove_vm_area (const void *addr) { -struct vmap_area *va; -might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; +return __remove_vm_area ( +find_unlink_vmap_area ((unsigned long ) addr)); } static inline void set_area_direct_map(const struct vm_struct *area,
@@ -2636,16 +2650,17 @@ static inline void set_area_direct_map(const struct vm_struct *area,set_direct_map (area->pages[i]); } -/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +/* Handle removing and resetting vm mappings related to the VA's vm_struct. */ +static void va_remove_mappings (struct vmap_area *va,int deallocate_pages) { +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)
@@ -2690,6 +2705,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;
@@ -2698,19 +2714,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +va_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i;
Signature · 1 line
2.30.2
Uladzislau Rezki (Sony)
maintainerPATCH v3 2/3 mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Switch from find_vmap_area() to find_unlink_vmap_area() to prevent a double access to the vmap_area_lock: one for finding area, second time is for unlinking from a tree.
--- mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmalloc.c +1 -1
@@ -2252,7 +2252,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));
Signature · 1 line
2.30.2
Re: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
mm/vmalloc.c void vm_unmap_ram(const void *mem, unsigned int count) · line 2258
Quoted 11 lines · Uladzislau Rezki (Sony)
mm/vmalloc.c +1
-1@@ -2252,7 +2252,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));
Don't we also need to remove the manual unlink that was done
here previously? Actually it seems like that manual unlink is missing
after patch 1, creating a bisection hazard. So either add it there, or just fold this patch into the previous one.
Uladzislau Rezki
maintainerRe: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Quoted 7 lines · Uladzislau Rezki (Sony)
Quoted 6 lines
mm/vmalloc.c +0
-0@@ -2252,7 +2252,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; }
Quoted 11 lines
mm/vmalloc.c void vm_unmap_ram(const void *mem, unsigned int count) · line 2258
Quoted 5 lines
mm/vmalloc.c +1
-1- va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));Don't we also need to remove the manual unlink that was done
here previously? Actually it seems like that manual unlink is missing
after patch 1, creating a bisection hazard. So either add it there, or just fold this patch into the previous one.
Right. In terms of bisection it is not so good. I think folding is the best.
Andrew, could you please fold this patch into the:
[PATCH v3 1/3] mm: vmalloc: Avoid calling __find_vmap_area() twice in __vunmap() ?
or should i send a v4 instead?
Thank you in advance!
--
Uladzislau Rezki
Andrew Morton
maintainerRe: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Quoted 9 lines · On Fri, 23 Dec 2022 17:41:48 +0100 Uladzislau Rezki <[email protected]> wrote:
Quoted 5 lines
Don't we also need to remove the manual unlink that was done
here previously? Actually it seems like that manual unlink is missing
after patch 1, creating a bisection hazard. So either add it there, or just fold this patch into the previous one.Right. In terms of bisection it is not so good. I think folding is the best.
Andrew, could you please fold this patch into the:
which patch ;)
Quoted 3 lines
[PATCH v3 1/3] mm: vmalloc: Avoid calling
__find_vmap_area()twice in__vunmap()?or should i send a v4 instead?
Yes please.
Uladzislau Rezki
maintainerRe: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Quoted 7 lines · Andrew Morton
Quoted 5 lines · On Fri, 23 Dec 2022 17:41:48 +0100 Uladzislau Rezki <[email protected]> wrote:
Quoted 5 lines
Don't we also need to remove the manual unlink that was done
here previously? Actually it seems like that manual unlink is missing
after patch 1, creating a bisection hazard. So either add it there, or just fold this patch into the previous one.
Quoted 7 lines
Quoted 4 lines
Right. In terms of bisection it is not so good. I think folding is the best.
Andrew, could you please fold this patch into the:
which patch ;)
Currently the next-20221226 contains three patches:
<snip>
[1]
commit c83b70c3cc1ecf99897ca0ea6e44aa2125a61ccb
Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:54 2022 +0100
mm: vmalloc: replace BUG_ON() by WARN_ON_ONCE()
[2]
commit 8a85ea97b35924ee39d51e00ecb3f6d07f748a36
Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:53 2022 +0100
mm: vmalloc: switch to find_unlink_vmap_area() in vm_unmap_ram()
[3]
commit a7c84c673c71cdfad20fe25e5d2051ed229859f7
Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:52 2022 +0100
mm: vmalloc: avoid calling __find_vmap_area() twise in __vunmap()
<snip>
It would be good if you could fold [2] into [3] making it as one patch. The problem is that, if we leave it as it is, the bisection mechanism would consider [3] as a buggy patch, because it is not fully accomplished and depends on [2].
Is that OK for you, i mean to squash on your own? Or i just should resend one more time?
Thank you in advance!
--
Uladzislau Rezki
Andrew Morton
maintainerRe: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Quoted 15 lines · On Thu, 29 Dec 2022 13:47:43 +0100 Uladzislau Rezki <[email protected]> wrote:
[2]
commit8a85ea97b35924ee39d51e00ecb3f6d07f748a36Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:53 2022 +0100mm: vmalloc: switch to
find_unlink_vmap_area()invm_unmap_ram()[3]
commita7c84c673c71cdfad20fe25e5d2051ed229859f7Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:52 2022 +0100mm: vmalloc: avoid calling
__find_vmap_area()twise in__vunmap()
<snip>
Quoted 6 lines
It would be good if you could fold [2] into [3] making it as one patch. The problem is that, if we leave it as it is, the bisection mechanism would consider [3] as a buggy patch, because it is not fully accomplished and depends on [2].
Is that OK for you, i mean to squash on your own?
I did that. I updated the "mm: vmalloc: avoid calling __find_vmap_area() twice in __vunmap()" accordingly, thanks.
Uladzislau Rezki
maintainerRe: [PATCH v3 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Quoted 22 lines · Andrew Morton
Quoted 20 lines · On Thu, 29 Dec 2022 13:47:43 +0100 Uladzislau Rezki <[email protected]> wrote:
[2]
commit8a85ea97b35924ee39d51e00ecb3f6d07f748a36Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:53 2022 +0100mm: vmalloc: switch to
find_unlink_vmap_area()invm_unmap_ram()[3]
commita7c84c673c71cdfad20fe25e5d2051ed229859f7Author: Uladzislau Rezki (Sony) <[email protected]> Date: Wed Dec 21 18:44:52 2022 +0100mm: vmalloc: avoid calling
__find_vmap_area()twise in__vunmap()
<snip>It would be good if you could fold [2] into [3] making it as one patch. The problem is that, if we leave it as it is, the bisection mechanism would consider [3] as a buggy patch, because it is not fully accomplished and depends on [2].
Quoted 5 lines
Quoted 1 line
Is that OK for you, i mean to squash on your own?
I did that. I updated the "mm: vmalloc: avoid calling
__find_vmap_area()twice in__vunmap()" accordingly, thanks.
At least bisection part will not detect anything wrong now.
Happy New Year and thank you!
--
Uladzislau Rezki
Uladzislau Rezki (Sony)
maintainerPATCH v3 3/3 mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
In reply to Uladzislau Rezki (Sony)Currently a vm_unmap_ram() functions triggers a BUG() if an area is not found. Replace it by the WARN_ON_ONCE() error message and keep machine alive instead of stopping it.
The worst case is a memory leaking.
--- mm/vmalloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
mm/vmalloc.c +3 -1
@@ -2253,7 +2253,9 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); +if (WARN_ON_ONCE (!va)) +return ; +debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));free_unmap_vmap_area (va);
Signature · 1 line
2.30.2
Re: [PATCH v3 3/3] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
Looks good:
Quoted 218 lines · Uladzislau Rezki (Sony)
Currently the
__vunmap()path calls__find_vmap_area()twice. Once on entry to check that the area exists, then inside theremove_vm_area()function which also performs a new search for the VA.In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that removes without searching.In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where the already detached VA is passed as a parameter, so there is no need to find it again.Performance wise, i use
test_vmalloc.shwith 32 threads doing alloc free on a 64-CPUs-x86_64-box:perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k]__vunmap
- 30.92%__vunmap
- 17.67%_raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33%remove_vm_area
- 11.79%free_vmap_area_noflush
- 11.18%_raw_spin_lock
native_queued_spin_lock_slowpath
0.76%free_unref_pageperf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k]__vunmap
- 11.23%__vunmap
- 8.28%find_unlink_vmap_area
- 7.95%_raw_spin_lock
7.44%native_queued_spin_lock_slowpath
- 1.93%free_vmap_area_noflush
- 0.56%_raw_spin_lock
0.53%native_queued_spin_lock_slowpath
0.60%__vunmap_range_noflush
__vunmap()consumes around ~20% less CPU cycles on this test.v2 -> v3:
- update commit message;
- rename thevm_remove_mappings()to theva_remove_mappings();
- move va-unlinking to the callers so thefree_vmap_area_noflush()now expects a VA that has been disconnected;
- eliminate a local variable in theremove_vm_area().Reported-by Roman Gushchin <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 77 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 30 deletions(-)mm/vmalloc.c +47
-31@@ -1815,9 +1815,9 @@ static void drain_vmap_area_work(struct work_struct *work) }/* -* Free a vmap area, caller ensuring that the area has been unmapped -* and flush_cache_vunmap had been called for the correct range -* previously. +* Free a vmap area, caller ensuring that the area has been unmapped, +* unlinked and flush_cache_vunmap had been called for the correct +* range previously. */ static void free_vmap_area_noflush (struct vmap_area *va) {@@ -1825,9 +1825,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (WARN_ON_ONCE (!list_empty (&va->list))) +return ; nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1870,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2015,6 +2027,10 @@ static void free_vmap_block(struct vmap_block *vb) tmp =xa_erase (&vmap_blocks,addr_to_vb_idx (vb->va->va_start));BUG_ON (tmp != vb); +spin_lock (&vmap_area_lock); +unlink_va (vb->va, &vmap_area_root); +spin_unlock (&vmap_area_lock); +free_vmap_area_noflush (vb->va);kfree_rcu (vb, rcu_head); }@@ -2591,6 +2607,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address @@ -2603,26 +2633,10 @@ struct vm_struct *find_vm_area(const void *addr)*/ struct vm_struct *remove_vm_area (const void *addr) { -struct vmap_area *va; -might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; +return __remove_vm_area ( +find_unlink_vmap_area ((unsigned long ) addr)); } static inline void set_area_direct_map(const struct vm_struct *area,@@ -2636,16 +2650,17 @@ static inline void set_area_direct_map(const struct vm_struct *area,set_direct_map (area->pages[i]); } -/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +/* Handle removing and resetting vm mappings related to the VA's vm_struct. */ +static void va_remove_mappings (struct vmap_area *va,int deallocate_pages) { +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)@@ -2690,6 +2705,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;@@ -2698,19 +2714,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +va_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i; --
Quoted 2 lines
2.30.2
All looks good to me! Great work! Feel free to add the below to all patches in series:-
Uladzislau Rezki
maintainerQuoted 224 lines · Lorenzo Stoakes
Quoted 218 lines · Uladzislau Rezki (Sony)
Currently the
__vunmap()path calls__find_vmap_area()twice. Once on entry to check that the area exists, then inside theremove_vm_area()function which also performs a new search for the VA.In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that removes without searching.In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where the already detached VA is passed as a parameter, so there is no need to find it again.Performance wise, i use
test_vmalloc.shwith 32 threads doing alloc free on a 64-CPUs-x86_64-box:perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k]__vunmap
- 30.92%__vunmap
- 17.67%_raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33%remove_vm_area
- 11.79%free_vmap_area_noflush
- 11.18%_raw_spin_lock
native_queued_spin_lock_slowpath
0.76%free_unref_pageperf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k]__vunmap
- 11.23%__vunmap
- 8.28%find_unlink_vmap_area
- 7.95%_raw_spin_lock
7.44%native_queued_spin_lock_slowpath
- 1.93%free_vmap_area_noflush
- 0.56%_raw_spin_lock
0.53%native_queued_spin_lock_slowpath
0.60%__vunmap_range_noflush
__vunmap()consumes around ~20% less CPU cycles on this test.v2 -> v3:
- update commit message;
- rename thevm_remove_mappings()to theva_remove_mappings();
- move va-unlinking to the callers so thefree_vmap_area_noflush()now expects a VA that has been disconnected;
- eliminate a local variable in theremove_vm_area().Reported-by Roman Gushchin <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 77 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 30 deletions(-)mm/vmalloc.c +47
-31@@ -1815,9 +1815,9 @@ static void drain_vmap_area_work(struct work_struct *work) }/* -* Free a vmap area, caller ensuring that the area has been unmapped -* and flush_cache_vunmap had been called for the correct range -* previously. +* Free a vmap area, caller ensuring that the area has been unmapped, +* unlinked and flush_cache_vunmap had been called for the correct +* range previously. */ static void free_vmap_area_noflush (struct vmap_area *va) {@@ -1825,9 +1825,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (WARN_ON_ONCE (!list_empty (&va->list))) +return ; nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1870,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2015,6 +2027,10 @@ static void free_vmap_block(struct vmap_block *vb) tmp =xa_erase (&vmap_blocks,addr_to_vb_idx (vb->va->va_start));BUG_ON (tmp != vb); +spin_lock (&vmap_area_lock); +unlink_va (vb->va, &vmap_area_root); +spin_unlock (&vmap_area_lock); +free_vmap_area_noflush (vb->va);kfree_rcu (vb, rcu_head); }@@ -2591,6 +2607,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address @@ -2603,26 +2633,10 @@ struct vm_struct *find_vm_area(const void *addr)*/ struct vm_struct *remove_vm_area (const void *addr) { -struct vmap_area *va; -might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; +return __remove_vm_area ( +find_unlink_vmap_area ((unsigned long ) addr)); } static inline void set_area_direct_map(const struct vm_struct *area,@@ -2636,16 +2650,17 @@ static inline void set_area_direct_map(const struct vm_struct *area,set_direct_map (area->pages[i]); } -/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +/* Handle removing and resetting vm mappings related to the VA's vm_struct. */ +static void va_remove_mappings (struct vmap_area *va,int deallocate_pages) { +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)@@ -2690,6 +2705,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;@@ -2698,19 +2714,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +va_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i; --Quoted 2 lines
2.30.2
All looks good to me! Great work! Feel free to add the below to all patches in series:-
Quoted 2 lines
Reviewed-by Lorenzo Stoakes <[email protected]>
Added :)
Thank you for review!
--
Uladzislau Rezki
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1829
Quoted 9 lines · Uladzislau Rezki (Sony)
mm/vmalloc.c +2
-3@@ -1825,9 +1825,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (WARN_ON_ONCE (!list_empty (&va->list))) +return ;
I'd just drop this check as the function is not exported.
Otherwise looks good:
Uladzislau Rezki
maintainerQuoted 14 lines · Christoph Hellwig
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1829
Quoted 9 lines · Uladzislau Rezki (Sony)
mm/vmalloc.c +2
-3@@ -1825,9 +1825,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (WARN_ON_ONCE (!list_empty (&va->list))) +return ;I'd just drop this check as the function is not exported.
Quoted 4 lines
Otherwise looks good:
Reviewed-by Christoph Hellwig <[email protected]>
Thanks!
--
Uladzislau Rezki