mm: vmalloc: Avoid of calling __find_vmap_area() twise in __vunmap()
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH v2 1/3 mm: vmalloc: Avoid of calling __find_vmap_area() twise in __vunmap()
Currently __vunmap() path calls __find_vmap_area() two times. One on entry to check that area exists, second time inside remove_vm_area() function that also performs a new search of VA.
In order to improvie it from a performance point of view we split remove_vm_area() into two new parts:
- find_unlink_vmap_area() that does a search and unlink from tree;
- __remove_vm_area() that does a removing but without searching.
In this case there is no any functional change for remove_vm_area() whereas vm_remove_mappings(), where a second search happens, switches to the __remove_vm_area() variant where already detached VA is passed as a parameter, so there is no need to find it again.
Performance wise, i use test_vmalloc.sh with 32 threads doing alloc free on a 64-CPUs-x86_64-box:
perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k] __vunmap
- 30.92% __vunmap
- 17.67% _raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33% remove_vm_area
- 11.79% free_vmap_area_noflush
- 11.18% _raw_spin_lock
native_queued_spin_lock_slowpath
0.76% free_unref_page
perf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k] __vunmap
- 11.23% __vunmap
- 8.28% find_unlink_vmap_area
- 7.95% _raw_spin_lock
7.44% native_queued_spin_lock_slowpath
- 1.93% free_vmap_area_noflush
- 0.56% _raw_spin_lock
0.53% native_queued_spin_lock_slowpath
0.60% __vunmap_range_noflush
__vunmap() consumes around ~20% less CPU cycles on this test.
--- mm/vmalloc.c | 66 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 24 deletions(-)
mm/vmalloc.c +42 -24
@@ -1825,9 +1825,11 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + } nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);
@@ -1871,6 +1873,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /*
@@ -2591,6 +2606,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address
@@ -2607,22 +2636,8 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; + va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va); } static inline void set_area_direct_map(const struct vm_struct *area,
@@ -2637,15 +2652,16 @@ static inline void set_area_direct_map(const struct vm_struct *area, }/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +static void vm_remove_mappings (struct vmap_area *va,int deallocate_pages) { +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)
@@ -2690,6 +2706,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;
@@ -2698,19 +2715,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +vm_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i;
Signature · 1 line
2.30.2
Uladzislau Rezki (Sony)
maintainerPATCH v2 2/3 mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Switch from find_vmap_area() to find_unlink_vmap_area() to prevent a double access to the vmap_area_lock: one for finding area, second time is for unlinking from a tree.
--- mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmalloc.c +1 -1
@@ -2251,7 +2251,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));
Signature · 1 line
2.30.2
Re: [PATCH v2 2/3] mm: vmalloc: Switch to find_unlink_vmap_area() in vm_unmap_ram()
Looks good:
Quoted 14 lines · Uladzislau Rezki (Sony)
Switch from
find_vmap_area()tofind_unlink_vmap_area()to prevent a double access to thevmap_area_lock: one for finding area, second time is for unlinking from a tree.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)mm/vmalloc.c +0
-0
mm/vmalloc.c void vm_unmap_ram(const void *mem, unsigned int count) · line 2254
Quoted 5 lines
mm/vmalloc.c +1
-1@@ -2251,7 +2251,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);
I can't find find_unlink_vmap_area in current -next, but shouldn't this also switch from free_vmap_area_noflush to something that doesn't unlink from the list and avoid the lock?
In general the code could probably use a bit of refactoring to split unmapping from freeing.
Uladzislau Rezki (Sony)
maintainerPATCH v2 3/3 mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
In reply to Uladzislau Rezki (Sony)Currently a vm_unmap_ram() functions triggers a BUG() if an area is not found. Replace it by the WARN_ON_ONCE() error message and keep machine alive instead of stopping it.
The worst case is a memory leaking.
--- mm/vmalloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
mm/vmalloc.c +3 -1
@@ -2252,7 +2252,9 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); +if (WARN_ON_ONCE (!va)) +return ; +debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));free_unmap_vmap_area (va);
Signature · 1 line
2.30.2
Re: [PATCH v2 3/3] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
Quoted 27 lines · Uladzislau Rezki (Sony)
Currently a
vm_unmap_ram()functions triggers aBUG()if an area is not found. Replace it by theWARN_ON_ONCE()error message and keep machine alive instead of stopping it.The worst case is a memory leaking.
Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)mm/vmalloc.c +3
-2@@ -2252,7 +2252,9 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); +if (WARN_ON_ONCE (!va)) +return ; +debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));free_unmap_vmap_area (va); --
Quoted 2 lines
2.30.2
Some pedantic grammar/spelling stuff:-
(I know it can be a little annoying to get grammatical suggestions so I do hope that it isn't too irritating!)
For the Subject line:-
'mm: vmalloc: Avoid of calling __find_vmap_area() twise in __vunmap()' -> 'mm: vmalloc: Avoid calling __find_vmap_area() twice in __vunmap()'
Quoted 3 lines · Uladzislau Rezki (Sony)
Currently
__vunmap()path calls__find_vmap_area()two times. One on entry to check that area exists, second time insideremove_vm_area()function that also performs a new search of VA.
Perhaps slightly tweak to:-
"Currently the __vunmap() path calls __find_vmap_area() twice. Once on entry
to check that the area exists, then inside the remove_vm_area() function which also performs a new search for the VA."
Quoted 5 lines
In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that does a removing but without searching.
'that does a removing but without searching' reads better I think as 'that removes without searching'.
Quoted 6 lines
In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where already detached VA is passed as a parameter, so there is no need to find it again.
'where already detached VA' -> 'where the already detached VA' as a minor nit here!
Quoted 25 lines
Performance wise, i use
test_vmalloc.shwith 32 threads doing alloc free on a 64-CPUs-x86_64-box:perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k]__vunmap
- 30.92%__vunmap
- 17.67%_raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33%remove_vm_area
- 11.79%free_vmap_area_noflush
- 11.18%_raw_spin_lock
native_queued_spin_lock_slowpath
0.76%free_unref_pageperf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k]__vunmap
- 11.23%__vunmap
- 8.28%find_unlink_vmap_area
- 7.95%_raw_spin_lock
7.44%native_queued_spin_lock_slowpath
- 1.93%free_vmap_area_noflush
- 0.56%_raw_spin_lock
0.53%native_queued_spin_lock_slowpath
0.60%__vunmap_range_noflush
Quoted 1 line
__vunmap()consumes around ~20% less CPU cycles on this test.
Very nice, amazing work!
Quoted 13 lines
Reported-by Roman Gushchin <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 66 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 24 deletions(-)mm/vmalloc.c +0
-0@@ -1825,9 +1825,11 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1832
Quoted 10 lines
mm/vmalloc.c +5
-3unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + }
Do we want to do the same in free_vmap_area()?
Quoted 57 lines
mm/vmalloc.c +27
-9nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1873,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2591,6 +2606,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address @@ -2607,22 +2636,8 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm);
mm/vmalloc.c struct vm_struct *remove_vm_area(const void *addr) · line 2641
Quoted 10 lines
mm/vmalloc.c +2
-7-free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; + va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va); }
Really nice separation of concerns and cleanup.
mm/vmalloc.c static inline void set_area_direct_map(const struct vm_struct *area, · line 2655
Quoted 8 lines
mm/vmalloc.c +1
-1static inline void set_area_direct_map(const struct vm_struct *area,@@ -2637,15 +2652,16 @@ static inline void set_area_direct_map(const struct vm_struct *area, }/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +static void vm_remove_mappings (struct vmap_area *va,int deallocate_pages)
Perhaps rename this to va_remove_mappings() or vmap_area_remove_mappings() since it is now explicitly accepting a vmap_area rather than vm_struct?
Quoted 14 lines
mm/vmalloc.c +2
-1{ +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)
mm/vmalloc.c static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) · line 2708
Quoted 3 lines
mm/vmalloc.c +0
-0@@ -2690,6 +2706,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area;
Feels like it's getting a bit confusing with 'va' representing vmap_area and 'area' which represents... vm_struct (this file has a bunch of naming inconsistencies like this actually), perhaps rename this to 'vm'?
Quoted 29 lines
mm/vmalloc.c +5
-4+struct vmap_area *va;if (!addr)return ;@@ -2698,19 +2715,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +vm_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i; --
Quoted 2 lines
2.30.2
Other than some pendatic points about grammar/naming this looks really good!
Uladzislau Rezki
maintainerQuoted 5 lines
Some pedantic grammar/spelling stuff:-
(I know it can be a little annoying to get grammatical suggestions so I do hope that it isn't too irritating!)
It is absolutely OK :)
Quoted 5 lines
For the Subject line:-
'mm: vmalloc: Avoid of calling__find_vmap_area()twise in__vunmap()' -> 'mm: vmalloc: Avoid calling__find_vmap_area()twice in__vunmap()'
Will fix in the v3.
Quoted 11 lines · Uladzislau Rezki (Sony)
Quoted 3 lines
Currently
__vunmap()path calls__find_vmap_area()two times. One on entry to check that area exists, second time insideremove_vm_area()function that also performs a new search of VA.Perhaps slightly tweak to:-
"Currently the
__vunmap()path calls__find_vmap_area()twice. Once on entry
to check that the area exists, then inside theremove_vm_area()function which also performs a new search for the VA."
Will fix in the v3.
Quoted 9 lines
Quoted 5 lines
In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that does a removing but without searching.'that does a removing but without searching' reads better I think as 'that removes without searching'.
Will fix in the v3.
Quoted 10 lines
Quoted 6 lines
In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where already detached VA is passed as a parameter, so there is no need to find it again.'where already detached VA' -> 'where the already detached VA' as a minor nit here!
Will fix it.
Quoted 25 lines
Quoted 25 lines
Performance wise, i use
test_vmalloc.shwith 32 threads doing alloc free on a 64-CPUs-x86_64-box:perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k]__vunmap
- 30.92%__vunmap
- 17.67%_raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33%remove_vm_area
- 11.79%free_vmap_area_noflush
- 11.18%_raw_spin_lock
native_queued_spin_lock_slowpath
0.76%free_unref_pageperf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k]__vunmap
- 11.23%__vunmap
- 8.28%find_unlink_vmap_area
- 7.95%_raw_spin_lock
7.44%native_queued_spin_lock_slowpath
- 1.93%free_vmap_area_noflush
- 0.56%_raw_spin_lock
0.53%native_queued_spin_lock_slowpath
0.60%__vunmap_range_noflush
Quoted 4 lines
Quoted 1 line
__vunmap()consumes around ~20% less CPU cycles on this test.Very nice, amazing work!
Thanks!
Quoted 15 lines
Quoted 15 lines
Reported-by Roman Gushchin <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 66 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 24 deletions(-)mm/vmalloc.c +0
-0@@ -1825,9 +1825,11 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy;
Quoted 11 lines
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1832
Quoted 8 lines
mm/vmalloc.c +5
-3-spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + }Do we want to do the same in
free_vmap_area()?
The free_vmap_area() is a bit special. It only pairs with alloc_vmap_area(). There are two users and both invoke free_vmap_area() in a error path. So probably it would be good to remove it fully. But it requires some refactoring.
Quoted 61 lines
Quoted 61 lines
mm/vmalloc.c +27
-13nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1873,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2591,6 +2606,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address @@ -2607,22 +2636,8 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - }
Quoted 9 lines
mm/vmalloc.c struct vm_struct *remove_vm_area(const void *addr) · line 2641
Quoted 6 lines
mm/vmalloc.c +2
-3- -spin_unlock (&vmap_area_lock); -return NULL ; + va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va); }Really nice separation of concerns and cleanup.
Thanks!
Quoted 5 lines
Quoted 5 lines
mm/vmalloc.c +0
-0static inline void set_area_direct_map(const struct vm_struct *area,@@ -2637,15 +2652,16 @@ static inline void set_area_direct_map(const struct vm_struct *area, }
Quoted 7 lines
mm/vmalloc.c static inline void set_area_direct_map(const struct vm_struct *area, · line 2655
Quoted 3 lines
mm/vmalloc.c +1
-1/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +static void vm_remove_mappings (struct vmap_area *va,int deallocate_pages)Perhaps rename this to
va_remove_mappings()orvmap_area_remove_mappings()since it is now explicitly accepting avmap_arearather thanvm_struct?
I agree. There is a discrepancy. I can rename it to the va_remove_mappings() if there are no any complains from others.
Quoted 11 lines
Quoted 11 lines
mm/vmalloc.c +2
-1{ +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);
Quoted 11 lines
mm/vmalloc.c static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) · line 2708
Quoted 6 lines
mm/vmalloc.c +0
-0/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)@@ -2690,6 +2706,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area;Feels like it's getting a bit confusing with 'va' representing
vmap_areaand 'area' which represents...vm_struct(this file has a bunch of naming inconsistencies like this actually), perhaps rename this to 'vm'?
We can. I think it should be a separate patch-set for refactoring of things like: va, vm, area, vmap, etc :)
Quoted 26 lines
Quoted 26 lines
mm/vmalloc.c +5
-3+struct vmap_area *va;if (!addr)return ;@@ -2698,19 +2715,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +vm_remove_mappings (va, deallocate_pages);
Quoted 8 lines
Quoted 5 lines
mm/vmalloc.c +0
-1if (deallocate_pages) {int i; --2.30.2
Other than some pendatic points about grammar/naming this looks really good!
Thank you for the review!
--
Uladzislau Rezki
A sorry. I need more coffee, this is not the cover letter, but the patch that introduceѕ find_unlink_vmap_area.
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1832
Quoted 8 lines
mm/vmalloc.c +5
-3-spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + }
As mentioned before, I'd much rather move this into the callers.
mm/vmalloc.c struct vm_struct *remove_vm_area(const void *addr) · line 2640
Quoted 2 lines
mm/vmalloc.c +2
-0+ va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va);
This can drop the va local variable now.
Uladzislau Rezki
maintainerQuoted 3 lines
A sorry. I need more coffee, this is not the cover letter, but the patch that introduceѕ
find_unlink_vmap_area.
Sorry. I should post it with a cover latter to make it less confusing.
Quoted 11 lines
mm/vmalloc.c static void free_vmap_area_noflush(struct vmap_area *va) · line 1832
Quoted 8 lines
mm/vmalloc.c +5
-3-spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + }As mentioned before, I'd much rather move this into the callers.
Agree. There is only one caller, it is the free_vmap_block(). Will fix in the v3.
Quoted 5 lines
mm/vmalloc.c struct vm_struct *remove_vm_area(const void *addr) · line 2640
Quoted 2 lines
mm/vmalloc.c +2
-0+ va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va);This can drop the va local variable now.
Do you mean like:
struct vm_struct *remove_vm_area(const void *addr)
{
might_sleep();
return __remove_vm_area(
find_unlink_vmap_area((unsigned long) addr));
}?
Thanks for review!
--
Uladzislau Rezki
Quoted 1 line · Uladzislau Rezki
Do you mean like:
Yes.
Uladzislau Rezki
maintainerQuoted 5 lines · Christoph Hellwig
Quoted 1 line · Uladzislau Rezki
Do you mean like:
Yes.
OK. Will eliminate that local variable.
Thanks.
--
Uladzislau Rezki
Quoted 9 lines · Uladzislau Rezki (Sony)
Currently
__vunmap()path calls__find_vmap_area()two times. One on entry to check that area exists, second time insideremove_vm_area()function that also performs a new search of VA.In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that does a removing but without searching.
Quoted 4 lines
In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where already detached VA is passed as a parameter, so there is no need to find it again.
I like this patch. This takes off the va->vm clearning too. Finally I don't need to worry about the va->flags clearing during unmapping when reading out vmap_block areas.
Quoted 173 lines
Performance wise, i use
test_vmalloc.shwith 32 threads doing alloc free on a 64-CPUs-x86_64-box:perf without this patch:
- 31.41% 0.50% vmalloc_test/10 [kernel.vmlinux] [k]__vunmap
- 30.92%__vunmap
- 17.67%_raw_spin_lock
native_queued_spin_lock_slowpath
- 12.33%remove_vm_area
- 11.79%free_vmap_area_noflush
- 11.18%_raw_spin_lock
native_queued_spin_lock_slowpath
0.76%free_unref_pageperf with this patch:
- 11.35% 0.13% vmalloc_test/14 [kernel.vmlinux] [k]__vunmap
- 11.23%__vunmap
- 8.28%find_unlink_vmap_area
- 7.95%_raw_spin_lock
7.44%native_queued_spin_lock_slowpath
- 1.93%free_vmap_area_noflush
- 0.56%_raw_spin_lock
0.53%native_queued_spin_lock_slowpath
0.60%__vunmap_range_noflush
__vunmap()consumes around ~20% less CPU cycles on this test.Reported-by Roman Gushchin <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 66 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 24 deletions(-)mm/vmalloc.c +42
-24@@ -1825,9 +1825,11 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + } nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1873,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2591,6 +2606,20 @@ struct vm_struct *find_vm_area(const void *addr)return va->vm; } +static struct vm_struct *__remove_vm_area (struct vmap_area *va) +{ +struct vm_struct *vm; + +if (!va || !va->vm) +return NULL ; + + vm = va->vm; +kasan_free_module_shadow (vm); +free_unmap_vmap_area (va); + +return vm; +} +/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address @@ -2607,22 +2636,8 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { -struct vm_struct *vm = va->vm; - - va->vm =NULL ; -spin_unlock (&vmap_area_lock); - -kasan_free_module_shadow (vm); -free_unmap_vmap_area (va); - -return vm; - } - -spin_unlock (&vmap_area_lock); -return NULL ; + va =find_unlink_vmap_area ((unsigned long ) addr); +return __remove_vm_area (va); } static inline void set_area_direct_map(const struct vm_struct *area,@@ -2637,15 +2652,16 @@ static inline void set_area_direct_map(const struct vm_struct *area, }/* Handle removing and resetting vm mappings related to the vm_struct. */ -static void vm_remove_mappings (struct vm_struct *area,int deallocate_pages) +static void vm_remove_mappings (struct vmap_area *va,int deallocate_pages) { +struct vm_struct *area = va->vm;unsigned long start =ULONG_MAX , end =0 ;unsigned int page_order =vm_area_page_order (area);int flush_reset = area->flags &VM_FLUSH_RESET_PERMS ;int flush_dmap =0 ;int i; -remove_vm_area (area->addr); +__remove_vm_area (va);/* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */ if (!flush_reset)@@ -2690,6 +2706,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;@@ -2698,19 +2715,20 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); -if (unlikely (!area)) { + va =find_unlink_vmap_area ((unsigned long )addr); +if (unlikely (!va)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);return ; } + area = va->vm;debug_check_no_locks_freed (area->addr,get_vm_area_size (area));debug_check_no_obj_freed (area->addr,get_vm_area_size (area));kasan_poison_vmalloc (area->addr,get_vm_area_size (area)); -vm_remove_mappings (area, deallocate_pages); +vm_remove_mappings (va, deallocate_pages);if (deallocate_pages) {int i;Signature · 1 line
2.30.2
Uladzislau Rezki
maintainerQuoted 10 lines · Baoquan He
Quoted 9 lines · Uladzislau Rezki (Sony)
Currently
__vunmap()path calls__find_vmap_area()two times. One on entry to check that area exists, second time insideremove_vm_area()function that also performs a new search of VA.In order to improvie it from a performance point of view we split
remove_vm_area()into two new parts:
-find_unlink_vmap_area()that does a search and unlink from tree;
-__remove_vm_area()that does a removing but without searching.
Quoted 9 lines
Quoted 4 lines
In this case there is no any functional change for
remove_vm_area()whereasvm_remove_mappings(), where a second search happens, switches to the__remove_vm_area()variant where already detached VA is passed as a parameter, so there is no need to find it again.I like this patch. This takes off the
va->vmclearning too. Finally I don't need to worry about theva->flagsclearing during unmapping when reading outvmap_blockareas.
Thanks. This patch was one of the reason to help out with the per-cpu busy areas tracking/reading of your work :)
--
Uladzsislau Rezki