mm: vmalloc: Avoid a double lookup of freed VA in a tree
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH 1/2 mm: vmalloc: Avoid a double lookup of freed VA in a tree
When a VA is freed over a main path, for example by invoking the vfree() function, a tree is accessed two times what is odd:
vfree():
__vunmap()
__find_vmap_area()
vm_remove_mappings()
remove_vm_area()
__find_vmap_area()__find_vmap_area() are called two times. Fix it by introducing a find_unlink_vmap_area() helper that finds and un-links a VA from a tree.
Performance test results on a single CPU:
- fix_size_alloc_test loops: 1000000 avg: 476847 usec - full_fit_alloc_test loops: 1000000 avg: 806746 usec - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec - random_size_alloc_test loops: 1000000 avg: 7441322 usec - fix_align_alloc_test loops: 1000000 avg: 1411132 usec
All test took worker0=87650866284 cycles
- fix_size_alloc_test loops: 1000000 avg: 490713 usec - full_fit_alloc_test loops: 1000000 avg: 579162 usec - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec - random_size_alloc_test loops: 1000000 avg: 5824449 usec - fix_align_alloc_test loops: 1000000 avg: 984735 usec
All test took worker0=67952362802 cycles
--- mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-)
mm/vmalloc.c +28 -12
@@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +/* +* A free_vmap_block() is left. It is NOT a main free path. +*/ +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + } nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);
@@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /*
@@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));
@@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { + va =find_unlink_vmap_area ((unsigned long ) addr); +if (va) {struct vm_struct *vm = va->vm; - va->vm =NULL ; -spin_unlock (&vmap_area_lock); -kasan_free_module_shadow (vm);free_unmap_vmap_area (va);return vm; } -spin_unlock (&vmap_area_lock);return NULL ; }
@@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;
@@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); + va =find_unlink_vmap_area ((unsigned long )addr); + area = va->vm; +if (unlikely (!area)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);
Signature · 1 line
2.30.2
Uladzislau Rezki (Sony)
maintainerPATCH 2/2 mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
Currently a vm_unmap_ram() functions triggers a BUG() if an area is not found. Replace it by the WARN_ON_ONCE() error message and keep machine alive instead of stopping it.
The worst case is a memory leaking.
--- mm/vmalloc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
mm/vmalloc.c +6 -4
@@ -2255,10 +2255,12 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); -debug_check_no_locks_freed ((void *)va->va_start, - (va->va_end - va->va_start)); -free_unmap_vmap_area (va); + +if (!WARN_ON_ONCE (!va)) { +debug_check_no_locks_freed ((void *)va->va_start, + (va->va_end - va->va_start)); +free_unmap_vmap_area (va); + } }EXPORT_SYMBOL (vm_unmap_ram);
Signature · 1 line
2.30.2
Uladzislau Rezki
maintainerRe: [PATCH 2/2] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
Quoted 35 lines · Uladzislau Rezki (Sony)
Currently a
vm_unmap_ram()functions triggers aBUG()if an area is not found. Replace it by theWARN_ON_ONCE()error message and keep machine alive instead of stopping it.The worst case is a memory leaking.
Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)mm/vmalloc.c +6
-4@@ -2255,10 +2255,12 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); -debug_check_no_locks_freed ((void *)va->va_start, - (va->va_end - va->va_start)); -free_unmap_vmap_area (va); + +if (!WARN_ON_ONCE (!va)) { +debug_check_no_locks_freed ((void *)va->va_start, + (va->va_end - va->va_start)); +free_unmap_vmap_area (va); + } }EXPORT_SYMBOL (vm_unmap_ram);Signature · 1 line
2.30.2
Will send a v2.
--
Uladzislau Rezki
Re: [PATCH 2/2] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
In reply to Uladzislau Rezki (Sony)Quoted 33 lines · Uladzislau Rezki (Sony)
Currently a
vm_unmap_ram()functions triggers aBUG()if an area is not found. Replace it by theWARN_ON_ONCE()error message and keep machine alive instead of stopping it.The worst case is a memory leaking.
Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)mm/vmalloc.c +6
-5@@ -2255,10 +2255,12 @@ void vm_unmap_ram(const void *mem, unsigned int count) } va =find_unlink_vmap_area (addr); -BUG_ON (!va); -debug_check_no_locks_freed ((void *)va->va_start, - (va->va_end - va->va_start)); -free_unmap_vmap_area (va); + +if (!WARN_ON_ONCE (!va)) { +debug_check_no_locks_freed ((void *)va->va_start, + (va->va_end - va->va_start)); +free_unmap_vmap_area (va); + } }EXPORT_SYMBOL (vm_unmap_ram); --
Quoted 2 lines
2.30.2
Absolutely in favour of this in principle (BUG_ON() is something we should resort to in only the direst of circumstances), one small nit - perhaps it'd be neater to simply make this a guard clause? E.g.:-
if (!WARN_ON_ONCE(!va)) return; ...
Re: [PATCH 2/2] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
On Tue, Dec 20, 2022 at 06:53:18PM +0000, Lorenzo Stoakes wrote:
[snip
Quoted 8 lines
Absolutely in favour of this in principle (
BUG_ON()is something we should resort to in only the direst of circumstances), one small nit - perhaps it'd be neater to simply make this a guard clause? E.g.:-if (!WARN_ON_ONCE(!va)) return; ...
Made a mistake here, meant to say
if (WARN_ON_ONCE(!va))
return;
Of course :)
Apologies, only just noticed you are sending a v2, perhaps something to consider for it? But will hold off further review until you send it! :)
Uladzislau Rezki
maintainerRe: [PATCH 2/2] mm: vmalloc: Replace BUG_ON() by WARN_ON_ONCE()
Quoted 14 lines · Lorenzo Stoakes
On Tue, Dec 20, 2022 at 06:53:18PM +0000, Lorenzo Stoakes wrote:
[snipQuoted 8 lines
Absolutely in favour of this in principle (
BUG_ON()is something we should resort to in only the direst of circumstances), one small nit - perhaps it'd be neater to simply make this a guard clause? E.g.:-if (!WARN_ON_ONCE(!va)) return; ...Made a mistake here, meant to say
Review
Quoted 2 lines
(quoted diff) +0
-0if (WARN_ON_ONCE(!va)) return;
Agree. This looks better. Less confusing :)
I will post a v2.
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 132 lines · Uladzislau Rezki (Sony)
When a VA is freed over a main path, for example by invoking the
vfree()function, a tree is accessed two times what is odd:
vfree():
__vunmap()__find_vmap_area() vm_remove_mappings() remove_vm_area() __find_vmap_area()
__find_vmap_area()are called two times. Fix it by introducing afind_unlink_vmap_area()helper that finds and un-links a VA from a tree.Performance test results on a single CPU:
- fix_size_alloc_test loops: 1000000 avg: 476847 usec - full_fit_alloc_test loops: 1000000 avg: 806746 usec - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec - random_size_alloc_test loops: 1000000 avg: 7441322 usec - fix_align_alloc_test loops: 1000000 avg: 1411132 usecAll test took worker0=87650866284 cycles
- fix_size_alloc_test loops: 1000000 avg: 490713 usec - full_fit_alloc_test loops: 1000000 avg: 579162 usec - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec - random_size_alloc_test loops: 1000000 avg: 5824449 usec - fix_align_alloc_test loops: 1000000 avg: 984735 usecAll test took worker0=67952362802 cycles
Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-)mm/vmalloc.c +28
-12@@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +/* +* A free_vmap_block() is left. It is NOT a main free path. +*/ +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + } nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));@@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { + va =find_unlink_vmap_area ((unsigned long ) addr); +if (va) {struct vm_struct *vm = va->vm; - va->vm =NULL ; -spin_unlock (&vmap_area_lock); -kasan_free_module_shadow (vm);free_unmap_vmap_area (va);return vm; } -spin_unlock (&vmap_area_lock);return NULL ; }@@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;@@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ; - area =find_vm_area (addr); + va =find_unlink_vmap_area ((unsigned long )addr); + area = va->vm; +if (unlikely (!area)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);Signature · 1 line
2.30.2
Will send a v2.
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 123 lines · Uladzislau Rezki
Quoted 122 lines · Uladzislau Rezki (Sony)
When a VA is freed over a main path, for example by invoking the
vfree()function, a tree is accessed two times what is odd:
vfree():
__vunmap()__find_vmap_area() vm_remove_mappings() remove_vm_area() __find_vmap_area()
__find_vmap_area()are called two times. Fix it by introducing afind_unlink_vmap_area()helper that finds and un-links a VA from a tree.Performance test results on a single CPU:
- fix_size_alloc_test loops: 1000000 avg: 476847 usec - full_fit_alloc_test loops: 1000000 avg: 806746 usec - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec - random_size_alloc_test loops: 1000000 avg: 7441322 usec - fix_align_alloc_test loops: 1000000 avg: 1411132 usecAll test took worker0=87650866284 cycles
- fix_size_alloc_test loops: 1000000 avg: 490713 usec - full_fit_alloc_test loops: 1000000 avg: 579162 usec - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec - random_size_alloc_test loops: 1000000 avg: 5824449 usec - fix_align_alloc_test loops: 1000000 avg: 984735 usecAll test took worker0=67952362802 cycles
Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-)mm/vmalloc.c +25
-11@@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va)unsigned long va_start = va->va_start;unsigned long nr_lazy; -spin_lock (&vmap_area_lock); -unlink_va (va, &vmap_area_root); -spin_unlock (&vmap_area_lock); +/* +* A free_vmap_block() is left. It is NOT a main free path. +*/ +if (!list_empty (&va->list)) { +spin_lock (&vmap_area_lock); +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + } nr_lazy =atomic_long_add_return ((va->va_end - va->va_start) >>PAGE_SHIFT , &vmap_lazy_nr);@@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)return va; } +static struct vmap_area *find_unlink_vmap_area (unsigned long addr) +{ +struct vmap_area *va; + +spin_lock (&vmap_area_lock); + va =__find_vmap_area (addr, &vmap_area_root); +if (va) +unlink_va (va, &vmap_area_root); +spin_unlock (&vmap_area_lock); + +return va; +} +/*** Per cpu kva allocator ***/ /* @@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)return ; } - va =find_vmap_area (addr); + va =find_unlink_vmap_area (addr);BUG_ON (!va);debug_check_no_locks_freed ((void *)va->va_start, (va->va_end - va->va_start));@@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr)might_sleep (); -spin_lock (&vmap_area_lock); - va =__find_vmap_area ((unsigned long )addr, &vmap_area_root); -if (va && va->vm) { + va =find_unlink_vmap_area ((unsigned long ) addr); +if (va) {struct vm_struct *vm = va->vm; - va->vm =NULL ; -spin_unlock (&vmap_area_lock); -kasan_free_module_shadow (vm);free_unmap_vmap_area (va);return vm; } -spin_unlock (&vmap_area_lock);return NULL ; }@@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)static void __vunmap (const void *addr,int deallocate_pages) {struct vm_struct *area; +struct vmap_area *va;if (!addr)return ;@@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) addr))return ;
Quoted 10 lines
Quoted 10 lines
mm/vmalloc.c +3
-1- area =find_vm_area (addr); + va =find_unlink_vmap_area ((unsigned long )addr); + area = va->vm; +if (unlikely (!area)) {WARN (1 ,KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n" , addr);Signature · 1 line
2.30.2
Will send a v2.
--
Uladzislau Rezki