kmemleak: Turn kmemleak_lock and object->lock to raw_spinlock_t
Subsystems (2)
PATCH 1/1 kmemleak: Turn kmemleak_lock and object->lock to raw_spinlock_t
From: He Zhe <[email protected]>
kmemleak_lock as a rwlock on RT can possibly be acquired in atomic context which does work.
Since the kmemleak operation is performed in atomic context make it a raw_spinlock_t so it can also be acquired on RT. This is used for debugging and is not enabled by default in a production like environment (where performance/latency matters) so it makes sense to make it a raw_spinlock_t instead trying to get rid of the atomic context. Turn also the kmemleak_object->lock into raw_spinlock_t which is acquired (nested) while the kmemleak_lock is held.
The time spent in "echo scan > kmemleak" slightly improved on 64core box with this patch applied after boot.
[bigeasy: Redo the description, update comments. Merge the individual bits:
He Zhe did the kmemleak_lock, Liu Haitao the ->lock and Yongxin Liu forwarded Liu's patch.]
---
Catalin's ACK is from
https://lore.kernel.org/linux-mm/[email protected]/
My numbers/testing:
test: root@box:/sys/kernel/debug# for a in 1 2 3 4 5; do time echo scan > kmemleak; done without patch:
|real 0m12.742s |user 0m0.000s |sys 0m12.741s | |real 0m12.732s |user 0m0.000s |sys 0m12.732s | |real 0m12.731s |user 0m0.000s |sys 0m12.731s | |real 0m21.228s |user 0m0.000s |sys 0m12.866s | |real 0m12.736s |user 0m0.000s |sys 0m12.736s
patch applied:
|real 0m10.396s |user 0m0.000s |sys 0m10.396s | |real 0m10.387s |user 0m0.000s |sys 0m10.387s | |real 0m18.134s |user 0m0.000s |sys 0m10.519s | |real 0m10.379s |user 0m0.000s |sys 0m10.380s | |real 0m10.389s |user 0m0.000s |sys 0m10.389s
mm/kmemleak.c | 112 +++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 56 deletions(-)
mm/kmemleak.c +56 -56
@@ -13,7 +13,7 @@* * The following locks and mutexes are used by kmemleak: * -* - kmemleak_lock (rwlock): protects the object_list modifications and +* - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and * accesses to the object_tree_root. The object_list is the main list * holding the metadata (struct kmemleak_object) for the allocated memory * blocks. The object_tree_root is a red black tree used to look-up
@@ -22,13 +22,13 @@* object_tree_root in the create_object() function called from the * kmemleak_alloc() callback and removed in delete_object() called from the * kmemleak_free() callback -* - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to -* the metadata (e.g. count) are protected by this lock. Note that some -* members of this structure may be protected by other means (atomic or -* kmemleak_lock). This lock is also held when scanning the corresponding -* memory block to avoid the kernel freeing it via the kmemleak_free() -* callback. This is less heavyweight than holding a global lock like -* kmemleak_lock during scanning +* - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object. +* Accesses to the metadata (e.g. count) are protected by this lock. Note +* that some members of this structure may be protected by other means +* (atomic or kmemleak_lock). This lock is also held when scanning the +* corresponding memory block to avoid the kernel freeing it via the +* kmemleak_free() callback. This is less heavyweight than holding a global +* lock like kmemleak_lock during scanning. * - scan_mutex (mutex): ensures that only one thread may scan the memory for * unreferenced objects at a time. The gray_list contains the objects which * are already referenced or marked as false positives and need to be
@@ -135,7 +135,7 @@ struct kmemleak_scan_area {* (use_count) and freed using the RCU mechanism. */ struct kmemleak_object { -spinlock_t lock; +raw_spinlock_t lock;unsigned int flags;/* object status flags */ struct list_head object_list;struct list_head gray_list;
@@ -191,8 +191,8 @@ static int mem_pool_free_count = ARRAY_SIZE(mem_pool);static LIST_HEAD (mem_pool_free_list);/* search tree for object boundaries */ static struct rb_root object_tree_root =RB_ROOT ; -/* rw_lock protecting the access to object_list and object_tree_root */ -static DEFINE_RWLOCK (kmemleak_lock); +/* protecting the access to object_list and object_tree_root */ +static DEFINE_RAW_SPINLOCK (kmemleak_lock);/* allocation caches for kmemleak internal data */ static struct kmem_cache *object_cache;
@@ -426,7 +426,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp) }/* slab allocation failed, try the memory pool */ -write_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags); object =list_first_entry_or_null (&mem_pool_free_list,typeof (*object), object_list);if (object)
@@ -435,7 +435,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp) object = &mem_pool[--mem_pool_free_count]; else pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n"); -write_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags);return object; }
@@ -453,9 +453,9 @@ static void mem_pool_free(struct kmemleak_object *object) }/* add the object to the memory pool free list */ -write_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags);list_add (&object->object_list, &mem_pool_free_list); -write_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags); }/*
@@ -514,9 +514,9 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)struct kmemleak_object *object;rcu_read_lock (); -read_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags); object =lookup_object (ptr, alias); -read_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags);/* check whether the object is still available */ if (object && !get_object (object))
@@ -546,11 +546,11 @@ static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int aliunsigned long flags;struct kmemleak_object *object; -write_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags); object =lookup_object (ptr, alias);if (object)__remove_object (object); -write_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags);return object; }
@@ -585,7 +585,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,INIT_LIST_HEAD (&object->object_list);INIT_LIST_HEAD (&object->gray_list);INIT_HLIST_HEAD (&object->area_list); -spin_lock_init (&object->lock); +raw_spin_lock_init (&object->lock);atomic_set (&object->use_count,1 ); object->flags =OBJECT_ALLOCATED ; object->pointer = ptr;
@@ -617,7 +617,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,/* kernel backtrace */ object->trace_len =__save_stack_trace (object->trace); -write_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags); untagged_ptr = (unsigned long )kasan_reset_tag ((void *)ptr); min_addr =min (min_addr, untagged_ptr);
@@ -649,7 +649,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,list_add_tail_rcu (&object->object_list, &object_list); out: -write_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags);return object; }
@@ -667,9 +667,9 @@ static void __delete_object(struct kmemleak_object *object)* Locking here also ensures that the corresponding memory block * cannot be freed when it is being scanned. */ -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags); object->flags &= ~OBJECT_ALLOCATED ; -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object); }
@@ -739,9 +739,9 @@ static void paint_it(struct kmemleak_object *object, int color) {unsigned long flags; -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);__paint_it (object, color); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }static void paint_ptr (unsigned long ptr,int color)
@@ -798,7 +798,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)if (scan_area_cache) area =kmem_cache_alloc (scan_area_cache,gfp_kmemleak_mask (gfp)); -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if (!area) {pr_warn_once ("Cannot allocate a scan area, scanning the full object\n" );/* mark the object for full scan to avoid false positives */
@@ -820,7 +820,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)hlist_add_head (&area->node, &object->area_list); out_unlock: -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object); }
@@ -842,9 +842,9 @@ static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)return ; } -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags); object->excess_ref = excess_ref; -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object); }
@@ -864,9 +864,9 @@ static void object_no_scan(unsigned long ptr)return ; } -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags); object->flags |=OBJECT_NO_SCAN ; -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object); }
@@ -1026,9 +1026,9 @@ void __ref kmemleak_update_trace(const void *ptr)return ; } -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags); object->trace_len =__save_stack_trace (object->trace); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object); }
@@ -1233,7 +1233,7 @@ static void scan_block(void *_start, void *_end,unsigned long flags;unsigned long untagged_ptr; -read_lock_irqsave (&kmemleak_lock, flags); +raw_spin_lock_irqsave (&kmemleak_lock, flags);for (ptr = start; ptr < end; ptr++) {struct kmemleak_object *object;unsigned long pointer;
@@ -1268,7 +1268,7 @@ static void scan_block(void *_start, void *_end,* previously acquired in scan_object(). These locks are * enclosed by scan_mutex. */ -spin_lock_nested (&object->lock,SINGLE_DEPTH_NESTING ); +raw_spin_lock_nested (&object->lock,SINGLE_DEPTH_NESTING );/* only pass surplus references (object already gray) */ if (color_gray (object)) { excess_ref = object->excess_ref;
@@ -1277,7 +1277,7 @@ static void scan_block(void *_start, void *_end, excess_ref =0 ;update_refs (object); } -spin_unlock (&object->lock); +raw_spin_unlock (&object->lock);if (excess_ref) { object =lookup_object (excess_ref,0 );
@@ -1286,12 +1286,12 @@ static void scan_block(void *_start, void *_end,if (object == scanned)/* circular reference, ignore */ continue ; -spin_lock_nested (&object->lock,SINGLE_DEPTH_NESTING ); +raw_spin_lock_nested (&object->lock,SINGLE_DEPTH_NESTING );update_refs (object); -spin_unlock (&object->lock); +raw_spin_unlock (&object->lock); } } -read_unlock_irqrestore (&kmemleak_lock, flags); +raw_spin_unlock_irqrestore (&kmemleak_lock, flags); }/*
@@ -1324,7 +1324,7 @@ static void scan_object(struct kmemleak_object *object)* Once the object->lock is acquired, the corresponding memory block * cannot be freed (the same lock is acquired in delete_object). */ -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if (object->flags &OBJECT_NO_SCAN ) goto out;if (!(object->flags &OBJECT_ALLOCATED ))
@@ -1344,9 +1344,9 @@ static void scan_object(struct kmemleak_object *object)if (start >= end)break ; -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);cond_resched (); -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags); }while (object->flags &OBJECT_ALLOCATED ); }else hlist_for_each_entry (area, &object->area_list, node)
@@ -1354,7 +1354,7 @@ static void scan_object(struct kmemleak_object *object) (void *)(area->start + area->size), object); out: -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }/*
@@ -1407,7 +1407,7 @@ static void kmemleak_scan(void)/* prepare the kmemleak_object's */ rcu_read_lock ();list_for_each_entry_rcu (object, &object_list, object_list) { -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);#ifdef DEBUG /* * With a few exceptions there should be a maximum of
@@ -1424,7 +1424,7 @@ static void kmemleak_scan(void)if (color_gray (object) &&get_object (object))list_add_tail (&object->gray_list, &gray_list); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }rcu_read_unlock ();
@@ -1492,14 +1492,14 @@ static void kmemleak_scan(void)*/ rcu_read_lock ();list_for_each_entry_rcu (object, &object_list, object_list) { -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if (color_white (object) && (object->flags &OBJECT_ALLOCATED ) &&update_checksum (object) &&get_object (object)) {/* color it gray temporarily */ object->count = object->min_count;list_add_tail (&object->gray_list, &gray_list); } -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }rcu_read_unlock ();
@@ -1519,7 +1519,7 @@ static void kmemleak_scan(void)*/ rcu_read_lock ();list_for_each_entry_rcu (object, &object_list, object_list) { -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if (unreferenced_object (object) && !(object->flags &OBJECT_REPORTED )) { object->flags |=OBJECT_REPORTED ;
@@ -1529,7 +1529,7 @@ static void kmemleak_scan(void) new_leaks++; } -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }rcu_read_unlock ();
@@ -1681,10 +1681,10 @@ static int kmemleak_seq_show(struct seq_file *seq, void *v)struct kmemleak_object *object = v;unsigned long flags; -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if ((object->flags &OBJECT_REPORTED ) &&unreferenced_object (object))print_unreferenced (seq, object); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);return 0 ; }
@@ -1714,9 +1714,9 @@ static int dump_str_object_info(const char *str)return -EINVAL ; } -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);dump_object_info (object); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags);put_object (object);return 0 ;
@@ -1735,11 +1735,11 @@ static void kmemleak_clear(void)rcu_read_lock ();list_for_each_entry_rcu (object, &object_list, object_list) { -spin_lock_irqsave (&object->lock, flags); +raw_spin_lock_irqsave (&object->lock, flags);if ((object->flags &OBJECT_REPORTED ) &&unreferenced_object (object))__paint_it (object,KMEMLEAK_GREY ); -spin_unlock_irqrestore (&object->lock, flags); +raw_spin_unlock_irqrestore (&object->lock, flags); }rcu_read_unlock ();
Signature · 1 line
2.24.0