Skip to contentSkip to conversation

rcu: Dump memory object info if callback function is invalid

PATCH In review Zhen Lei 5 participants 18 messages Nov 11, 2022 – Nov 17, 2022

Subsystems (5)
lore
Share & tools
lore

Zhen Lei

author
#0 lore raw

PATCH v2 1/1 rcu: Dump memory object info if callback function is invalid

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for rhp->func to be set to NULL. This defeats the debugging prints used by __call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---
 kernel/rcu/rcu.h      | 7 +++++++
 kernel/rcu/srcutiny.c | 1 +
 kernel/rcu/srcutree.c | 1 +
 kernel/rcu/tasks.h    | 1 +
 kernel/rcu/tiny.c     | 1 +
 kernel/rcu/tree.c     | 1 +
 6 files changed, 12 insertions(+)

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

kernel/rcu/rcu.h +7 -0
index 65704cbc9df7b3d..32ab45fabf8eebf 100644
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
 }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp)
+{
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c +1 -0
index 33adafdad261389..5e7f336baa06ae0 100644
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c +1 -0
index ca4b5dcec675bac..294972e66b31863 100644
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h +1 -0
index b0b885e071fa8dc..b7f8c67c586cdc4 100644
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tiny.c +1 -0
index bb8f7d270f01747..56e9a5d91d97ec5 100644
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
 	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c +1 -0
index 15aaff3203bf2d0..ed93ddb8203d42c 100644
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Signature · 1 line
2.25.1
#1 lore raw
Quoted 50 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for rhp->func to be set to NULL. This defeats the debugging prints used by __call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Thanks
Zqiang

Quoted 87 lines
kernel/rcu/rcu.h +7 -0
index 65704cbc9df7b3d..32ab45fabf8eebf 100644
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
 }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp)
+{
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c +1 -0
index 33adafdad261389..5e7f336baa06ae0 100644
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c +1 -0
index ca4b5dcec675bac..294972e66b31863 100644
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h +1 -0
index b0b885e071fa8dc..b7f8c67c586cdc4 100644
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tiny.c +1 -0
index bb8f7d270f01747..56e9a5d91d97ec5 100644
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
 	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c +1 -0
index 15aaff3203bf2d0..ed93ddb8203d42c 100644
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Signature · 1 line
2.25.1
#2 lore raw
Quoted 59 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 50 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for rhp->func to be set to NULL. This defeats the debugging prints used by __call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

Quoted 9 lines

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Right, thanks. I just saw the robot's report. So this patch should be dropped. I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

Quoted 95 lines

Thanks
Zqiang

Quoted 87 lines
kernel/rcu/rcu.h +7 -0
index 65704cbc9df7b3d..32ab45fabf8eebf 100644
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
 }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp)
+{
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c +1 -0
index 33adafdad261389..5e7f336baa06ae0 100644
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c +1 -0
index ca4b5dcec675bac..294972e66b31863 100644
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h +1 -0
index b0b885e071fa8dc..b7f8c67c586cdc4 100644
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tiny.c +1 -0
index bb8f7d270f01747..56e9a5d91d97ec5 100644
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
 	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c +1 -0
index 15aaff3203bf2d0..ed93ddb8203d42c 100644
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Signature · 1 line
2.25.1

.

Signature · 2 lines
Regards,
  Zhen Lei
#3 lore raw
Quoted 71 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

--
Regards,
Zhen Lei

maintainer
#4 lore raw
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

Thanx, Paul

Quoted 140 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

--
Regards,
Zhen Lei

#5 lore raw
Quoted 76 lines · Paul E. McKenney
Quoted 75 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 73 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

Quoted 6 lines
Quoted 2 lines

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

It's about to panic, so no chance to schedule.

Quoted 4 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

Quoted 145 lines

Thanx, Paul

Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
#6 lore raw
Quoted 93 lines · Leizhen (ThunderTown)
Quoted 76 lines · Paul E. McKenney
Quoted 75 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 73 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

Quoted 6 lines
Quoted 2 lines

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

It's about to panic, so no chance to schedule.

Quoted 4 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

Quoted 7 lines

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:

Oh, perhaps no new function is needed, mem_dump_obj() itself prints debugging information. I will try a mm patch first.

Quoted 154 lines

if (locked)

    return;

spin_lock(&vmap_area_lock);

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

Quoted 145 lines

Thanx, Paul

Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
maintainer
#7 lore raw
Quoted 96 lines · Leizhen (ThunderTown)
Quoted 95 lines · Leizhen (ThunderTown)
Quoted 76 lines · Paul E. McKenney
Quoted 75 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 73 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

Quoted 6 lines
Quoted 2 lines

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

It's about to panic, so no chance to schedule.

Quoted 4 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Quoted 6 lines
Quoted 3 lines

Careful analysis is required, which may cause other problems.

Or in new function:

Oh, perhaps no new function is needed, mem_dump_obj() itself prints debugging information. I will try a mm patch first.

That does sound very worth trying!

Thanx, Paul

Quoted 158 lines
Quoted 154 lines

if (locked)

    return;

spin_lock(&vmap_area_lock);

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

Quoted 145 lines

Thanx, Paul

Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
maintainer
#8 lore raw
In reply to Leizhen (ThunderTown)
Quoted 77 lines · Leizhen (ThunderTown)
Quoted 76 lines · Paul E. McKenney
Quoted 75 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 73 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

Quoted 10 lines
Quoted 3 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Quoted 8 lines

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

Thanx, Paul

Quoted 146 lines
Quoted 142 lines
Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
#9 lore raw
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Leizhen (ThunderTown)
Quoted 82 lines · Paul E. McKenney
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

Quoted 7 lines
Quoted 1 line

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

OK, I think I understand what you mean.
if (!f)

	schedule_work(&work);
else
	f(rhp)
Quoted 16 lines
Quoted 16 lines
Quoted 4 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

Quoted 6 lines
Quoted 3 lines

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

OK, I got it.

Quoted 151 lines

Thanx, Paul

Quoted 137 lines
Quoted 137 lines
Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 96 lines

Thanks
Zqiang

Quoted 87 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -1
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
--
Quoted 1 line

2.25.1

.

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
maintainer
#10 lore raw
Quoted 94 lines · Leizhen (ThunderTown)
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Leizhen (ThunderTown)
Quoted 82 lines · Paul E. McKenney
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

Quoted 7 lines
Quoted 1 line

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

Quoted 5 lines

OK, I think I understand what you mean.
if (!f)

	schedule_work(&work);
else
	f(rhp)

Yes, except that the "schedule_work()" also needs to be accompanied by something that refuses to execute the rest of those callbacks. This needs to break out of the loop (or return) and to adjust counts, among other things. This might be as easy as setting count to the negative of the length of the "rcl" list, but does need some attention to the code following the callback-invocation loop.

Thanx, Paul

Quoted 179 lines
Quoted 15 lines
Quoted 15 lines
Quoted 3 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

Quoted 6 lines
Quoted 3 lines

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

OK, I got it.

Quoted 151 lines

Thanx, Paul

Quoted 137 lines
Quoted 137 lines
Quoted 137 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 96 lines

Thanks
Zqiang

Quoted 87 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -1
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
--
Quoted 1 line

2.25.1

.

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

--
Regards,
Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
#11 lore raw
Quoted 101 lines · Paul E. McKenney
Quoted 94 lines · Leizhen (ThunderTown)
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Leizhen (ThunderTown)
Quoted 82 lines · Paul E. McKenney
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

Quoted 7 lines
Quoted 1 line

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

Quoted 5 lines

OK, I think I understand what you mean.
if (!f)

	schedule_work(&work);
else
	f(rhp)
Quoted 6 lines

Yes, except that the "schedule_work()" also needs to be accompanied by something that refuses to execute the rest of those callbacks. This needs to break out of the loop (or return) and to adjust counts, among other things. This might be as easy as setting count to the negative of the length of the "rcl" list, but does need some attention to the code following the callback-invocation loop.

Yes, doing so would cause other problems. As you mentioned, the ->next pointer is likely also trashed. Some nodes may need to be executed in sequence. For such a weak debug function, it's not worth the risk, or overly complicated thinking.

Quoted 24 lines

Thanx, Paul

Quoted 21 lines
Quoted 21 lines
Quoted 10 lines
Quoted 3 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Quoted 8 lines

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

The memory modules are mature and stable, so your concerns may not be true.

Quoted 160 lines
Quoted 146 lines

OK, I got it.

Quoted 143 lines

Thanx, Paul

Quoted 140 lines
Quoted 140 lines
Quoted 140 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

--
Regards,
Zhen Lei

Quoted 12 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
maintainer
#12 lore raw
Quoted 109 lines · Leizhen (ThunderTown)
Quoted 101 lines · Paul E. McKenney
Quoted 94 lines · Leizhen (ThunderTown)
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Leizhen (ThunderTown)
Quoted 82 lines · Paul E. McKenney
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

Quoted 7 lines
Quoted 1 line

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

Quoted 5 lines

OK, I think I understand what you mean.
if (!f)

	schedule_work(&work);
else
	f(rhp)
Quoted 6 lines

Yes, except that the "schedule_work()" also needs to be accompanied by something that refuses to execute the rest of those callbacks. This needs to break out of the loop (or return) and to adjust counts, among other things. This might be as easy as setting count to the negative of the length of the "rcl" list, but does need some attention to the code following the callback-invocation loop.

Quoted 4 lines

Yes, doing so would cause other problems. As you mentioned, the ->next pointer is likely also trashed. Some nodes may need to be executed in sequence. For such a weak debug function, it's not worth the risk, or overly complicated thinking.

Do we have similar deadlock issues with the calls to mem_dump_obj() in the call_rcu() code path? These are somewhat less concerning because they are invoked under a Kconfig option that is (as far as I know) rarely set, but still...

Thanx, Paul

Quoted 188 lines
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 3 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

The memory modules are mature and stable, so your concerns may not be true.

Quoted 160 lines
Quoted 146 lines

OK, I got it.

Quoted 143 lines

Thanx, Paul

Quoted 140 lines
Quoted 140 lines
Quoted 140 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

--
Regards,
Zhen Lei

Quoted 12 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
#13 lore raw
Quoted 109 lines · Leizhen (ThunderTown)
Quoted 101 lines · Paul E. McKenney
Quoted 94 lines · Leizhen (ThunderTown)
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Leizhen (ThunderTown)
Quoted 82 lines · Paul E. McKenney
Quoted 72 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 70 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

Quoted 7 lines
Quoted 1 line

It's about to panic, so no chance to schedule.

It won't panic if you drop the callback on the floor.

Though to your point, the ->next pointer is likely also trashed. So you could just drop the remainder of the callback list on the floor. That might provide a good (though not perfect) chance of getting decent output.

Quoted 5 lines

OK, I think I understand what you mean.
if (!f)

	schedule_work(&work);
else
	f(rhp)
Quoted 6 lines

Yes, except that the "schedule_work()" also needs to be accompanied by something that refuses to execute the rest of those callbacks. This needs to break out of the loop (or return) and to adjust counts, among other things. This might be as easy as setting count to the negative of the length of the "rcl" list, but does need some attention to the code following the callback-invocation loop.

Quoted 7 lines

Yes, doing so would cause other problems. As you mentioned, the ->next pointer is likely also trashed. Some nodes may need to be executed in sequence. For such a weak debug function, it's not worth the risk, or overly complicated thinking.

Do we have similar deadlock issues with the calls to mem_dump_obj() in the call_rcu() code path?

I think it exists, and also consider PREEMPT_RT kernel , because the vmap_area_lock spinlock convert to sleepable lock. I have sent a version of the patch.

Thanks
Zqiang

Quoted 194 lines

These are somewhat less concerning because
they are invoked under a Kconfig option that is (as far as I know) rarely set, but still...

Thanx, Paul

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 3 lines

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

locked = spin_is_locked(&vmap_area_lock);
if (!locked)
spin_lock(&vmap_area_lock)

Careful analysis is required, which may cause other problems.

Or in new function:
if (locked)

    return;

spin_lock(&vmap_area_lock);

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines

If there is a chance to dump the data, dump the data. If there is no chance to dump the data, do not dump the data. This is the fate of debugging information.

My concern is that there will be increasing numbers of special cases over time.

The memory modules are mature and stable, so your concerns may not be true.

Quoted 160 lines
Quoted 146 lines

OK, I got it.

Quoted 143 lines

Thanx, Paul

Quoted 140 lines
Quoted 140 lines
Quoted 140 lines
mm/util.c +8 -4
index 12984e76767e..86da0739fe5d 100644
@@ -1119,14 +1119,18 @@ void mem_dump_obj(void *object)
 {
        const char *type;
 
+       if (is_vmalloc_addr(object)) {
+               if (in_task() && vmalloc_dump_obj(object))
+                       return;
+               type = "vmalloc memory";
+               goto end;
+       }
+
        if (kmem_valid_obj(object)) {
                kmem_dump_obj(object);
                return;
        }
 
-       if (vmalloc_dump_obj(object))
-               return;
-
        if (virt_addr_valid(object))
                type = "non-slab/vmalloc memory";
        else if (object == NULL)
@@ -1135,7 +1139,7 @@ void mem_dump_obj(void *object)
                type = "zero-size pointer";
        else
                type = "non-paged memory";
-
+end:
        pr_cont(" %s\n", type);
 }
 EXPORT_SYMBOL_GPL(mem_dump_obj);

Thanks
Zqiang

Quoted 91 lines

Thanks
Zqiang

Quoted 86 lines
kernel/rcu/rcu.h index renamed from kernel/rcu/rcu.h +0 -0

65704cbc9df7b3d..32ab45fabf8eebf 100644

kernel/rcu/rcu.h +6 -0
@@ -10,6 +10,7 @@
 #ifndef __LINUX_RCU_H
 #define __LINUX_RCU_H
 
+#include <linux/mm.h>
 #include <trace/events/rcu.h>
 
 /*
@@ -211,6 +212,12 @@ static inline void debug_rcu_head_unqueue(struct 
 rcu_head *head) }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
+static inline void debug_rcu_head_callback(struct rcu_head *rhp) {
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+
 extern int rcu_cpu_stall_suppress_at_boot;
 
 static inline bool rcu_stall_is_suppressed_at_boot(void)
kernel/rcu/srcutiny.c index renamed from kernel/rcu/srcutiny.c +0 -0

33adafdad261389..5e7f336baa06ae0 100644

kernel/rcu/srcutiny.c +1 -0
@@ -138,6 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	while (lh) {
 		rhp = lh;
 		lh = lh->next;
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/srcutree.c index renamed from kernel/rcu/srcutree.c +0 -0

ca4b5dcec675bac..294972e66b31863 100644

kernel/rcu/srcutree.c +1 -0
@@ -1631,6 +1631,7 @@ static void srcu_invoke_callbacks(struct work_struct *work)
 	rhp = rcu_cblist_dequeue(&ready_cbs);
 	for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {
 		debug_rcu_head_unqueue(rhp);
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
 		local_bh_enable();
kernel/rcu/tasks.h index renamed from kernel/rcu/tasks.h +0 -0

b0b885e071fa8dc..b7f8c67c586cdc4 100644

kernel/rcu/tasks.h +1 -0
@@ -478,6 +478,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu
 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
 	len = rcl.len;
 	for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = 
 rcu_cblist_dequeue(&rcl)) {
+		debug_rcu_head_callback(rhp);
 		local_bh_disable();
 		rhp->func(rhp);
		local_bh_enable();
kernel/rcu/tiny.c index renamed from kernel/rcu/tiny.c +0 -0

bb8f7d270f01747..56e9a5d91d97ec5 100644

kernel/rcu/tiny.c +1 -0
@@ -97,6 +97,7 @@ static inline bool rcu_reclaim_tiny(struct rcu_head 
 *head)
 
 	trace_rcu_invoke_callback("", head);
 	f = head->func;
+	debug_rcu_head_callback(head);
 	WRITE_ONCE(head->func, (rcu_callback_t)0L);
 	f(head);
	rcu_lock_release(&rcu_callback_map);
kernel/rcu/tree.c index renamed from kernel/rcu/tree.c +0 -0

15aaff3203bf2d0..ed93ddb8203d42c 100644

kernel/rcu/tree.c +1 -0
@@ -2088,6 +2088,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
 		trace_rcu_invoke_callback(rcu_state.name, rhp);
 
 		f = rhp->func;
+		debug_rcu_head_callback(rhp);
 		WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
 		f(rhp);
 
Quoted 5 lines
Quoted 2 lines

--
2.25.1

.

--
Regards,
Zhen Lei

Quoted 12 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei

.

Signature · 2 lines
Regards,
  Zhen Lei
maintainer
#14 lore raw
In reply to Paul E. McKenney
Quoted 87 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 72 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 70 lines
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

<snip>

mm/vmalloc.c +14 -1
index ca71de7c9d77..956eb28f9c77 100644
@@ -2591,6 +2591,19 @@ struct vm_struct *find_vm_area(const void *addr)
 	return va->vm;
 }
 
+static struct vm_struct *
+find_vm_area_trylock(const void *addr)
+{
+	struct vmap_area *va = NULL;
+
+	if (spin_trylock(&vmap_area_lock)) {
+		va = __find_vmap_area((unsigned long)addr, &vmap_area_root);
+		spin_unlock(&vmap_area_lock);
+	}
+
+	return va ? va->vm : NULL;
+}
+
 /**
  * remove_vm_area - find and remove a continuous kernel virtual area
  * @addr:	    base address
@@ -4048,7 +4061,7 @@ bool vmalloc_dump_obj(void *object)
 	struct vm_struct *vm;
 	void *objp = (void *)PAGE_ALIGN((unsigned long)object);
 
-	vm = find_vm_area(objp);
+	vm = find_vm_area_trylock(objp);
 	if (!vm)
 		return false;
 	pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",

<snip>

There is one issue though, it is not correct to reference vm->members after a lock is dropped because of: use after free bugs. It is better to embed the search and read out: nr_pages, addr, caller and drop the lock.

--
Uladzislau Rezki

#15 lore raw
Quoted 98 lines · Uladzislau Rezki
Quoted 87 lines · On Fri, Nov 11, 2022 at 01:05:56PM +0000, Zhang, Qiang1 wrote:
Quoted 72 lines · On 2022/11/11 19:54, Zhang, Qiang1 wrote:
Quoted 70 lines
Quoted 52 lines

When a structure containing an RCU callback rhp is (incorrectly) freed and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used rhp->by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, which expect to identify the offending code using the identity of this function.

And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things are even worse, as can be seen from this splat:

Unable to handle kernel NULL pointer dereference at virtual address 0 

... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284) (rcu_core) from (__do_softirq+0x24c/0x344) (__do_softirq) from (__irq_exit_rcu+0x64/0x108) (__irq_exit_rcu) from (irq_exit+0x8/0x10) (irq_exit) from (__handle_domain_irq+0x74/0x9c) (__handle_domain_irq) from (gic_handle_irq+0x8c/0x98) (gic_handle_irq) from (__irq_svc+0x5c/0x94) (__irq_svc) from (arch_cpu_idle+0x20/0x3c) (arch_cpu_idle) from (default_idle_call+0x4c/0x78) (default_idle_call) from (do_idle+0xf8/0x150) (do_idle) from (cpu_startup_entry+0x18/0x20) (cpu_startup_entry) from (0xc01530)

This commit therefore adds calls to mem_dump_obj(rhp) to output some information, for example:

slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256

This provides the rough size of the memory block and the offset of the rcu_head structure, which as least provides at least a few clues to help locate the problem. If the problem is reproducible, additional slab debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can provide significantly more information.

---

kernel/rcu/rcu.h      | 7 +++++++
kernel/rcu/srcutiny.c | 1 +
kernel/rcu/srcutree.c | 1 +
kernel/rcu/tasks.h    | 1 +
kernel/rcu/tiny.c     | 1 +
kernel/rcu/tree.c     | 1 +
6 files changed, 12 insertions(+)
Quoted 4 lines

v1 --> v2:
1. Remove condition "(unsigned long)rhp->func & 0x3", it have problems on x86.
2. Paul E. McKenney helped me update the commit message, thanks.

Hi, Zhen Lei

Maybe the following scenarios should be considered:

CPU 0
tasks context
spin_lock(&vmap_area_lock)
Interrupt
RCU softirq
rcu_do_batch
mem_dump_obj
vmalloc_dump_obj
spin_lock(&vmap_area_lock) <-- deadlock

Quoted 5 lines
Quoted 2 lines

Right, thanks. I just saw the robot's report. So this patch should be dropped.
I'll try to add an helper in mm, where I can check whether the lock has been held, and dump the content of memory object.

This is a workaround, or maybe try a modification like the following, of course, need to ask Paul's opinion.

Another approach is to schedule a workqueue handler to do the mem_dump_obj(). This would allow mem_dump_obj() to run in a clean environment.

This would allow vmalloc_dump_obj() to be called unconditionally.

Other thoughts?

<snip>

mm/vmalloc.c +2 -0
index ca71de7c9d77..956eb28f9c77 100644
@@ -2591,6 +2591,19 @@ struct vm_struct *find_vm_area(const void *addr)
 	return va->vm;
 }
 
+static struct vm_struct *
+find_vm_area_trylock(const void *addr)

mm/vmalloc.c struct vm_struct *find_vm_area(const void *addr) · line 2605

Quoted 10 lines
mm/vmalloc.c +10 -0
+{
+	struct vmap_area *va = NULL;
+
+	if (spin_trylock(&vmap_area_lock)) {
+		va = __find_vmap_area((unsigned long)addr, &vmap_area_root);
+		spin_unlock(&vmap_area_lock);
+	}
+
+	return va ? va->vm : NULL;
+}

This method also has the problems mentioned by Andrew Morton.
https://lkml.org/lkml/2022/11/14/1238

Quoted 24 lines
mm/vmalloc.c +2 -1
+
 /**
  * remove_vm_area - find and remove a continuous kernel virtual area
  * @addr:	    base address
@@ -4048,7 +4061,7 @@ bool vmalloc_dump_obj(void *object)
 	struct vm_struct *vm;
 	void *objp = (void *)PAGE_ALIGN((unsigned long)object);
 
-	vm = find_vm_area(objp);
+	vm = find_vm_area_trylock(objp);
 	if (!vm)
 		return false;
 	pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
<snip>

There is one issue though, it is not correct to reference vm->members after a lock is dropped because of: use after free bugs. It is better to embed the search and read out: nr_pages, addr, caller and drop the lock.

--
Uladzislau Rezki
.

Signature · 2 lines
Regards,
  Zhen Lei
#16 lore raw
In reply to Zhen Lei

kernel/rcu/rcu.h static inline void debug_rcu_head_unqueue(struct rcu_head *head) · line 220

Quoted 8 lines
kernel/rcu/rcu.h +6 -0
+static inline void debug_rcu_head_callback(struct rcu_head *rhp)
+{
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+

The mm/util.c definition of mem_dump_object() says:
* This function uses pr_cont(), so that the caller is expected to have
* printed out whatever preamble is appropriate.

so this needs to call pr_alert() or pr_err() before that to explain what is being printed (with no \n), like these:

kernel/rcu/rcutorture.c: pr_alert("mem_dump_obj(%px):", &rhp); kernel/rcu/rcutorture.c: mem_dump_obj(&rhp);
...
kernel/rcu/tree.c: pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); kernel/rcu/tree.c: mem_dump_obj(head);

#17 lore raw
Quoted 13 lines · Robert (Servers)

kernel/rcu/rcu.h static inline void debug_rcu_head_unqueue(struct rcu_head *head) · line 220

Quoted 8 lines
kernel/rcu/rcu.h +6 -0
+static inline void debug_rcu_head_callback(struct rcu_head *rhp)
+{
+	if (unlikely(!rhp->func))
+		mem_dump_obj(rhp);
+}
+

The mm/util.c definition of mem_dump_object() says:
* This function uses pr_cont(), so that the caller is expected to have
* printed out whatever preamble is appropriate.

Quoted 8 lines

so this needs to call pr_alert() or pr_err() before that to explain what is being printed (with no \n), like these:

kernel/rcu/rcutorture.c: pr_alert("mem_dump_obj(%px):", &rhp); kernel/rcu/rcutorture.c: mem_dump_obj(&rhp);
...
kernel/rcu/tree.c: pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); kernel/rcu/tree.c: mem_dump_obj(head);

Yes, right, thanks.

Quoted 4 lines

.

Signature · 2 lines
Regards,
  Zhen Lei