rcu: Fix data-race due to atomic_t copy-by-value
PATCH 1/1 rcu: Fix data-race due to atomic_t copy-by-value
This fixes a data-race where atomic_t dynticks is copied by value. The copy is performed non-atomically, resulting in a data-race if dynticks is updated concurrently.
This data-race was found with KCSAN:
==================================================================
BUG: KCSAN: data-race in dyntick_save_progress_counter / rcu_irq_enter
write to 0xffff989dbdbe98e0 of 4 bytes by task 10 on cpu 3:
atomic_add_return include/asm-generic/atomic-instrumented.h:78 [inline] rcu_dynticks_snap kernel/rcu/tree.c:310 [inline] dyntick_save_progress_counter+0x43/0x1b0 kernel/rcu/tree.c:984 force_qs_rnp+0x183/0x200 kernel/rcu/tree.c:2286 rcu_gp_fqs kernel/rcu/tree.c:1601 [inline] rcu_gp_fqs_loop+0x71/0x880 kernel/rcu/tree.c:1653 rcu_gp_kthread+0x22c/0x3b0 kernel/rcu/tree.c:1799 kthread+0x1b5/0x200 kernel/kthread.c:255 <snip>
read to 0xffff989dbdbe98e0 of 4 bytes by task 154 on cpu 7:
rcu_nmi_enter_common kernel/rcu/tree.c:828 [inline] rcu_irq_enter+0xda/0x240 kernel/rcu/tree.c:870 irq_enter+0x5/0x50 kernel/softirq.c:347 <snip>
Reported by Kernel Concurrency Sanitizer on:
CPU: 7 PID: 154 Comm: kworker/7:1H Not tainted 5.3.0+ #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Workqueue: kblockd blk_mq_run_work_fn ==================================================================
--- include/trace/events/rcu.h | 4 ++-- kernel/rcu/tree.c | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-)
include/trace/events/rcu.h +2 -2
@@ -442,7 +442,7 @@ TRACE_EVENT_RCU(rcu_fqs,*/ TRACE_EVENT_RCU(rcu_dyntick, - TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks), + TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks), TP_ARGS(polarity, oldnesting, newnesting, dynticks),
@@ -457,7 +457,7 @@ TRACE_EVENT_RCU(rcu_dyntick, __entry->polarity = polarity; __entry->oldnesting = oldnesting; __entry->newnesting = newnesting; - __entry->dynticks =atomic_read (&dynticks); + __entry->dynticks = dynticks; ), TP_printk("%s %lx %lx %#3x", __entry->polarity,
kernel/rcu/tree.c +6 -5
@@ -576,7 +576,7 @@ static void rcu_eqs_enter(bool user) }lockdep_assert_irqs_disabled (); -trace_rcu_dyntick (TPS ("Start" ), rdp->dynticks_nesting,0 , rdp->dynticks); +trace_rcu_dyntick (TPS ("Start" ), rdp->dynticks_nesting,0 ,atomic_read (&rdp->dynticks));WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current)); rdp =this_cpu_ptr (&rcu_data);do_nocb_deferred_wakeup (rdp);
@@ -649,14 +649,15 @@ static __always_inline void rcu_nmi_exit_common(bool irq)* leave it in non-RCU-idle state. */ if (rdp->dynticks_nmi_nesting !=1 ) { -trace_rcu_dyntick (TPS ("--=" ), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting -2 , rdp->dynticks); +trace_rcu_dyntick (TPS ("--=" ), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting -2 , +atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,/* No store tearing. */ rdp->dynticks_nmi_nesting -2 );return ; }/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ -trace_rcu_dyntick (TPS ("Startirq" ), rdp->dynticks_nmi_nesting,0 , rdp->dynticks); +trace_rcu_dyntick (TPS ("Startirq" ), rdp->dynticks_nmi_nesting,0 ,atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,0 );/* Avoid store tearing. */ if (irq)
@@ -743,7 +744,7 @@ static void rcu_eqs_exit(bool user)rcu_dynticks_task_exit ();rcu_dynticks_eqs_exit ();rcu_cleanup_after_idle (); -trace_rcu_dyntick (TPS ("End" ), rdp->dynticks_nesting,1 , rdp->dynticks); +trace_rcu_dyntick (TPS ("End" ), rdp->dynticks_nesting,1 ,atomic_read (&rdp->dynticks));WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current));WRITE_ONCE (rdp->dynticks_nesting,1 );WARN_ON_ONCE (rdp->dynticks_nmi_nesting);
@@ -827,7 +828,7 @@ static __always_inline void rcu_nmi_enter_common(bool irq) }trace_rcu_dyntick (incby ==1 ?TPS ("Endirq" ) :TPS ("++=" ), rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, rdp->dynticks); + rdp->dynticks_nmi_nesting + incby,atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,/* Prevent store tearing. */ rdp->dynticks_nmi_nesting + incby);barrier ();
Signature · 1 line
2.23.0.581.g78d2f28ef7-goog
Paul E. McKenney
maintainerQuoted 25 lines · Marco Elver
This fixes a data-race where
atomic_t dynticksis copied by value. The copy is performed non-atomically, resulting in a data-race ifdynticksis updated concurrently.This data-race was found with KCSAN:
==================================================================BUG: KCSAN: data-race in dyntick_save_progress_counter / rcu_irq_enterwrite to
0xffff989dbdbe98e0of 4 bytes by task 10 on cpu 3:atomic_add_return include/asm-generic/atomic-instrumented.h:78 [inline] rcu_dynticks_snap kernel/rcu/tree.c:310 [inline] dyntick_save_progress_counter+0x43/0x1b0 kernel/rcu/tree.c:984 force_qs_rnp+0x183/0x200 kernel/rcu/tree.c:2286 rcu_gp_fqs kernel/rcu/tree.c:1601 [inline] rcu_gp_fqs_loop+0x71/0x880 kernel/rcu/tree.c:1653 rcu_gp_kthread+0x22c/0x3b0 kernel/rcu/tree.c:1799 kthread+0x1b5/0x200 kernel/kthread.c:255 <snip>read to
0xffff989dbdbe98e0of 4 bytes by task 154 on cpu 7:rcu_nmi_enter_common kernel/rcu/tree.c:828 [inline] rcu_irq_enter+0xda/0x240 kernel/rcu/tree.c:870 irq_enter+0x5/0x50 kernel/softirq.c:347 <snip>
Quoted 7 lines
Reported by Kernel Concurrency Sanitizer on:
CPU: 7 PID: 154 Comm: kworker/7:1H Not tainted 5.3.0+ #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Workqueue: kblockd blk_mq_run_work_fn ==================================================================Signed-off-by Marco Elver <[email protected]>
Good catch, queued for review and testing.
Thanx, Paul
Quoted 89 lines
Cc Paul E. McKenney <[email protected]>Cc Josh Triplett <[email protected]>Cc Steven Rostedt <[email protected]>Cc Mathieu Desnoyers <[email protected]>Cc Lai Jiangshan <[email protected]>Cc Joel Fernandes <[email protected]>Cc Ingo Molnar <[email protected]>Cc Dmitry Vyukov <[email protected]>--- include/trace/events/rcu.h | 4 ++-- kernel/rcu/tree.c | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-)include/trace/events/rcu.h +2
-2@@ -442,7 +442,7 @@ TRACE_EVENT_RCU(rcu_fqs,*/ TRACE_EVENT_RCU(rcu_dyntick, - TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks), + TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks), TP_ARGS(polarity, oldnesting, newnesting, dynticks),@@ -457,7 +457,7 @@ TRACE_EVENT_RCU(rcu_dyntick, __entry->polarity = polarity; __entry->oldnesting = oldnesting; __entry->newnesting = newnesting; - __entry->dynticks =atomic_read (&dynticks); + __entry->dynticks = dynticks; ), TP_printk("%s %lx %lx %#3x", __entry->polarity,kernel/rcu/tree.c +6
-5@@ -576,7 +576,7 @@ static void rcu_eqs_enter(bool user) }lockdep_assert_irqs_disabled (); -trace_rcu_dyntick (TPS ("Start" ), rdp->dynticks_nesting,0 , rdp->dynticks); +trace_rcu_dyntick (TPS ("Start" ), rdp->dynticks_nesting,0 ,atomic_read (&rdp->dynticks));WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current)); rdp =this_cpu_ptr (&rcu_data);do_nocb_deferred_wakeup (rdp);@@ -649,14 +649,15 @@ static __always_inline void rcu_nmi_exit_common(bool irq)* leave it in non-RCU-idle state. */ if (rdp->dynticks_nmi_nesting !=1 ) { -trace_rcu_dyntick (TPS ("--=" ), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting -2 , rdp->dynticks); +trace_rcu_dyntick (TPS ("--=" ), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting -2 , +atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,/* No store tearing. */ rdp->dynticks_nmi_nesting -2 );return ; }/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ -trace_rcu_dyntick (TPS ("Startirq" ), rdp->dynticks_nmi_nesting,0 , rdp->dynticks); +trace_rcu_dyntick (TPS ("Startirq" ), rdp->dynticks_nmi_nesting,0 ,atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,0 );/* Avoid store tearing. */ if (irq)@@ -743,7 +744,7 @@ static void rcu_eqs_exit(bool user)rcu_dynticks_task_exit ();rcu_dynticks_eqs_exit ();rcu_cleanup_after_idle (); -trace_rcu_dyntick (TPS ("End" ), rdp->dynticks_nesting,1 , rdp->dynticks); +trace_rcu_dyntick (TPS ("End" ), rdp->dynticks_nesting,1 ,atomic_read (&rdp->dynticks));WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current));WRITE_ONCE (rdp->dynticks_nesting,1 );WARN_ON_ONCE (rdp->dynticks_nmi_nesting);@@ -827,7 +828,7 @@ static __always_inline void rcu_nmi_enter_common(bool irq) }trace_rcu_dyntick (incby ==1 ?TPS ("Endirq" ) :TPS ("++=" ), rdp->dynticks_nmi_nesting, - rdp->dynticks_nmi_nesting + incby, rdp->dynticks); + rdp->dynticks_nmi_nesting + incby,atomic_read (&rdp->dynticks));WRITE_ONCE (rdp->dynticks_nmi_nesting,/* Prevent store tearing. */ rdp->dynticks_nmi_nesting + incby);barrier ();Signature · 1 line
2.23.0.581.g78d2f28ef7-goog