rcu: ftrace&x86/mce RCU lockdep fixes, extend RCU CPU stall
cover rcu: ftrace&x86/mce RCU lockdep fixes, extend RCU CPU stall
Hello!
This patch set gets rid of read_barrier_depends() in ftrace in favor of RCU APIs, increases the RCU CPU stall timeout if CONFIG_PROVE_RCU, and fixes an x86/mce lockdep splat.
Thanx, Paul
------------------------------------------------------------------------
arch/x86/kernel/cpu/mcheck/mce.c | 12 ++++++++---- kernel/rcutree.h | 22 +++++++++++++++------- kernel/trace/ftrace.c | 23 +++++++++++++---------- 3 files changed, 36 insertions(+), 21 deletions(-)
PATCH 3/3 x86/mce: fix RCU lockdep splats
Create an rcu_dereference_check_mce() that checks for RCU-sched read side and mce_read_mutex being held on update side. Replace uses of rcu_dereference() in arch/x86/kernel/cpu/mcheck/mce.c with this new macro.
--- arch/x86/kernel/cpu/mcheck/mce.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/mcheck/mce.c +8 -3
@@ -46,6 +46,11 @@#include "mce-internal.h" +#define rcu_dereference_check_mce (p) \ + rcu_dereference_check((p), \ + rcu_read_lock_sched_held() || \ + lockdep_is_held(&mce_read_mutex)) +#define CREATE_TRACE_POINTS #include <trace/events/mce.h>
@@ -158,7 +163,7 @@ void mce_log(struct mce *mce) mce->finished =0 ;wmb ();for (;;) { - entry =rcu_dereference (mcelog.next); + entry =rcu_dereference_check_mce (mcelog.next);for (;;) {/* * When the buffer fills up discard new entries.
@@ -1500,7 +1505,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,return -ENOMEM ;mutex_lock (&mce_read_mutex); - next =rcu_dereference (mcelog.next); + next =rcu_dereference_check_mce (mcelog.next);/* Only supports full reads right now */ if (*off !=0 || usize <MCE_LOG_LEN *sizeof (struct mce )) {
@@ -1565,7 +1570,7 @@ timeout:static unsigned int mce_poll (struct file *file,poll_table *wait) {poll_wait (file, &mce_wait, wait); -if (rcu_dereference (mcelog.next)) +if (rcu_dereference_check_mce (mcelog.next))return POLLIN |POLLRDNORM ;return 0 ; }
Signature · 1 line
1.6.6
PATCH 2/3 rcu: Increase RCU CPU stall timeouts if PROVE_RCU
In reply to Paul E. McKenneyCONFIG_PROVE_RCU imposes additional overhead on the kernel, so increase the RCU CPU stall timeouts in an attempt to allow for this effect.
--- kernel/rcutree.h | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-)
kernel/rcutree.h +15 -6
@@ -246,12 +246,21 @@ struct rcu_data {#define RCU_JIFFIES_TILL_FORCE_QS 3/* for rsp->jiffies_force_qs */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR -#define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ)/* for rsp->jiffies_stall */ -#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ)/* for rsp->jiffies_stall */ -#define RCU_STALL_RAT_DELAY 2/* Allow other CPUs time */ -/* to take at least one */ -/* scheduling clock irq */ -/* before ratting on them. */ + +#ifdef CONFIG_PROVE_RCU +#define RCU_STALL_DELAY_DELTA (5 * HZ) +#else +#define RCU_STALL_DELAY_DELTA 0 +#endif + +#define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ + RCU_STALL_DELAY_DELTA) +/* for rsp->jiffies_stall */ +#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ + RCU_STALL_DELAY_DELTA) +/* for rsp->jiffies_stall */ +#define RCU_STALL_RAT_DELAY 2/* Allow other CPUs time */ +/* to take at least one */ +/* scheduling clock irq */ +/* before ratting on them. */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
Signature · 1 line
1.6.6
PATCH 1/3 ftrace: replace read_barrier_depends() with rcu_dereference_raw()
In reply to Paul E. McKenneyReplace the calls to read_barrier_depends() in ftrace_list_func() with rcu_dereference_raw() to improve readability. The reason that we use rcu_dereference_raw() here is that removed entries are never freed, instead they are simply leaked. This is one of a very few cases where use of rcu_dereference_raw() is the long-term right answer. And I don't yet know of any others. ;-)
--- kernel/trace/ftrace.c | 22 +++++++++++++--------- 1 files changed, 13 insertions(+), 9 deletions(-)
kernel/trace/ftrace.c +13 -9
@@ -27,6 +27,7 @@#include <linux/ctype.h> #include <linux/list.h> #include <linux/hash.h> +#include <linux/rcupdate.h> #include <trace/events/sched.h>
@@ -88,18 +89,22 @@ ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;static int ftrace_set_func (unsigned long *array,int *idx,char *buffer);#endif +/* +* Traverse the ftrace_list, invoking all entries. The reason that we +* can use rcu_dereference_raw() is that elements removed from this list +* are simply leaked, so there is no need to interact with a grace-period +* mechanism. The rcu_dereference_raw() calls are needed to handle +* concurrent insertions into the ftrace_list. +* +* Silly Alpha and silly pointer-speculation compiler optimizations! +*/ static void ftrace_list_func (unsigned long ip,unsigned long parent_ip) { -struct ftrace_ops *op = ftrace_list; - -/* in case someone actually ports this to alpha! */ -read_barrier_depends (); +struct ftrace_ops *op =rcu_dereference_raw (ftrace_list);/*see above*/ while (op != &ftrace_list_end) { -/* silly alpha */ -read_barrier_depends (); op->func (ip, parent_ip); - op = op->next; + op =rcu_dereference_raw (op->next);/*see above*/ }; }
@@ -154,8 +159,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)* the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ -smp_wmb (); - ftrace_list = ops; +rcu_assign_pointer (ftrace_list, ops);if (ftrace_enabled) {ftrace_func_t func;
Signature · 1 line
1.6.6
Re: [PATCH tip/core/rcu 1/3] ftrace: replace read_barrier_depends() with rcu_dereference_raw()
Quoted 8 lines · Paul E. McKenney
Replace the calls to
read_barrier_depends()inftrace_list_func()withrcu_dereference_raw()to improve readability. The reason that we usercu_dereference_raw()here is that removed entries are never freed, instead they are simply leaked. This is one of a very few cases where use ofrcu_dereference_raw()is the long-term right answer. And I don't yet know of any others. ;-)Cc Steven Rostedt <[email protected]>
Thanks Paul!
kernel/trace/ftrace.c static int __register_ftrace_function(struct ftrace_ops *ops) · line 162
Quoted 13 lines
Cc Frederic Weisbecker <[email protected]>Cc Ingo Molnar <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>---
kernel/trace/ftrace.c +1
-2@@ -154,8 +159,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)* the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ -smp_wmb (); - ftrace_list = ops; +rcu_assign_pointer (ftrace_list, ops);
[ Off topic ]
I looked at rcu_assign_pointer() and it is:
#define rcu_assign_pointer(p, v) \
({ \
if (!__builtin_constant_p(v) || \
((v) != NULL)) \
smp_wmb(); \
(p) = (v); \
})My question is, why that crazy if? The only time that will fail is if we are assigning the constant NULL to p. What makes NULL so important here? Can't there be a case when assigning NULL to p will require that wmb()?
-- Steve
Quoted 3 lines
kernel/trace/ftrace.c +0
-0if (ftrace_enabled) {ftrace_func_t func;
Re: [PATCH tip/core/rcu 1/3] ftrace: replace read_barrier_depends() with rcu_dereference_raw()
Quoted 40 lines · Steven Rostedt
Quoted 8 lines · Paul E. McKenney
Replace the calls to
read_barrier_depends()inftrace_list_func()withrcu_dereference_raw()to improve readability. The reason that we usercu_dereference_raw()here is that removed entries are never freed, instead they are simply leaked. This is one of a very few cases where use ofrcu_dereference_raw()is the long-term right answer. And I don't yet know of any others. ;-)Cc Steven Rostedt <[email protected]>Acked-by Steven Rostedt <[email protected]>Thanks Paul!
kernel/trace/ftrace.c static int __register_ftrace_function(struct ftrace_ops *ops) · line 162
Quoted 13 lines
Cc Frederic Weisbecker <[email protected]>Cc Ingo Molnar <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>---
kernel/trace/ftrace.c +1
-2@@ -154,8 +159,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)* the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ -smp_wmb (); - ftrace_list = ops; +rcu_assign_pointer (ftrace_list, ops);[ Off topic ]
I looked at
rcu_assign_pointer()and it is:#define rcu_assign_pointer(p, v) \ ({ \ if (!__builtin_constant_p(v) || \ ((v) != NULL)) \ smp_wmb(); \ (p) = (v); \ })
Quoted 3 lines
My question is, why that crazy if? The only time that will fail is if we are assigning the constant NULL to p. What makes NULL so important here? Can't there be a case when assigning NULL to p will require that
wmb()?
The barrier ensures that the reader can't see the new p and the old *p. Since you can't look at *NULL, that concern doesn't apply.
- Josh Triplett
Steven Rostedt
maintainerRe: [PATCH tip/core/rcu 1/3] ftrace: replace read_barrier_depends() with rcu_dereference_raw()
Quoted 8 lines · Josh Triplett
Quoted 8 lines
#define rcu_assign_pointer(p, v) \ ({ \ if (!__builtin_constant_p(v) || \ ((v) != NULL)) \ smp_wmb(); \ (p) = (v); \ })
Quoted 6 lines
Quoted 3 lines
My question is, why that crazy if? The only time that will fail is if we are assigning the constant NULL to p. What makes NULL so important here? Can't there be a case when assigning NULL to p will require that
wmb()?The barrier ensures that the reader can't see the new p and the old *p. Since you can't look at *NULL, that concern doesn't apply.
Thanks for the explanation.
Question 2)
Then why the !__builtin_constant_p(v)?
If v is NULL, then the same should apply even if it is not a constant? What am I missing?
-- Steve
Re: [PATCH tip/core/rcu 1/3] ftrace: replace read_barrier_depends() with rcu_dereference_raw()
Quoted 17 lines · Steven Rostedt
Quoted 8 lines · Josh Triplett
Quoted 8 lines
#define rcu_assign_pointer(p, v) \ ({ \ if (!__builtin_constant_p(v) || \ ((v) != NULL)) \ smp_wmb(); \ (p) = (v); \ })Quoted 6 lines
Quoted 3 lines
My question is, why that crazy if? The only time that will fail is if we are assigning the constant NULL to p. What makes NULL so important here? Can't there be a case when assigning NULL to p will require that
wmb()?The barrier ensures that the reader can't see the new p and the old *p. Since you can't look at *NULL, that concern doesn't apply.
Quoted 8 lines
Thanks for the explanation.
Question 2)
Then why the !__builtin_constant_p(v)?
If v is NULL, then the same should apply even if it is not a constant? What am I missing?
Checking for __builtin_constant_p(v) ensures that this test happens at compile time, and thus no conditional occurs at runtime. Together with the assumption of compiler constant folding and dead code elimination, this test means "if you can tell at compile time that the call assigns NULL, emit no barrier, otherwise emit a barrier". Under no circumstances will this macro actually emit conditional code.
- Josh Triplett
Steven Rostedt
maintainerRe: [PATCH tip/core/rcu 1/3] ftrace: replace read_barrier_depends() with rcu_dereference_raw()
Quoted 7 lines · Josh Triplett
Quoted 6 lines
Question 2)
Then why the !__builtin_constant_p(v)?
If v is NULL, then the same should apply even if it is not a constant? What am I missing?
Quoted 6 lines
Checking for
__builtin_constant_p(v)ensures that this test happens at compile time, and thus no conditional occurs at runtime. Together with the assumption of compiler constant folding and dead code elimination, this test means "if you can tell at compile time that the call assigns NULL, emit no barrier, otherwise emit a barrier". Under no circumstances will this macro actually emit conditional code.
Ah OK!
So the benefit of not doing a smb_wmb() when a variable is NULL is outweighed by the benefits of removing branches and extra code.
Yes it now makes sense. Only remove the wmb() when we can guarantee that it is never needed, and avoid unnecessary branches when it may not be needed.
Thanks for clarifying!
-- Steve
[tip:core/urgent] ftrace: Replace read_barrier_depends() with rcu_dereference_raw()
In reply to Paul E. McKenneyCommit-ID: 3f379b03fbfddd20536389a85c6456f8233d1f8d Gitweb: http://git.kernel.org/tip/3f379b03fbfddd20536389a85c6456f8233d1f8d Author: Paul E. McKenney <[email protected]> AuthorDate: Fri, 5 Mar 2010 15:03:25 -0800 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 11 Mar 2010 13:38:01 +0100
ftrace: Replace read_barrier_depends() with rcu_dereference_raw()
Replace the calls to read_barrier_depends() in ftrace_list_func() with rcu_dereference_raw() to improve readability. The reason that we use rcu_dereference_raw() here is that removed entries are never freed, instead they are simply leaked. This is one of a very few cases where use of rcu_dereference_raw() is the long-term right answer. And I don't yet know of any others. ;-)
LKML-Reference: <[email protected]>
--- kernel/trace/ftrace.c | 22 +++++++++++++--------- 1 files changed, 13 insertions(+), 9 deletions(-)
kernel/trace/ftrace.c +13 -9
@@ -27,6 +27,7 @@#include <linux/ctype.h> #include <linux/list.h> #include <linux/hash.h> +#include <linux/rcupdate.h> #include <trace/events/sched.h>
@@ -88,18 +89,22 @@ ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;static int ftrace_set_func (unsigned long *array,int *idx,char *buffer);#endif +/* +* Traverse the ftrace_list, invoking all entries. The reason that we +* can use rcu_dereference_raw() is that elements removed from this list +* are simply leaked, so there is no need to interact with a grace-period +* mechanism. The rcu_dereference_raw() calls are needed to handle +* concurrent insertions into the ftrace_list. +* +* Silly Alpha and silly pointer-speculation compiler optimizations! +*/ static void ftrace_list_func (unsigned long ip,unsigned long parent_ip) { -struct ftrace_ops *op = ftrace_list; - -/* in case someone actually ports this to alpha! */ -read_barrier_depends (); +struct ftrace_ops *op =rcu_dereference_raw (ftrace_list);/*see above*/ while (op != &ftrace_list_end) { -/* silly alpha */ -read_barrier_depends (); op->func (ip, parent_ip); - op = op->next; + op =rcu_dereference_raw (op->next);/*see above*/ }; }
@@ -154,8 +159,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)* the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ -smp_wmb (); - ftrace_list = ops; +rcu_assign_pointer (ftrace_list, ops);if (ftrace_enabled) {ftrace_func_t func;
[tip:core/urgent] rcu: Increase RCU CPU stall timeouts if PROVE_RCU
In reply to Paul E. McKenneyCommit-ID: 007b09243b099811124f69d492adeebe9e439f96 Gitweb: http://git.kernel.org/tip/007b09243b099811124f69d492adeebe9e439f96 Author: Paul E. McKenney <[email protected]> AuthorDate: Fri, 5 Mar 2010 15:03:26 -0800 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 11 Mar 2010 13:38:01 +0100
rcu: Increase RCU CPU stall timeouts if PROVE_RCU
CONFIG_PROVE_RCU imposes additional overhead on the kernel, so increase the RCU CPU stall timeouts in an attempt to allow for this effect.
LKML-Reference: <[email protected]>
--- kernel/rcutree.h | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-)
kernel/rcutree.h +15 -6
@@ -246,12 +246,21 @@ struct rcu_data {#define RCU_JIFFIES_TILL_FORCE_QS 3/* for rsp->jiffies_force_qs */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR -#define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ)/* for rsp->jiffies_stall */ -#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ)/* for rsp->jiffies_stall */ -#define RCU_STALL_RAT_DELAY 2/* Allow other CPUs time */ -/* to take at least one */ -/* scheduling clock irq */ -/* before ratting on them. */ + +#ifdef CONFIG_PROVE_RCU +#define RCU_STALL_DELAY_DELTA (5 * HZ) +#else +#define RCU_STALL_DELAY_DELTA 0 +#endif + +#define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ + RCU_STALL_DELAY_DELTA) +/* for rsp->jiffies_stall */ +#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ + RCU_STALL_DELAY_DELTA) +/* for rsp->jiffies_stall */ +#define RCU_STALL_RAT_DELAY 2/* Allow other CPUs time */ +/* to take at least one */ +/* scheduling clock irq */ +/* before ratting on them. */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
[tip:core/urgent] x86/mce: Fix RCU lockdep splats
In reply to Paul E. McKenneyCommit-ID: f56e8a0765cc4374e02f4e3a79e2427b5096b075 Gitweb: http://git.kernel.org/tip/f56e8a0765cc4374e02f4e3a79e2427b5096b075 Author: Paul E. McKenney <[email protected]> AuthorDate: Fri, 5 Mar 2010 15:03:27 -0800 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 11 Mar 2010 13:38:02 +0100
x86/mce: Fix RCU lockdep splats
Create an rcu_dereference_check_mce() that checks for RCU-sched read side and mce_read_mutex being held on update side. Replace uses of rcu_dereference() in arch/x86/kernel/cpu/mcheck/mce.c with this new macro.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/mcheck/mce.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/mcheck/mce.c +8 -3
@@ -46,6 +46,11 @@#include "mce-internal.h" +#define rcu_dereference_check_mce (p) \ + rcu_dereference_check((p), \ + rcu_read_lock_sched_held() || \ + lockdep_is_held(&mce_read_mutex)) +#define CREATE_TRACE_POINTS #include <trace/events/mce.h>
@@ -158,7 +163,7 @@ void mce_log(struct mce *mce) mce->finished =0 ;wmb ();for (;;) { - entry =rcu_dereference (mcelog.next); + entry =rcu_dereference_check_mce (mcelog.next);for (;;) {/* * When the buffer fills up discard new entries.
@@ -1500,7 +1505,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,return -ENOMEM ;mutex_lock (&mce_read_mutex); - next =rcu_dereference (mcelog.next); + next =rcu_dereference_check_mce (mcelog.next);/* Only supports full reads right now */ if (*off !=0 || usize <MCE_LOG_LEN *sizeof (struct mce )) {
@@ -1565,7 +1570,7 @@ timeout:static unsigned int mce_poll (struct file *file,poll_table *wait) {poll_wait (file, &mce_wait, wait); -if (rcu_dereference (mcelog.next)) +if (rcu_dereference_check_mce (mcelog.next))return POLLIN |POLLRDNORM ;return 0 ; }
[tip:core/urgent] x86/mce: Fix build bug with CONFIG_PROVE_LOCKING=y && CONFIG_X86_MCE_INTEL=y
In reply to Paul E. McKenneyCommit-ID: 2aa2b50dd62b5d0675bd7453fbeb5732dc2d7866 Gitweb: http://git.kernel.org/tip/2aa2b50dd62b5d0675bd7453fbeb5732dc2d7866 Author: Ingo Molnar <[email protected]> AuthorDate: Sun, 14 Mar 2010 08:57:03 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 14 Mar 2010 08:57:03 +0100
x86/mce: Fix build bug with CONFIG_PROVE_LOCKING=y && CONFIG_X86_MCE_INTEL=y
Commit f56e8a076 "x86/mce: Fix RCU lockdep splats" introduced the following build bug:
arch/x86/kernel/cpu/mcheck/mce.c: In function 'mce_log':
arch/x86/kernel/cpu/mcheck/mce.c:166: error: 'mce_read_mutex' undeclared (first use in this function) arch/x86/kernel/cpu/mcheck/mce.c:166: error: (Each undeclared identifier is reported only once arch/x86/kernel/cpu/mcheck/mce.c:166: error: for each function it appears in.)
Move the in-the-middle-of-file lock variable up to the variable definition section, the top of the .c file.
Cc: Paul E. McKenney <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/mcheck/mce.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -46,6 +46,8 @@#include "mce-internal.h" +static DEFINE_MUTEX (mce_read_mutex); +#define rcu_dereference_check_mce (p) \ rcu_dereference_check((p), \ rcu_read_lock_sched_held() || \
@@ -1490,8 +1492,6 @@ static void collect_tscs(void *data)rdtscll (cpu_tsc[smp_processor_id ()]); } -static DEFINE_MUTEX (mce_read_mutex); -static ssize_t mce_read (struct file *filp,char __user *ubuf,size_t usize,loff_t *off) {