Preemptible-RCU updates for v5.6
Subsystems (5)
Paul E. McKenney
maintainercover Preemptible-RCU updates for v5.6
Hello!
This series contains preemptible-RCU updates for v5.6:
1. Fix dump_tree hierarchy print always active, courtesy of
Stefan Reiter.
2. Avoid data-race in rcu_gp_fqs_check_wake(), courtesy of Eric
Dumazet.
3. Use lockdep rather than comment to enforce lock held.
4. Make PREEMPT_RCU be a modifier to TREE_RCU, courtesy of
Lai Jiangshan.
5. Use CONFIG_PREEMPTION where appropriate, courtesy of Sebastian
Andrzej Siewior.
6. Rename some instances of CONFIG_PREEMPTION to CONFIG_PREEMPT_RCU,
courtesy of Lai Jiangshan.
7. Clear .exp_hint only when deferred quiescent state has been
reported, courtesy of Lai Jiangshan.
8. Clear ->rcu_read_unlock_special only once, courtesy of Lai
Jiangshan.
9. Use READ_ONCE() for ->expmask in rcu_read_unlock_special().
10. Provide wrappers for uses of ->rcu_read_lock_nesting, courtesy
of Lai Jiangshan.
11. Avoid tick_dep_set_cpu() misordering.
Thanx, Paul
------------------------------------------------------------------------
include/linux/rcupdate.h | 8 +-- include/trace/events/rcu.h | 4 - kernel/rcu/Kconfig | 17 +++--- kernel/rcu/Makefile | 1 kernel/rcu/rcu.h | 2 kernel/rcu/rcutorture.c | 2 kernel/rcu/srcutiny.c | 2 kernel/rcu/tree.c | 20 ++++--- kernel/rcu/tree_exp.h | 6 +- kernel/rcu/tree_plugin.h | 118 +++++++++++++++++++++++++-------------------- kernel/rcu/tree_stall.h | 6 +- kernel/rcu/update.c | 2 kernel/sysctl.c | 2 13 files changed, 105 insertions(+), 85 deletions(-)
[email protected]
maintainerPATCH 1/11 rcu/nocb: Fix dump_tree hierarchy print always active
From: Stefan Reiter <[email protected]>
Commit 18cd8c93e69e ("rcu/nocb: Print gp/cb kthread hierarchy if dump_tree") added print statements to rcu_organize_nocb_kthreads for debugging, but incorrectly guarded them, causing the function to always spew out its message.
This patch fixes it by guarding both pr_alert statements with dump_tree, while also changing the second pr_alert to a pr_cont, to print the hierarchy in a single line (assuming that's how it was supposed to work).
--- kernel/rcu/tree_plugin.h | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-)
kernel/rcu/tree_plugin.h +17 -5
@@ -2321,6 +2321,8 @@ static void __init rcu_organize_nocb_kthreads(void) {int cpu;bool firsttime = true; +bool gotnocbs = false; +bool gotnocbscbs = true;int ls = rcu_nocb_gp_stride;int nl =0 ;/* Next GP kthread. */ struct rcu_data *rdp;
@@ -2343,21 +2345,31 @@ static void __init rcu_organize_nocb_kthreads(void) rdp =per_cpu_ptr (&rcu_data, cpu);if (rdp->cpu >= nl) {/* New GP kthread, set up for CBs & next GP. */ + gotnocbs = true; nl =DIV_ROUND_UP (rdp->cpu +1 , ls) * ls; rdp->nocb_gp_rdp = rdp; rdp_gp = rdp; -if (!firsttime && dump_tree) -pr_cont ("\n" ); - firsttime = false; -pr_alert ("%s: No-CB GP kthread CPU %d:" , __func__, cpu); +if (dump_tree) { +if (!firsttime) +pr_cont ("%s\n" , gotnocbscbs + ?"" :" (self only)" ); + gotnocbscbs = false; + firsttime = false; +pr_alert ("%s: No-CB GP kthread CPU %d:" , + __func__, cpu); + } }else {/* Another CB kthread, link to previous GP kthread. */ + gotnocbscbs = true; rdp->nocb_gp_rdp = rdp_gp; rdp_prev->nocb_next_cb_rdp = rdp; -pr_alert (" %d" , cpu); +if (dump_tree) +pr_cont (" %d" , cpu); } rdp_prev = rdp; } +if (gotnocbs && dump_tree) +pr_cont ("%s\n" , gotnocbscbs ?"" :" (self only)" ); }/*
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 2/11 rcu: Avoid data-race in rcu_gp_fqs_check_wake()
In reply to Paul E. McKenneyFrom: Eric Dumazet <[email protected]>
The rcu_gp_fqs_check_wake() function uses rcu_preempt_blocked_readers_cgp() to read ->gp_tasks while other cpus might overwrite this field.
We need READ_ONCE()/WRITE_ONCE() pairs to avoid compiler tricks and KCSAN splats like the following :
BUG: KCSAN: data-race in rcu_gp_fqs_check_wake / rcu_preempt_deferred_qs_irqrestore
write to 0xffffffff85a7f190 of 8 bytes by task 7317 on cpu 0:
rcu_preempt_deferred_qs_irqrestore+0x43d/0x580 kernel/rcu/tree_plugin.h:507 rcu_read_unlock_special+0xec/0x370 kernel/rcu/tree_plugin.h:659 __rcu_read_unlock+0xcf/0xe0 kernel/rcu/tree_plugin.h:394 rcu_read_unlock include/linux/rcupdate.h:645 [inline] __ip_queue_xmit+0x3b0/0xa40 net/ipv4/ip_output.c:533 ip_queue_xmit+0x45/0x60 include/net/ip.h:236 __tcp_transmit_skb+0xdeb/0x1cd0 net/ipv4/tcp_output.c:1158 __tcp_send_ack+0x246/0x300 net/ipv4/tcp_output.c:3685 tcp_send_ack+0x34/0x40 net/ipv4/tcp_output.c:3691 tcp_cleanup_rbuf+0x130/0x360 net/ipv4/tcp.c:1575 tcp_recvmsg+0x633/0x1a30 net/ipv4/tcp.c:2179 inet_recvmsg+0xbb/0x250 net/ipv4/af_inet.c:838 sock_recvmsg_nosec net/socket.c:871 [inline] sock_recvmsg net/socket.c:889 [inline] sock_recvmsg+0x92/0xb0 net/socket.c:885 sock_read_iter+0x15f/0x1e0 net/socket.c:967 call_read_iter include/linux/fs.h:1864 [inline] new_sync_read+0x389/0x4f0 fs/read_write.c:414
read to 0xffffffff85a7f190 of 8 bytes by task 10 on cpu 1:
rcu_gp_fqs_check_wake kernel/rcu/tree.c:1556 [inline] rcu_gp_fqs_check_wake+0x93/0xd0 kernel/rcu/tree.c:1546 rcu_gp_fqs_loop+0x36c/0x580 kernel/rcu/tree.c:1611 rcu_gp_kthread+0x143/0x220 kernel/rcu/tree.c:1768 kthread+0x1d4/0x200 drivers/block/aoe/aoecmd.c:1253 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:352
Reported by Kernel Concurrency Sanitizer on:
CPU: 1 PID: 10 Comm: rcu_preempt Not tainted 5.3.0+ #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
--- kernel/rcu/tree_plugin.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
kernel/rcu/tree_plugin.h +6 -5
@@ -220,7 +220,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)* blocked tasks. */ if (!rnp->gp_tasks && (blkd_state &RCU_GP_BLKD )) { - rnp->gp_tasks = &t->rcu_node_entry; +WRITE_ONCE (rnp->gp_tasks, &t->rcu_node_entry);WARN_ON_ONCE (rnp->completedqs == rnp->gp_seq); }if (!rnp->exp_tasks && (blkd_state &RCU_EXP_BLKD ))
@@ -340,7 +340,7 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);*/ static int rcu_preempt_blocked_readers_cgp (struct rcu_node *rnp) { -return rnp->gp_tasks !=NULL ; +return READ_ONCE (rnp->gp_tasks) !=NULL ; }/* Bias and limit values for ->rcu_read_lock_nesting. */
@@ -493,7 +493,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)trace_rcu_unlock_preempted_task (TPS ("rcu_preempt" ), rnp->gp_seq, t->pid);if (&t->rcu_node_entry == rnp->gp_tasks) - rnp->gp_tasks = np; +WRITE_ONCE (rnp->gp_tasks, np);if (&t->rcu_node_entry == rnp->exp_tasks) rnp->exp_tasks = np;if (IS_ENABLED (CONFIG_RCU_BOOST )) {
@@ -663,7 +663,7 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)dump_blkd_tasks (rnp,10 );if (rcu_preempt_has_tasks (rnp) && (rnp->qsmaskinit || rnp->wait_blkd_tasks)) { - rnp->gp_tasks = rnp->blkd_tasks.next; +WRITE_ONCE (rnp->gp_tasks, rnp->blkd_tasks.next); t = container_of(rnp->gp_tasks, struct task_struct, rcu_node_entry); trace_rcu_unlock_preempted_task(TPS("rcu_preempt-GPS"),
@@ -757,7 +757,8 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck)pr_info ("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n" , __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);pr_info ("%s: ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p\n" , - __func__, rnp->gp_tasks, rnp->boost_tasks, rnp->exp_tasks); + __func__,READ_ONCE (rnp->gp_tasks), rnp->boost_tasks, + rnp->exp_tasks);pr_info ("%s: ->blkd_tasks" , __func__); i =0 ;list_for_each (lhp, &rnp->blkd_tasks) {
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 3/11 rcu: Use lockdep rather than comment to enforce lock held
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
The rcu_preempt_check_blocked_tasks() function has a comment that states that the rcu_node structure's ->lock must be held, which might be informative, but which carries little weight if not read. This commit therefore removes this comment in favor of raw_lockdep_assert_held_rcu_node(), which will complain quite visibly if the required lock is not held.
--- kernel/rcu/tree_plugin.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/rcu/tree_plugin.h +2 -2
@@ -648,8 +648,7 @@ static void rcu_read_unlock_special(struct task_struct *t)* Check that the list of blocked tasks for the newly completed grace * period is in fact empty. It is a serious bug to complete a grace * period that still has RCU readers blocked! This function must be -* invoked -before- updating this rnp's ->gp_seq, and the rnp's ->lock -* must be held by the caller. +* invoked -before- updating this rnp's ->gp_seq. * * Also, if there are blocked tasks on the list, they automatically * block the newly created grace period, so set up ->gp_tasks accordingly.
@@ -659,6 +658,7 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)struct task_struct *t;RCU_LOCKDEP_WARN (preemptible (),"rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n" ); +raw_lockdep_assert_held_rcu_node (rnp);if (WARN_ON_ONCE (rcu_preempt_blocked_readers_cgp (rnp)))dump_blkd_tasks (rnp,10 ); if (rcu_preempt_has_tasks(rnp) &&
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 4/11 rcu: Make PREEMPT_RCU be a modifier to TREE_RCU
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
Currently PREEMPT_RCU and TREE_RCU are mutually exclusive Kconfig options. But PREEMPT_RCU actually specifies a kind of TREE_RCU, namely a preemptible TREE_RCU. This commit therefore makes PREEMPT_RCU be a modifer to the TREE_RCU Kconfig option. This has the benefit of simplifying several of the #if expressions that formerly needed to check both, but now need only check one or the other.
--- include/linux/rcupdate.h | 4 ++-- include/trace/events/rcu.h | 4 ++-- kernel/rcu/Kconfig | 13 +++++++------ kernel/rcu/Makefile | 1 - kernel/rcu/rcu.h | 2 +- kernel/rcu/update.c | 2 +- kernel/sysctl.c | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-)
include/linux/rcupdate.h +2 -2
@@ -167,7 +167,7 @@ do { \* TREE_RCU and rcu_barrier_() primitives in TINY_RCU. */ -#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_PREEMPT_RCU ) +#if defined(CONFIG_TREE_RCU )#include <linux/rcutree.h> #elif defined(CONFIG_TINY_RCU )#include <linux/rcutiny.h>
@@ -601,7 +601,7 @@ do { \* read-side critical section that would block in a !PREEMPT kernel. * But if you want the full story, read on! * -* In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), +* In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU), * it is illegal to block while in an RCU read-side critical section. * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION * kernel builds, RCU read-side critical sections may be preempted,
include/trace/events/rcu.h +2 -2
@@ -41,7 +41,7 @@ TRACE_EVENT(rcu_utilization,TP_printk ("%s" , __entry->s) ); -#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_PREEMPT_RCU ) +#if defined(CONFIG_TREE_RCU )/* * Tracepoint for grace-period events. Takes a string identifying the
@@ -432,7 +432,7 @@ TRACE_EVENT_RCU(rcu_fqs, __entry->cpu, __entry->qsevent) ); -#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */ +#endif /* #if defined(CONFIG_TREE_RCU) */ /* * Tracepoint for dyntick-idle entry/exit events. These take a string
kernel/rcu/Kconfig +7 -6
@@ -7,7 +7,7 @@ menu "RCU Subsystem" config TREE_RCU bool - default y if !PREEMPTION && SMP + default y if SMP help This option selects the RCU implementation that is designed for very large SMP system with hundreds or
@@ -17,6 +17,7 @@ config TREE_RCU config PREEMPT_RCU bool default y if PREEMPTION + select TREE_RCU help This option selects the RCU implementation that is designed for very large SMP systems with hundreds or
@@ -78,7 +79,7 @@ config TASKS_RCU user-mode execution as quiescent states. config RCU_STALL_COMMON - def_bool ( TREE_RCU || PREEMPT_RCU ) + def_bool TREE_RCU help This option enables RCU CPU stall code that is common between the TINY and TREE variants of RCU. The purpose is to allow
@@ -86,13 +87,13 @@ config RCU_STALL_COMMON making these warnings mandatory for the tree variants. config RCU_NEED_SEGCBLIST - def_bool ( TREE_RCU || PREEMPT_RCU || TREE_SRCU ) + def_bool ( TREE_RCU || TREE_SRCU ) config RCU_FANOUT int "Tree-based hierarchical RCU fanout value" range 2 64 if 64BIT range 2 32 if !64BIT - depends on (TREE_RCU || PREEMPT_RCU) && RCU_EXPERT + depends on TREE_RCU && RCU_EXPERT default 64 if 64BIT default 32 if !64BIT help
@@ -112,7 +113,7 @@ config RCU_FANOUT_LEAF int "Tree-based hierarchical RCU leaf-level fanout value" range 2 64 if 64BIT range 2 32 if !64BIT - depends on (TREE_RCU || PREEMPT_RCU) && RCU_EXPERT + depends on TREE_RCU && RCU_EXPERT default 16 help This option controls the leaf-level fanout of hierarchical
@@ -187,7 +188,7 @@ config RCU_BOOST_DELAY config RCU_NOCB_CPU bool "Offload RCU callback processing from boot-selected CPUs" - depends on TREE_RCU || PREEMPT_RCU + depends on TREE_RCU depends on RCU_EXPERT || NO_HZ_FULL default n help
kernel/rcu/Makefile +0 -1
@@ -9,6 +9,5 @@ obj-$(CONFIG_TINY_SRCU) += srcutiny.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o obj-$(CONFIG_RCU_PERF_TEST) += rcuperf.o obj-$(CONFIG_TREE_RCU) += tree.o -obj-$(CONFIG_PREEMPT_RCU) += tree.o obj-$(CONFIG_TINY_RCU) += tiny.o obj-$(CONFIG_RCU_NEED_SEGCBLIST) += rcu_segcblist.o
kernel/rcu/rcu.h +1 -1
@@ -454,7 +454,7 @@ enum rcutorture_type {INVALID_RCU_FLAVOR }; -#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_PREEMPT_RCU ) +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq); void do_trace_rcu_torture_read(const char *rcutorturename,
kernel/rcu/update.c +1 -1
@@ -435,7 +435,7 @@ struct debug_obj_descr rcuhead_debug_descr = {EXPORT_SYMBOL_GPL (rcuhead_debug_descr);#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ -#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_PREEMPT_RCU ) || defined(CONFIG_RCU_TRACE ) +#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_RCU_TRACE )void do_trace_rcu_torture_read (const char *rcutorturename,struct rcu_head *rhp,unsigned long secs,unsigned long c_old,unsigned long c)
kernel/sysctl.c +1 -1
@@ -1268,7 +1268,7 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_do_static_key, },#endif -#if defined(CONFIG_TREE_RCU ) || defined(CONFIG_PREEMPT_RCU ) +#if defined(CONFIG_TREE_RCU ) { .procname = "panic_on_rcu_stall", .data = &sysctl_panic_on_rcu_stall,
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 5/11 rcu: Use CONFIG_PREEMPTION where appropriate
In reply to Paul E. McKenneyFrom: Sebastian Andrzej Siewior <[email protected]>
The config option CONFIG_PREEMPT' is used for the preemption model "Low-Latency Desktop". The config option CONFIG_PREEMPTION' is enabled when kernel preemption is enabled which is true for the preemption model CONFIG_PREEMPT' and CONFIG_PREEMPT_RT'.
Use CONFIG_PREEMPTION' if it applies to both preemption models and not just to CONFIG_PREEMPT'.
--- include/linux/rcupdate.h | 4 ++-- kernel/rcu/Kconfig | 4 ++-- kernel/rcu/rcutorture.c | 2 +- kernel/rcu/srcutiny.c | 2 +- kernel/rcu/tree.c | 4 ++-- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_plugin.h | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-)
include/linux/rcupdate.h +2 -2
@@ -154,7 +154,7 @@ static inline void exit_tasks_rcu_finish(void) { }* * This macro resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() -* machinery were to be shut off, as some advocate for PREEMPT kernels. +* machinery were to be shut off, as some advocate for PREEMPTION kernels. */ #define cond_resched_tasks_rcu_qs () \ do { \
@@ -598,7 +598,7 @@ do { \* * You can avoid reading and understanding the next paragraph by * following this rule: don't put anything in an rcu_read_lock() RCU -* read-side critical section that would block in a !PREEMPT kernel. +* read-side critical section that would block in a !PREEMPTION kernel. * But if you want the full story, read on! * * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
kernel/rcu/Kconfig +2 -2
@@ -201,8 +201,8 @@ config RCU_NOCB_CPU specified at boot time by the rcu_nocbs parameter. For each such CPU, a kthread ("rcuox/N") will be created to invoke callbacks, where the "N" is the CPU being offloaded, and where - the "p" for RCU-preempt (PREEMPT kernels) and "s" for RCU-sched - (!PREEMPT kernels). Nothing prevents this kthread from running + the "p" for RCU-preempt (PREEMPTION kernels) and "s" for RCU-sched + (!PREEMPTION kernels). Nothing prevents this kthread from running on the specified CPUs, but (1) the kthreads may be preempted between each callback, and (2) affinity or cgroups can be used to force the kthreads to run on whatever set of CPUs is desired.
kernel/rcu/rcutorture.c +1 -1
@@ -1730,7 +1730,7 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)// Give the scheduler a chance, even on nohz_full CPUs. static void rcu_torture_fwd_prog_cond_resched (unsigned long iter) { -if (IS_ENABLED (CONFIG_PREEMPT ) &&IS_ENABLED (CONFIG_NO_HZ_FULL )) { +if (IS_ENABLED (CONFIG_PREEMPTION ) &&IS_ENABLED (CONFIG_NO_HZ_FULL )) {// Real call_rcu() floods hit userspace, so emulate that. if (need_resched () || (iter &0xfff ))schedule ();
kernel/rcu/srcutiny.c +1 -1
@@ -103,7 +103,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock);/* * Workqueue handler to drive one grace period and invoke any callbacks -* that become ready as a result. Single-CPU and !PREEMPT operation +* that become ready as a result. Single-CPU and !PREEMPTION operation * means that we get away with murder on synchronization. ;-) */ void srcu_drive_gp (struct work_struct *wp)
kernel/rcu/tree.c +2 -2
@@ -2698,9 +2698,9 @@ EXPORT_SYMBOL_GPL(kfree_call_rcu);/* * During early boot, any blocking grace-period wait automatically -* implies a grace period. Later on, this is never the case for PREEMPT. +* implies a grace period. Later on, this is never the case for PREEMPTION. * -* Howevr, because a context switch is a grace period for !PREEMPT, any +* Howevr, because a context switch is a grace period for !PREEMPTION, any * blocking grace-period wait automatically implies a grace period if * there is only one CPU online at any point time during execution of * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
kernel/rcu/tree_exp.h +1 -1
@@ -670,7 +670,7 @@ static void rcu_exp_handler(void *unused) } } -/* PREEMPT=y, so no PREEMPT=n expedited grace period to clean up after. */ +/* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */ static void sync_sched_exp_online_cleanup (int cpu) { }
kernel/rcu/tree_plugin.h +2 -2
@@ -789,7 +789,7 @@ static void __init rcu_bootup_announce(void) }/* -* Note a quiescent state for PREEMPT=n. Because we do not need to know +* Note a quiescent state for PREEMPTION=n. Because we do not need to know * how many quiescent states passed, just if there was at least one since * the start of the grace period, this just sets a flag. The caller must * have disabled preemption.
@@ -839,7 +839,7 @@ void rcu_all_qs(void)EXPORT_SYMBOL_GPL (rcu_all_qs);/* -* Note a PREEMPT=n context switch. The caller must have disabled interrupts. +* Note a PREEMPTION=n context switch. The caller must have disabled interrupts. */ void rcu_note_context_switch (bool preempt) {
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 6/11 rcu: Rename some instance of CONFIG_PREEMPTION to CONFIG_PREEMPT_RCU
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
CONFIG_PREEMPTION and CONFIG_PREEMPT_RCU are always identical, but some code depends on CONFIG_PREEMPTION to access to rcu_preempt functionality. This patch changes CONFIG_PREEMPTION to CONFIG_PREEMPT_RCU in these cases.
--- kernel/rcu/tree.c | 4 ++-- kernel/rcu/tree_stall.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-)
kernel/rcu/tree.c +2 -2
@@ -1934,7 +1934,7 @@ rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)struct rcu_node *rnp_p;raw_lockdep_assert_held_rcu_node (rnp); -if (WARN_ON_ONCE (!IS_ENABLED (CONFIG_PREEMPTION )) || +if (WARN_ON_ONCE (!IS_ENABLED (CONFIG_PREEMPT_RCU )) ||WARN_ON_ONCE (rcu_preempt_blocked_readers_cgp (rnp)) || rnp->qsmask !=0 ) {raw_spin_unlock_irqrestore_rcu_node (rnp, flags);
@@ -2294,7 +2294,7 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) mask =0 ;raw_spin_lock_irqsave_rcu_node (rnp, flags);if (rnp->qsmask ==0 ) { -if (!IS_ENABLED (CONFIG_PREEMPTION ) || +if (!IS_ENABLED (CONFIG_PREEMPT_RCU ) ||rcu_preempt_blocked_readers_cgp (rnp)) {/* * No point in scanning bits because they
kernel/rcu/tree_stall.h +3 -3
@@ -163,7 +163,7 @@ static void rcu_iw_handler(struct irq_work *iwp)// // Printing RCU CPU stall warnings -#ifdef CONFIG_PREEMPTION +#ifdef CONFIG_PREEMPT_RCU /* * Dump detailed information for all tasks blocking the current RCU
@@ -215,7 +215,7 @@ static int rcu_print_task_stall(struct rcu_node *rnp) return ndetected; } -#else /* #ifdef CONFIG_PREEMPTION */ +#else /* #ifdef CONFIG_PREEMPT_RCU */ /* * Because preemptible RCU does not exist, we never have to check for
@@ -233,7 +233,7 @@ static int rcu_print_task_stall(struct rcu_node *rnp) {return 0 ; } -#endif /* #else #ifdef CONFIG_PREEMPTION */ +#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* * Dump stacks of all tasks running on stalled CPUs. First try using
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 7/11 rcu: Clear .exp_hint only when deferred quiescent state has been reported
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
Currently, the .exp_hint flag is cleared in rcu_read_unlock_special(), which works, but which can also prevent subsequent rcu_read_unlock() calls from helping expedite the quiescent state needed by an ongoing expedited RCU grace period. This commit therefore defers clearing of .exp_hint from rcu_read_unlock_special() to rcu_preempt_deferred_qs_irqrestore(), thus ensuring that intervening calls to rcu_read_unlock() have a chance to help end the expedited grace period.
--- kernel/rcu/tree_plugin.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/rcu/tree_plugin.h +1 -2
@@ -444,6 +444,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)local_irq_restore (flags);return ; } + t->rcu_read_unlock_special.b.exp_hint = false; t->rcu_read_unlock_special.b.deferred_qs = false;if (special.b.need_qs) {rcu_qs ();
@@ -610,7 +611,6 @@ static void rcu_read_unlock_special(struct task_struct *t)struct rcu_data *rdp =this_cpu_ptr (&rcu_data);struct rcu_node *rnp = rdp->mynode; - t->rcu_read_unlock_special.b.exp_hint = false; exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) || (rdp->grpmask & rnp->expmask) ||tick_nohz_full_cpu (rdp->cpu);
@@ -640,7 +640,6 @@ static void rcu_read_unlock_special(struct task_struct *t)local_irq_restore (flags);return ; } -WRITE_ONCE (t->rcu_read_unlock_special.b.exp_hint, false);rcu_preempt_deferred_qs_irqrestore (t, flags); }
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 8/11 rcu: Clear ->rcu_read_unlock_special only once
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
In rcu_preempt_deferred_qs_irqrestore(), ->rcu_read_unlock_special is cleared one piece at a time. Given that the "if" statements in this function use the copy in "special", this commit removes the clearing of the individual pieces in favor of clearing ->rcu_read_unlock_special in one go just after it has been determined to be non-zero.
--- kernel/rcu/tree_plugin.h | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-)
kernel/rcu/tree_plugin.h +3 -16
@@ -444,16 +444,9 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)local_irq_restore (flags);return ; } - t->rcu_read_unlock_special.b.exp_hint = false; - t->rcu_read_unlock_special.b.deferred_qs = false; -if (special.b.need_qs) { + t->rcu_read_unlock_special.s =0 ; +if (special.b.need_qs)rcu_qs (); - t->rcu_read_unlock_special.b.need_qs = false; -if (!t->rcu_read_unlock_special.s && !rdp->exp_deferred_qs) { -local_irq_restore (flags); -return ; - } - }/* * Respond to a request by an expedited grace period for a
@@ -461,17 +454,11 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)* tasks are handled when removing the task from the * blocked-tasks list below. */ -if (rdp->exp_deferred_qs) { +if (rdp->exp_deferred_qs)rcu_report_exp_rdp (rdp); -if (!t->rcu_read_unlock_special.s) { -local_irq_restore (flags); -return ; - } - }/* Clean up if blocked during RCU read-side critical section. */ if (special.b.blocked) { - t->rcu_read_unlock_special.b.blocked = false;/* * Remove this task from the list it blocked on. The task
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 9/11 rcu: Use READ_ONCE() for ->expmask in rcu_read_unlock_special()
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
The rcu_node structure's ->expmask field is updated only when holding the ->lock, but is also accessed locklessly. This means that all ->expmask updates must use WRITE_ONCE() and all reads carried out without holding ->lock must use READ_ONCE(). This commit therefore changes the lockless ->expmask read in rcu_read_unlock_special() to use READ_ONCE().
--- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/tree_plugin.h +1 -1
@@ -599,7 +599,7 @@ static void rcu_read_unlock_special(struct task_struct *t)struct rcu_node *rnp = rdp->mynode; exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) || - (rdp->grpmask & rnp->expmask) || + (rdp->grpmask &READ_ONCE (rnp->expmask)) ||tick_nohz_full_cpu (rdp->cpu);// Need to defer quiescent state until everything is enabled. if (irqs_were_disabled && use_softirq &&
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 10/11 rcu: Provide wrappers for uses of ->rcu_read_lock_nesting
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
This commit provides wrapper functions for uses of ->rcu_read_lock_nesting to improve readability and to ease future changes to support inlining of __rcu_read_lock() and __rcu_read_unlock().
--- kernel/rcu/tree_exp.h | 4 ++-- kernel/rcu/tree_plugin.h | 53 +++++++++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 21 deletions(-)
kernel/rcu/tree_exp.h +2 -2
@@ -610,7 +610,7 @@ static void rcu_exp_handler(void *unused)* critical section. If also enabled or idle, immediately * report the quiescent state, otherwise defer. */ -if (!t->rcu_read_lock_nesting) { +if (!rcu_preempt_depth ()) {if (!(preempt_count () & (PREEMPT_MASK |SOFTIRQ_MASK )) ||rcu_dynticks_curr_cpu_in_eqs ()) {rcu_report_exp_rdp (rdp);
@@ -634,7 +634,7 @@ static void rcu_exp_handler(void *unused)* can have caused this quiescent state to already have been * reported, so we really do need to check ->expmask. */ -if (t->rcu_read_lock_nesting >0 ) { +if (rcu_preempt_depth () >0 ) {raw_spin_lock_irqsave_rcu_node (rnp, flags);if (rnp->expmask & rdp->grpmask) { rdp->exp_deferred_qs = true;
kernel/rcu/tree_plugin.h +34 -19
130 lines not on this page · raw message
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 11/11 rcu: Avoid tick_dep_set_cpu() misordering
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
In the current code, rcu_nmi_enter_common() might decide to turn on the tick using tick_dep_set_cpu(), but be delayed just before doing so. Then the grace-period kthread might notice that the CPU in question had in fact gone through a quiescent state, thus turning off the tick using tick_dep_clear_cpu(). The later invocation of tick_dep_set_cpu() would then incorrectly leave the tick on.
This commit therefore enlists the aid of the leaf rcu_node structure's ->lock to ensure that decisions to enable or disable the tick are carried out before they can be reversed.
--- kernel/rcu/tree.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
kernel/rcu/tree.c +9 -3
@@ -800,8 +800,8 @@ void rcu_user_exit(void)*/ static __always_inline void rcu_nmi_enter_common(bool irq) { -struct rcu_data *rdp =this_cpu_ptr (&rcu_data);long incby =2 ; +struct rcu_data *rdp =this_cpu_ptr (&rcu_data);/* Complain about underflow. */ WARN_ON_ONCE (rdp->dynticks_nmi_nesting <0 );
@@ -828,8 +828,13 @@ static __always_inline void rcu_nmi_enter_common(bool irq) }else if (tick_nohz_full_cpu (rdp->cpu) && rdp->dynticks_nmi_nesting ==DYNTICK_IRQ_NONIDLE &&READ_ONCE (rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) { - rdp->rcu_forced_tick = true; -tick_dep_set_cpu (rdp->cpu,TICK_DEP_BIT_RCU ); +raw_spin_lock_rcu_node (rdp->mynode); +// Recheck under lock. +if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { + rdp->rcu_forced_tick = true; +tick_dep_set_cpu (rdp->cpu,TICK_DEP_BIT_RCU ); + } +raw_spin_unlock_rcu_node (rdp->mynode); } trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), rdp->dynticks_nmi_nesting,
@@ -898,6 +903,7 @@ void rcu_irq_enter_irqson(void)*/ static void rcu_disable_urgency_upon_qs (struct rcu_data *rdp) { +raw_lockdep_assert_held_rcu_node (rdp->mynode);WRITE_ONCE (rdp->rcu_urgent_qs, false);WRITE_ONCE (rdp->rcu_need_heavy_qs, false);if (tick_nohz_full_cpu (rdp->cpu) && rdp->rcu_forced_tick) {
Signature · 1 line
2.9.5