rcu: Expedited GP polling improvements
Frederic Weisbecker
maintainercover rcu: Expedited GP polling improvements
I initially wanted to shape that into the form of reviews or questions but I thought I might as well send patches.
Be careful, it's fairly possible I misunderstood some things and those patches are completely off the track.
At least TREE07 looks happy.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
rcu/dev
HEAD: dd1f68246e04fe03fde46cd55f1c87ea92a6c57e
Thanks,
Frederic
---
Frederic Weisbecker (4):
rcu: Remove needless polling work requeue for further waiter
rcu: No need to reset the poll request flag before completion
rcu: Perform early sequence fetch for polling locklessly
rcu: Name internal polling flag
kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 18 +++++++----------- 3 files changed, 13 insertions(+), 12 deletions(-)
Frederic Weisbecker
maintainerPATCH 1/4 rcu: Remove needless polling work requeue for further waiter
If another expedited polling site is waiting for the next grace period, there is no need to requeue the work because it is guaranteed to be either already queued or executing the actual polling upon completion.
--- kernel/rcu/tree_exp.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/rcu/tree_exp.h +1 -3
@@ -919,9 +919,7 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)__synchronize_rcu_expedited (true);raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) && !sync_exp_work_done (s)) -queue_work (rcu_gp_wq, &rnp->exp_poll_wq); -else +if (!(s &0x1 ) &&sync_exp_work_done (s)) rnp->exp_seq_poll_rq |=0x1 ;raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }
Signature · 1 line
2.25.1
Frederic Weisbecker
maintainerPATCH 3/4 rcu: Perform early sequence fetch for polling locklessly
In reply to Frederic WeisbeckerThe workqueue ordering guarantees that the work sees all the accesses of the task prior to its call to the corresponding queue_work().
Therefore the sequence to poll can be retrieved locklessly.
The only downside is that it is then possible to miss the 0x1 flag set by a prior work. But this could already happen concurrently anyway after the exp_poll_lock is unlocked. In the worst case the slow path involving synchronize_rcu_expedited() takes care of the situation.
--- kernel/rcu/tree_exp.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
kernel/rcu/tree_exp.h +3 -5
@@ -909,9 +909,7 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp) struct rcu_node *rnp = container_of(wp, struct rcu_node, exp_poll_wq);unsigned long s; -raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); - s = rnp->exp_seq_poll_rq; -raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); + s =READ_ONCE (rnp->exp_seq_poll_rq);if (s &0x1 )return ;while (!sync_exp_work_done (s))
@@ -919,7 +917,7 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq;if (!(s &0x1 ) &&sync_exp_work_done (s)) - rnp->exp_seq_poll_rq |=0x1 ; +WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }
@@ -949,7 +947,7 @@ unsigned long start_poll_synchronize_rcu_expedited(void)if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags);if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { - rnp->exp_seq_poll_rq = s; +WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq); }
Signature · 1 line
2.25.1
Frederic Weisbecker
maintainerPATCH 2/4 rcu: No need to reset the poll request flag before completion
In reply to Frederic WeisbeckerThe flag allowing to requeue the polling work is reset before the polling even starts. However there is no point in having two competing polling on the same grace period. Just reset the flag once we have completed the grace period only.
--- kernel/rcu/tree_exp.h | 1 - 1 file changed, 1 deletion(-)
kernel/rcu/tree_exp.h +0 -1
@@ -911,7 +911,6 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; - rnp->exp_seq_poll_rq |=0x1 ;raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags);if (s &0x1 )return ;
Signature · 1 line
2.25.1
Frederic Weisbecker
maintainerPATCH 4/4 rcu: Name internal polling flag
In reply to Frederic WeisbeckerGive a proper self-explanatory name to the expedited grace period internal polling flag.
--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)
kernel/rcu/rcu.h +5 -0
@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1 +/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current().
kernel/rcu/tree.c +1 -1
@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }
kernel/rcu/tree_exp.h +5 -4
@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s; s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }
@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);
Signature · 1 line
2.25.1
Re: [PATCH 4/4] rcu: Name internal polling flag
Quoted 19 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +0
-0
kernel/rcu/rcu.h line 36
Quoted 7 lines
kernel/rcu/rcu.h +4
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1
From what I understood, this flag is intended for lifecycle management
of the ->exp_seq_poll_rq; with the flag set meaning that we need to re-poll, which could be used for cases, where there is long gap between 2 polls, such that the sequence wraps around. So, maybe we can name it as RCU_EXP_SEQ_POLL_EXPIRED? However, my understanding could be wrong here.
Thanks
Neeraj
Quoted 49 lines
kernel/rcu/rcu.h +1
-0+/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). kernel/rcu/tree.c +1
-1@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }kernel/rcu/tree_exp.h +5
-4@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s; s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);
Frederic Weisbecker
maintainerRe: [PATCH 4/4] rcu: Name internal polling flag
Quoted 26 lines · Neeraj Upadhyay
Quoted 23 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +2
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals.
Quoted 8 lines
kernel/rcu/rcu.h line 36
Quoted 2 lines
kernel/rcu/rcu.h +2
-0+*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1From what I understood, this flag is intended for lifecycle management of the ->exp_seq_poll_rq; with the flag set meaning that we need to re-poll, which could be used for cases, where there is long gap between 2 polls, such that the sequence wraps around. So, maybe we can name it as
RCU_EXP_SEQ_POLL_EXPIRED? However, my understanding could be wrong here.
It may be confusing also because this patchset slightly extends the role of this bit.
Before the patchset, the role is indeed to deal with wrapping. After the patchset it deals with wrapping and the polling cycle.
I would say that before the patchset, the name could be RCU_EXP_SEQ_POLLABLE, and after the patchset it can indeed be RCU_EXP_SEQ_POLL_EXPIRED.
Paul E. McKenney
maintainerRe: [PATCH 4/4] rcu: Name internal polling flag
In reply to Frederic WeisbeckerQuoted 48 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +5
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1 +/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). kernel/rcu/tree.c +1
-1@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }kernel/rcu/tree_exp.h +0
-0
kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 916
Quoted 8 lines
kernel/rcu/tree_exp.h +1
-1@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s; s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);
One additional question. If we re-read rnp->exp_seq_poll_rq on each pass through the loop, wouldn't we have less trouble with counter wrap?
Thanx, Paul
Quoted 22 lines
kernel/rcu/tree_exp.h +4
-3raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);Signature · 1 line
2.25.1
Frederic Weisbecker
maintainerRe: [PATCH 4/4] rcu: Name internal polling flag
Quoted 51 lines · Paul E. McKenney
Quoted 50 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +5
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1 +/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). kernel/rcu/tree.c +1
-1@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }kernel/rcu/tree_exp.h +0
-0@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s;
Quoted 9 lines
kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 916
Quoted 6 lines
kernel/rcu/tree_exp.h +1
-1s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);One additional question. If we re-read
rnp->exp_seq_poll_rqon each pass through the loop, wouldn't we have less trouble with counter wrap?
We can indeed do that, though it won't eliminate the possibility of wrapping.
Quoted 25 lines
Thanx, Paul
Quoted 22 lines
kernel/rcu/tree_exp.h +4
-3raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);Signature · 1 line
2.25.1
Paul E. McKenney
maintainerRe: [PATCH 4/4] rcu: Name internal polling flag
Quoted 62 lines · Frederic Weisbecker
Quoted 51 lines · Paul E. McKenney
Quoted 50 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +5
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1 +/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). kernel/rcu/tree.c +1
-1@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }kernel/rcu/tree_exp.h +0
-0@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s;Quoted 9 lines
kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 916
Quoted 6 lines
kernel/rcu/tree_exp.h +1
-1s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);One additional question. If we re-read
rnp->exp_seq_poll_rqon each pass through the loop, wouldn't we have less trouble with counter wrap?
Quoted 1 line
We can indeed do that, though it won't eliminate the possibility of wrapping.
True. But in conjunction with an exact check for expired grace-period sequence number, it reduces the maximum addtional penalty for wrapping to two grace periods.
Thanx, Paul
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_exp.h +4
-3raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);Signature · 1 line
2.25.1
Paul E. McKenney
maintainerRe: [PATCH 4/4] rcu: Name internal polling flag
Quoted 63 lines · Paul E. McKenney
Quoted 62 lines · Frederic Weisbecker
Quoted 60 lines · Paul E. McKenney
Quoted 48 lines · Frederic Weisbecker
Give a proper self-explanatory name to the expedited grace period internal polling flag.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/rcu.h | 5 +++++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_exp.h | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-)kernel/rcu/rcu.h +5
-0@@ -30,6 +30,11 @@#define RCU_GET_STATE_USE_NORMAL 0x2#define RCU_GET_STATE_BAD_FOR_NORMAL (RCU_GET_STATE_FROM_EXPEDITED | RCU_GET_STATE_USE_NORMAL) +/* +* Low-order bit definitions for polled grace-period internals. +*/ +#define RCU_EXP_SEQ_POLL_DONE 0x1 +/* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). kernel/rcu/tree.c +1
-1@@ -4679,7 +4679,7 @@ static void __init rcu_init_one(void)spin_lock_init (&rnp->exp_lock);mutex_init (&rnp->boost_kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); - rnp->exp_seq_poll_rq =0x1 ; + rnp->exp_seq_poll_rq =RCU_EXP_SEQ_POLL_DONE ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp); } }kernel/rcu/tree_exp.h +0
-0kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 916
Quoted 8 lines
kernel/rcu/tree_exp.h +1
-1@@ -910,14 +910,14 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)unsigned long s; s =READ_ONCE (rnp->exp_seq_poll_rq); -if (s &0x1 ) +if (s &RCU_EXP_SEQ_POLL_DONE )return ;while (!sync_exp_work_done (s))__synchronize_rcu_expedited (true);One additional question. If we re-read
rnp->exp_seq_poll_rqon each pass through the loop, wouldn't we have less trouble with counter wrap?
Quoted 5 lines
Quoted 1 line
We can indeed do that, though it won't eliminate the possibility of wrapping.
True. But in conjunction with an exact check for expired grace-period sequence number, it reduces the maximum addtional penalty for wrapping to two grace periods.
Oh, and I did finally queue this series for testing and further review.
Thanx, Paul
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_exp.h +4
-3raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; -if (!(s &0x1 ) &&sync_exp_work_done (s)) -WRITE_ONCE (rnp->exp_seq_poll_rq, s |0x1 ); +if (!(s &RCU_EXP_SEQ_POLL_DONE ) &&sync_exp_work_done (s)) +WRITE_ONCE (rnp->exp_seq_poll_rq, s |RCU_EXP_SEQ_POLL_DONE );raw_spin_unlock_irqrestore (&rnp->exp_poll_lock, flags); }@@ -946,7 +946,8 @@ unsigned long start_poll_synchronize_rcu_expedited(void) rnp = rdp->mynode;if (rcu_init_invoked ())raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); -if ((rnp->exp_seq_poll_rq &0x1 ) ||ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) { +if ((rnp->exp_seq_poll_rq &RCU_EXP_SEQ_POLL_DONE ) || +ULONG_CMP_LT (rnp->exp_seq_poll_rq, s)) {WRITE_ONCE (rnp->exp_seq_poll_rq, s);if (rcu_init_invoked ())queue_work (rcu_gp_wq, &rnp->exp_poll_wq);Signature · 1 line
2.25.1
Frederic Weisbecker
maintainerRe: [PATCH 2/4] rcu: No need to reset the poll request flag before completion
In reply to Frederic WeisbeckerQuoted 19 lines · Frederic Weisbecker
The flag allowing to requeue the polling work is reset before the polling even starts. However there is no point in having two competing polling on the same grace period. Just reset the flag once we have completed the grace period only.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/tree_exp.h | 1 - 1 file changed, 1 deletion(-)kernel/rcu/tree_exp.h +0
-0
kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 914
Quoted 4 lines
kernel/rcu/tree_exp.h +0
-1@@ -911,7 +911,6 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; - rnp->exp_seq_poll_rq |=0x1 ;
On a second (or actually twentieth) thought, this patch and all those following make wrapping issues more likely:
* Before this patch, wrapping occuring *after* the 0x1 is set on the beginning of the workqueue is fine. The last vulnerable wrapping scenario is when the wrapping happens before we reach the beginning of the workqueue execution that sets the 0x1, so the work may happen not to be queued.
* After this patch, wrapping occuring *before* the GP completion in the workqueue will be ignored and fail. Still unlikely, but less unlikely than before this patch.
So please revert this series. Only the first patch "rcu: Remove needless polling work requeue for further waiter" still seem to make sense.
Thanks.
Paul E. McKenney
maintainerRe: [PATCH 2/4] rcu: No need to reset the poll request flag before completion
Quoted 34 lines · Frederic Weisbecker
Quoted 19 lines · Frederic Weisbecker
The flag allowing to requeue the polling work is reset before the polling even starts. However there is no point in having two competing polling on the same grace period. Just reset the flag once we have completed the grace period only.
Signed-off-by Frederic Weisbecker <[email protected]>Cc Neeraj Upadhyay <[email protected]>Cc Boqun Feng <[email protected]>Cc Uladzislau Rezki <[email protected]>Cc Joel Fernandes <[email protected]>--- kernel/rcu/tree_exp.h | 1 - 1 file changed, 1 deletion(-)kernel/rcu/tree_exp.h +0
-0kernel/rcu/tree_exp.h static void sync_rcu_do_polled_gp(struct work_struct *wp) · line 914
Quoted 4 lines
kernel/rcu/tree_exp.h +0
-1@@ -911,7 +911,6 @@ static void sync_rcu_do_polled_gp(struct work_struct *wp)raw_spin_lock_irqsave (&rnp->exp_poll_lock, flags); s = rnp->exp_seq_poll_rq; - rnp->exp_seq_poll_rq |=0x1 ;On a second (or actually twentieth) thought, this patch and all those following make wrapping issues more likely:
* Before this patch, wrapping occuring *after* the
0x1is set on the beginning of the workqueue is fine. The last vulnerable wrapping scenario is when the wrapping happens before we reach the beginning of the workqueue execution that sets the0x1, so the work may happen not to be queued.
Quoted 6 lines
* After this patch, wrapping occuring *before* the GP completion in the workqueue will be ignored and fail. Still unlikely, but less unlikely than before this patch.
So please revert this series. Only the first patch "rcu: Remove needless polling work requeue for further waiter" still seem to make sense.
I know that twentieth-thought feeling!
I reverted the following commits, and will remove the original and the reversion of each on my next rebase:
26632dde0c40 ("rcu: No need to reset the poll request flag before completion") b889e463d447 ("rcu: Perform early sequence fetch for polling locklessly") 11eccc01200f ("rcu: Name internal polling flag")
Would it make sense to apply rcu_seq_done_exact(), perhaps as follows? Or is there some reason this would cause problems?
Thanx, Paul
------------------------------------------------------------------------
kernel/rcu/tree_exp.h +1 -1
@@ -992,7 +992,7 @@ bool poll_state_synchronize_rcu_expedited(unsigned long oldstate)WARN_ON_ONCE (!(oldstate &RCU_GET_STATE_FROM_EXPEDITED ));if (oldstate &RCU_GET_STATE_USE_NORMAL )return poll_state_synchronize_rcu (oldstate & ~RCU_GET_STATE_BAD_FOR_NORMAL ); -if (!rcu_exp_gp_seq_done (oldstate & ~RCU_SEQ_STATE_MASK )) +if (!rcu_seq_done_exact (&rcu_state.expedited_sequence, oldstate & ~RCU_SEQ_STATE_MASK ))return false;smp_mb ();/* Ensure GP ends before subsequent accesses. */ return true;