rcu_sync: Cleanup the CONFIG_PROVE_RCU checks
cover rcu_sync: Cleanup the CONFIG_PROVE_RCU checks
Quoted 12 lines · Paul E. McKenney
Quoted 10 lines · Oleg Nesterov
Quoted 2 lines · Paul E. McKenney
This is obsolete, but its replacement is the same patch.
fbe3b97183f84155d81e506b1aa7d2ce986f7a36in linux-rcu.git#experimental I guess?Quoted 2 lines
Oleg, Davidlohr, am I missing something on how
percpu_rwsemor locktorture work?
Quoted 7 lines
Quoted 5 lines
No, I think the patch is fine. Thanks for doing this! I was going to send something like this change too. And in fact I am still thinking about another test which plays with
rcu_synconly, but probably we need some cleanups first (and we need them anyway). I'll try to do this a bit later.I would welcome an rcu_sync-specific torture patch!
I want it much more than you ;) I have already warned you, I'll send more rcu_sync patches. The current code is actually a very early draft which was written during the discussion with Peter a long ago. I sent it unchanged because a) it was already reviewed and b) I tested it a bit in the past.
We can greatly simplify this code and at the same time make it more useful. Actually I already have the patches. The 1st one removes rcu_sync->cb_state and gp_ops->sync(). This makes the state machine almost self-obvious and allows other improvements. See the resulting (pseudo) code at the end.
But again, I'll try very much to write the test before I send the patch.
Until then, let me send this trivial cleanup. The CONFIG_PROVE_RCU code looks trivial but imo really annoying. And it is not complete, so lets document this at least. Plus rcu_lockdep_assert() looks more consistent.
Quoted 11 lines
Quoted 9 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+void torture_percpu_rwsem_init(void) +{ + BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); +} +Aha, we don't really need this... I mean we can use the static initialiser which can also be used by uprobes and cgroups. I'll try to send the patch tomorrow.
Very good, please do!
Hmm. I am lier. I won't send this patch at least today.
The change I had in mind is very simple,
#define DECLARE_PERCPU_RWSEM(sem) \
static DEFINE_PER_CPU(unsigned int, sem##_counters); \
struct percpu_rw_semaphore sem = { \
.fast_read_ctr = &sem##_counters, \
... \
}and yes, uprobes and cgroups can use it.
But somehow I missed that we can't use it to define a _static_ sem,
static DECLARE_PERCPU_RWSEM(sem);
obviously won't work. And damn, I am shy to admit that I spent several hours trying to invent something but failed. Perhaps we can add 2 helpers, DECLARE_PERCPU_RWSEM_GLOBAL() and DECLARE_PERCPU_RWSEM_STATIC().
Oleg.
-------------------------------------------------------------------------------
static const struct {
void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
void (*wait)(void); // TODO: remove this
#ifdef CONFIG_PROVE_RCU
int (*held)(void);
#endif
} gp_ops[] = {
...
};// COMMENT to explain these states
enum { GP_IDLE = 0, GP_ENTER, GP_PASSED, GP_EXIT, GP_REPLAY };
#define rss_lock gp_wait.lock
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1!!!!!!!! // XXX code must be removed when we split rcu_sync_enter() into start + wait // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
static void rcu_sync_func(struct rcu_head *rcu)
{
struct rcu_sync *rsp = container_of(rcu, struct rcu_sync, cb_head);
unsigned long flags;
BUG_ON(rsp->gp_state == GP_IDLE);
BUG_ON(rsp->gp_state == GP_PASSED);
spin_lock_irqsave(&rsp->rss_lock, flags);
if (rsp->gp_count) {
/*
* COMMENT.
*/
rsp->gp_state = GP_PASSED;
wake_up_locked(&rsp->gp_wait);
} else if (rsp->gp_state == GP_REPLAY) {
/*
* A new rcu_sync_exit() has happened; requeue the callback
* to catch a later GP.
*/
rsp->gp_state = GP_EXIT;
gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);
} else {
/*
* We're at least a GP after rcu_sync_exit(); eveybody will now
* have observed the write side critical section. Let 'em rip!.
*/
BUG_ON(rsp->gp_state == GP_ENTER); // XXX
rsp->gp_state = GP_IDLE;
}
spin_unlock_irqrestore(&rsp->rss_lock, flags);
}static void rcu_sync_call(struct rcu_sync *rsp)
{
// TODO:
// This is called by might_sleep() code outside of ->rss_lock,
// we can avoid ->call() in some cases (say rcu_blocking_is_gp())
gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);
}void rcu_sync_enter(struct rcu_sync *rsp)
{
int gp_count, gp_state;
spin_lock_irq(&rsp->rss_lock);
gp_count = rsp->gp_count++;
gp_state = rsp->gp_state;
if (gp_state == GP_IDLE)
rsp->gp_state = GP_ENTER;
spin_unlock_irq(&rsp->rss_lock);
BUG_ON(gp_count != 0 && gp_state == GP_IDLE);
BUG_ON(gp_count == 0 && gp_state == GP_PASSED);
BUG_ON(gp_count == 0 && gp_state == GP_ENTER); // XXX
if (gp_state == GP_IDLE)
rcu_sync_call(rsp);
wait_event(rsp->gp_wait, rsp->gp_state != GP_ENTER);
BUG_ON(rsp->gp_state < GP_PASSED);
}void rcu_sync_exit(struct rcu_sync *rsp)
{
bool need_call;
BUG_ON(rsp->gp_state == GP_IDLE);
BUG_ON(rsp->gp_state == GP_ENTER); // XXX
spin_lock_irq(&rsp->rss_lock);
if (!--rsp->gp_count) {
if (rsp->gp_state == GP_PASSED) {
need_call = true;
rsp->gp_state = GP_EXIT;
} else if (rsp->gp_state == GP_EXIT) {
rsp->gp_state = GP_REPLAY;
}
}
spin_unlock_irq(&rsp->rss_lock);
// Comment to explain why we do not care if another enter()
// and perhaps even exit() comes after spin_unlock().
if (need_call)
rcu_sync_call(rsp);
}void rcu_sync_dtor(struct rcu_sync *rsp)
{
int gp_state;
BUG_ON(rsp->gp_count);
BUG_ON(rsp->gp_state == GP_ENTER); // XXX
BUG_ON(rsp->gp_state == GP_PASSED);
spin_lock_irq(&rsp->rss_lock);
if (rsp->gp_state == GP_REPLAY)
rsp->gp_state = GP_EXIT;
gp_state = rsp->gp_state;
spin_unlock_irq(&rsp->rss_lock);
// TODO: add another wake_up_locked() into rcu_sync_func(),
// use wait_event + spin_lock_wait, remove gp_ops->wait().
if (gp_state != GP_IDLE) {
gp_ops[rsp->gp_type].wait();
BUG_ON(rsp->gp_state != GP_IDLE);
}
}PATCH 1/1 rcu_sync: Cleanup the CONFIG_PROVE_RCU checks
1. Rename __rcu_sync_is_idle() to rcu_sync_lockdep_assert() and change it to use rcu_lockdep_assert().
2. Change rcu_sync_is_idle() to return rsp->gp_state == GP_IDLE unconditonally, this way we can remove the same check from rcu_sync_lockdep_assert() and clearly isolate the debugging code.
Note: rcu_sync_enter()->wait_event(gp_state == GP_PASSED) needs another CONFIG_PROVE_RCU check, the same we do in ->sync(); but this needs some simple preparations in the core RCU code to avoid the code duplication.
--- include/linux/rcu_sync.h | 7 +++---- kernel/rcu/sync.c | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-)
include/linux/rcu_sync.h +3 -4
@@ -40,7 +40,7 @@ struct rcu_sync {enum rcu_sync_type gp_type; }; -extern bool __rcu_sync_is_idle (struct rcu_sync *); +extern void rcu_sync_lockdep_assert (struct rcu_sync *);/** * rcu_sync_is_idle() - Are readers permitted to use their fastpaths?
@@ -53,10 +53,9 @@ extern bool __rcu_sync_is_idle(struct rcu_sync *);static inline bool rcu_sync_is_idle (struct rcu_sync *rsp) {#ifdef CONFIG_PROVE_RCU -return __rcu_sync_is_idle (rsp); -#else -return !rsp->gp_state;/* GP_IDLE */ +rcu_sync_lockdep_assert (rsp);#endif +return !rsp->gp_state;/* GP_IDLE */ }extern void rcu_sync_init (struct rcu_sync *,enum rcu_sync_type );
kernel/rcu/sync.c +3 -3
@@ -63,10 +63,10 @@ enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };#define rss_lock gp_wait.lock#ifdef CONFIG_PROVE_RCU -bool __rcu_sync_is_idle (struct rcu_sync *rsp) +void rcu_sync_lockdep_assert (struct rcu_sync *rsp) { -WARN_ON (!gp_ops[rsp->gp_type].held ()); -return rsp->gp_state ==GP_IDLE ; +rcu_lockdep_assert (gp_ops[rsp->gp_type].held (), +"suspicious rcu_sync_is_idle() usage" ); }#endif
Signature · 1 line
1.5.5.1
Paul E. McKenney
AppliedQuoted 8 lines · Oleg Nesterov
1. Rename
__rcu_sync_is_idle()torcu_sync_lockdep_assert()and change it to usercu_lockdep_assert().2. Change
rcu_sync_is_idle()to returnrsp->gp_state==GP_IDLEunconditonally, this way we can remove the same check fromrcu_sync_lockdep_assert()and clearly isolate the debugging code.
Quoted 6 lines
Note: rcu_sync_enter()->wait_event(gp_state ==
GP_PASSED) needs anotherCONFIG_PROVE_RCUcheck, the same we do in ->sync(); but this needs some simple preparations in the core RCU code to avoid the code duplication.Signed-off-by Oleg Nesterov <[email protected]>
Queued for v4.4, thank you, Oleg!
Thanx, Paul
Quoted 53 lines
--- include/linux/rcu_sync.h | 7 +++---- kernel/rcu/sync.c | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-)include/linux/rcu_sync.h +3
-4@@ -40,7 +40,7 @@ struct rcu_sync {enum rcu_sync_type gp_type; }; -extern bool __rcu_sync_is_idle (struct rcu_sync *); +extern void rcu_sync_lockdep_assert (struct rcu_sync *);/** * rcu_sync_is_idle() - Are readers permitted to use their fastpaths? @@ -53,10 +53,9 @@ extern bool __rcu_sync_is_idle(struct rcu_sync *);static inline bool rcu_sync_is_idle (struct rcu_sync *rsp) {#ifdef CONFIG_PROVE_RCU -return __rcu_sync_is_idle (rsp); -#else -return !rsp->gp_state;/* GP_IDLE */ +rcu_sync_lockdep_assert (rsp);#endif +return !rsp->gp_state;/* GP_IDLE */ }extern void rcu_sync_init (struct rcu_sync *,enum rcu_sync_type );kernel/rcu/sync.c +3
-3@@ -63,10 +63,10 @@ enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };#define rss_lock gp_wait.lock#ifdef CONFIG_PROVE_RCU -bool __rcu_sync_is_idle (struct rcu_sync *rsp) +void rcu_sync_lockdep_assert (struct rcu_sync *rsp) { -WARN_ON (!gp_ops[rsp->gp_type].held ()); -return rsp->gp_state ==GP_IDLE ; +rcu_lockdep_assert (gp_ops[rsp->gp_type].held (), +"suspicious rcu_sync_is_idle() usage" ); }#endif Signature · 1 line
1.5.5.1
Quoted 27 lines · Oleg Nesterov
Quoted 12 lines · Paul E. McKenney
Quoted 10 lines · Oleg Nesterov
Quoted 2 lines · Paul E. McKenney
This is obsolete, but its replacement is the same patch.
fbe3b97183f84155d81e506b1aa7d2ce986f7a36in linux-rcu.git#experimental I guess?Quoted 2 lines
Oleg, Davidlohr, am I missing something on how
percpu_rwsemor locktorture work?Quoted 7 lines
Quoted 5 lines
No, I think the patch is fine. Thanks for doing this! I was going to send something like this change too. And in fact I am still thinking about another test which plays with
rcu_synconly, but probably we need some cleanups first (and we need them anyway). I'll try to do this a bit later.I would welcome an rcu_sync-specific torture patch!
I want it much more than you ;) I have already warned you, I'll send more
rcu_syncpatches. The current code is actually a very early draft which was written during the discussion with Peter a long ago. I sent it unchanged because a) it was already reviewed and b) I tested it a bit in the past.
Quoted 7 lines
We can greatly simplify this code and at the same time make it more useful. Actually I already have the patches. The 1st one removes
rcu_sync->cb_stateandgp_ops->sync(). This makes the state machine almost self-obvious and allows other improvements. See the resulting (pseudo) code at the end.But again, I'll try very much to write the test before I send the patch.
That sounds very good! ;-)
Quoted 31 lines
Until then, let me send this trivial cleanup. The
CONFIG_PROVE_RCUcode looks trivial but imo really annoying. And it is not complete, so lets document this at least. Plusrcu_lockdep_assert()looks more consistent.Quoted 11 lines
Quoted 9 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+void torture_percpu_rwsem_init(void) +{ + BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); +} +Aha, we don't really need this... I mean we can use the static initialiser which can also be used by uprobes and cgroups. I'll try to send the patch tomorrow.
Very good, please do!
Hmm. I am lier. I won't send this patch at least today.
The change I had in mind is very simple,
#define DECLARE_PERCPU_RWSEM(sem) \ static DEFINE_PER_CPU(unsigned int, sem##_counters); \ struct percpu_rw_semaphore sem = { \ .fast_read_ctr = &sem##_counters, \ ... \ }and yes, uprobes and cgroups can use it.
Quoted 7 lines
But somehow I missed that we can't use it to define a _static_ sem,
static DECLARE_PERCPU_RWSEM(sem);obviously won't work. And damn, I am shy to admit that I spent several hours trying to invent something but failed. Perhaps we can add 2 helpers,
DECLARE_PERCPU_RWSEM_GLOBAL()andDECLARE_PERCPU_RWSEM_STATIC().
That is indeed what we do for SRCU for the same reason, DEFINE_SRCU() and DEFINE_STATIC_SRCU(), but with a common __DEFINE_SRCU() doing the actual work.
Thanx, Paul
Quoted 132 lines
Oleg.
-------------------------------------------------------------------------------
static const struct { void (*call)(struct rcu_head *, void (*)(struct rcu_head *)); void (*wait)(void); // TODO: remove this #ifdef CONFIG_PROVE_RCU int (*held)(void); #endif } gp_ops[] = { ... };// COMMENT to explain these states
enum {GP_IDLE= 0,GP_ENTER,GP_PASSED,GP_EXIT,GP_REPLAY};#define
rss_lockgp_wait.lock// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1!!!!!!!! // XXX code must be removed when we split
rcu_sync_enter()into start + wait // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!static void
rcu_sync_func(struct rcu_head *rcu){ struct rcu_sync *rsp = container_of(rcu, struct rcu_sync, cb_head); unsigned long flags; BUG_ON(rsp->gp_state == GP_IDLE); BUG_ON(rsp->gp_state == GP_PASSED); spin_lock_irqsave(&rsp->rss_lock, flags); if (rsp->gp_count) { /* * COMMENT. */ rsp->gp_state = GP_PASSED; wake_up_locked(&rsp->gp_wait); } else if (rsp->gp_state == GP_REPLAY) { /* * A new rcu_sync_exit() has happened; requeue the callback * to catch a later GP. */ rsp->gp_state = GP_EXIT; gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func); } else { /* * We're at least a GP after rcu_sync_exit(); eveybody will now * have observed the write side critical section. Let 'em rip!. */ BUG_ON(rsp->gp_state == GP_ENTER); // XXX rsp->gp_state = GP_IDLE; } spin_unlock_irqrestore(&rsp->rss_lock, flags); }static void
rcu_sync_call(struct rcu_sync *rsp){ // TODO: // This is called by might_sleep() code outside of ->rss_lock, // we can avoid ->call() in some cases (say rcu_blocking_is_gp()) gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func); }void
rcu_sync_enter(struct rcu_sync *rsp){ int gp_count, gp_state; spin_lock_irq(&rsp->rss_lock); gp_count = rsp->gp_count++; gp_state = rsp->gp_state; if (gp_state == GP_IDLE) rsp->gp_state = GP_ENTER; spin_unlock_irq(&rsp->rss_lock); BUG_ON(gp_count != 0 && gp_state == GP_IDLE); BUG_ON(gp_count == 0 && gp_state == GP_PASSED); BUG_ON(gp_count == 0 && gp_state == GP_ENTER); // XXX if (gp_state == GP_IDLE) rcu_sync_call(rsp); wait_event(rsp->gp_wait, rsp->gp_state != GP_ENTER); BUG_ON(rsp->gp_state < GP_PASSED); }void
rcu_sync_exit(struct rcu_sync *rsp){ bool need_call; BUG_ON(rsp->gp_state == GP_IDLE); BUG_ON(rsp->gp_state == GP_ENTER); // XXX spin_lock_irq(&rsp->rss_lock); if (!--rsp->gp_count) { if (rsp->gp_state == GP_PASSED) { need_call = true; rsp->gp_state = GP_EXIT; } else if (rsp->gp_state == GP_EXIT) { rsp->gp_state = GP_REPLAY; } } spin_unlock_irq(&rsp->rss_lock); // Comment to explain why we do not care if another enter() // and perhaps even exit() comes after spin_unlock(). if (need_call) rcu_sync_call(rsp); }void
rcu_sync_dtor(struct rcu_sync *rsp){ int gp_state; BUG_ON(rsp->gp_count); BUG_ON(rsp->gp_state == GP_ENTER); // XXX BUG_ON(rsp->gp_state == GP_PASSED); spin_lock_irq(&rsp->rss_lock); if (rsp->gp_state == GP_REPLAY) rsp->gp_state = GP_EXIT; gp_state = rsp->gp_state; spin_unlock_irq(&rsp->rss_lock); // TODO: add another wake_up_locked() into rcu_sync_func(), // use wait_event + spin_lock_wait, remove gp_ops->wait(). if (gp_state != GP_IDLE) { gp_ops[rsp->gp_type].wait(); BUG_ON(rsp->gp_state != GP_IDLE); } }