RCU-sync updates for v5.3
Subsystems (6)
cover RCU-sync updates for v5.3
Hello!
This series contains flavor-consolidation updates to RCU-sync:
1. Kill rcu_sync_type/gp_type, courtesy of Oleg Nesterov.
2. Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem in
uprobes, courtesy of Oleg Nesterov.
3. Add DEFINE_PERCPU_RWSEM(), use it to initialize
cgroup_threadgroup_rwsem in percpu-rwsem, courtesy of Oleg Nesterov.
4. Simplify the state machine, courtesy of Oleg Nesterov.
Thanx, Paul
------------------------------------------------------------------------
include/linux/percpu-rwsem.h | 10 + include/linux/rcu_sync.h | 40 ++----- kernel/cgroup/cgroup.c | 3 kernel/events/uprobes.c | 4 kernel/locking/percpu-rwsem.c | 2 kernel/rcu/sync.c | 220 +++++++++++++++++++----------------------- 6 files changed, 121 insertions(+), 158 deletions(-)
PATCH 1/4 rcu/sync: Kill rcu_sync_type/gp_type
From: Oleg Nesterov <[email protected]>
Now that the RCU flavors have been consolidated, rcu_sync_type makes no sense because none of internal update functions aside from .held() depend on gp_type. This commit therefore removes this field and consolidates the relevant code.
--- include/linux/percpu-rwsem.h | 2 +- include/linux/rcu_sync.h | 36 +++++++---------------- kernel/locking/percpu-rwsem.c | 2 +- kernel/rcu/sync.c | 55 ++++------------------------------- 4 files changed, 17 insertions(+), 78 deletions(-)
include/linux/percpu-rwsem.h +1 -1
@@ -20,7 +20,7 @@ struct percpu_rw_semaphore {#define DEFINE_STATIC_PERCPU_RWSEM (name) \ static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ static struct percpu_rw_semaphore name = { \ - .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC), \ + .rss = __RCU_SYNC_INITIALIZER(name.rss), \ .read_count = &__percpu_rwsem_rc_##name, \ .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ .writer = __RCUWAIT_INITIALIZER(name.writer), \
include/linux/rcu_sync.h +10 -26
@@ -13,8 +13,6 @@#include <linux/wait.h> #include <linux/rcupdate.h> -enum rcu_sync_type {RCU_SYNC ,RCU_SCHED_SYNC ,RCU_BH_SYNC }; -/* Structure to mediate between updaters and fastpath-using readers. */ struct rcu_sync {int gp_state;
@@ -23,52 +21,38 @@ struct rcu_sync {int cb_state; struct rcu_head cb_head; - - enum rcu_sync_type gp_type; }; -extern void rcu_sync_lockdep_assert (struct rcu_sync *); -/** * rcu_sync_is_idle() - Are readers permitted to use their fastpaths? * @rsp: Pointer to rcu_sync structure to use for synchronization * -* Returns true if readers are permitted to use their fastpaths. -* Must be invoked within an RCU read-side critical section whose -* flavor matches that of the rcu_sync struture. +* Returns true if readers are permitted to use their fastpaths. Must be +* invoked within some flavor of RCU read-side critical section. */ static inline bool rcu_sync_is_idle (struct rcu_sync *rsp) { -#ifdef CONFIG_PROVE_RCU -rcu_sync_lockdep_assert (rsp); -#endif +RCU_LOCKDEP_WARN (!rcu_read_lock_held () && + !rcu_read_lock_bh_held () && + !rcu_read_lock_sched_held (), +"suspicious rcu_sync_is_idle() usage" );return !rsp->gp_state;/* GP_IDLE */ } -extern void rcu_sync_init (struct rcu_sync *,enum rcu_sync_type ); +extern void rcu_sync_init (struct rcu_sync *);extern void rcu_sync_enter_start (struct rcu_sync *);extern void rcu_sync_enter (struct rcu_sync *);extern void rcu_sync_exit (struct rcu_sync *);extern void rcu_sync_dtor (struct rcu_sync *); -#define __RCU_SYNC_INITIALIZER (name, type) { \ +#define __RCU_SYNC_INITIALIZER (name) { \ .gp_state = 0, \ .gp_count = 0, \ .gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait), \ .cb_state = 0, \ - .gp_type = type, \ } -#define __DEFINE_RCU_SYNC (name, type) \ - struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type) - -#define DEFINE_RCU_SYNC (name) \ - __DEFINE_RCU_SYNC(name, RCU_SYNC) - -#define DEFINE_RCU_SCHED_SYNC (name) \ - __DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC) - -#define DEFINE_RCU_BH_SYNC (name) \ - __DEFINE_RCU_SYNC(name, RCU_BH_SYNC) +#define DEFINE_RCU_SYNC (name) \ + struct rcu_sync name = __RCU_SYNC_INITIALIZER(name)#endif /* _LINUX_RCU_SYNC_H_ */
kernel/locking/percpu-rwsem.c +1 -1
@@ -17,7 +17,7 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,return -ENOMEM ;/* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */ -rcu_sync_init (&sem->rss,RCU_SCHED_SYNC ); +rcu_sync_init (&sem->rss);__init_rwsem (&sem->rw_sem, name, rwsem_key);rcuwait_init (&sem->writer); sem->readers_block =0 ;
kernel/rcu/sync.c +5 -50
98 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 2/4 uprobes: Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem
In reply to Paul E. McKenneyFrom: Oleg Nesterov <[email protected]>
Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem.
--- kernel/events/uprobes.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/events/uprobes.c +1 -3
@@ -46,7 +46,7 @@ static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ ];#define uprobes_mmap_hash (v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) -static struct percpu_rw_semaphore dup_mmap_sem; +DEFINE_STATIC_PERCPU_RWSEM (dup_mmap_sem);/* Have a copy of original instruction */ #define UPROBE_COPY_INSN 0
@@ -2302,7 +2302,5 @@ void __init uprobes_init(void)for (i =0 ; i <UPROBES_HASH_SZ ; i++)mutex_init (&uprobes_mmap_mutex[i]); -BUG_ON (percpu_init_rwsem (&dup_mmap_sem)); -BUG_ON (register_die_notifier (&uprobe_exception_nb)); }
Signature · 1 line
2.17.1
PATCH 3/4 locking/percpu-rwsem: Add DEFINE_PERCPU_RWSEM(), use it to initialize cgroup_threadgroup_rwsem
In reply to Paul E. McKenneyFrom: Oleg Nesterov <[email protected]>
Turn DEFINE_STATIC_PERCPU_RWSEM() into __DEFINE_PERCPU_RWSEM() with the additional "is_static" argument to introduce DEFINE_PERCPU_RWSEM().
Change cgroup.c to use DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem).
--- include/linux/percpu-rwsem.h | 8 ++++++-- kernel/cgroup/cgroup.c | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-)
include/linux/percpu-rwsem.h +6 -2
@@ -17,14 +17,18 @@ struct percpu_rw_semaphore {int readers_block; }; -#define DEFINE_STATIC_PERCPU_RWSEM (name) \ +#define __DEFINE_PERCPU_RWSEM (name, is_static) \ static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ -static struct percpu_rw_semaphore name = { \ +is_static struct percpu_rw_semaphore name = { \ .rss = __RCU_SYNC_INITIALIZER(name.rss), \ .read_count = &__percpu_rwsem_rc_##name, \ .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ .writer = __RCUWAIT_INITIALIZER(name.writer), \ } +#define DEFINE_PERCPU_RWSEM(name) \ + __DEFINE_PERCPU_RWSEM(name, /* not static */) +#define DEFINE_STATIC_PERCPU_RWSEM (name) \ + __DEFINE_PERCPU_RWSEM(name, static)extern int __percpu_down_read (struct percpu_rw_semaphore *,int );extern void __percpu_up_read (struct percpu_rw_semaphore *);
kernel/cgroup/cgroup.c +1 -2
@@ -101,7 +101,7 @@ static DEFINE_SPINLOCK(cgroup_idr_lock);*/ static DEFINE_SPINLOCK (cgroup_file_kn_lock); -struct percpu_rw_semaphore cgroup_threadgroup_rwsem; +DEFINE_PERCPU_RWSEM (cgroup_threadgroup_rwsem);#define cgroup_assert_mutex_or_rcu_locked () \ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
@@ -5616,7 +5616,6 @@ int __init cgroup_init(void)int ssid;BUILD_BUG_ON (CGROUP_SUBSYS_COUNT >16 ); -BUG_ON (percpu_init_rwsem (&cgroup_threadgroup_rwsem));BUG_ON (cgroup_init_cftypes (NULL , cgroup_base_files));BUG_ON (cgroup_init_cftypes (NULL , cgroup1_base_files));
Signature · 1 line
2.17.1
PATCH 4/4 rcu/sync: Simplify the state machine
In reply to Paul E. McKenneyFrom: Oleg Nesterov <[email protected]>
With this patch rcu_sync has a single state variable and the transition rules become really simple:
GP_IDLE - owned by the first rcu_sync_enter() which moves it to
GP_ENTER - owned by rcu-callback which moves it to
GP_PASSED - owned by the last rcu_sync_exit() which moves it to
GP_EXIT - and this is the only "nontrivial" state.
rcu-callback moves it back to GP_IDLE unless another enter() comes before a GP pass.
If rcu-callback is invoked before the next rcu_sync_exit() it must see gp_count incremented by that enter() and set GP_PASSED.
Otherwise, if the next rcu_sync_exit() wins the race, it will move it to
GP_REPLAY - owned by rcu-callback which moves it to GP_EXIT
--- include/linux/rcu_sync.h | 4 +- kernel/rcu/sync.c | 165 ++++++++++++++++++++++----------------- 2 files changed, 96 insertions(+), 73 deletions(-)
include/linux/rcu_sync.h +1 -3
@@ -19,7 +19,6 @@ struct rcu_sync {int gp_count;wait_queue_head_t gp_wait; -int cb_state;struct rcu_head cb_head; };
@@ -36,7 +35,7 @@ static inline bool rcu_sync_is_idle(struct rcu_sync *rsp) !rcu_read_lock_bh_held () && !rcu_read_lock_sched_held (), "suspicious rcu_sync_is_idle() usage"); -return !rsp->gp_state;/* GP_IDLE */ +return !READ_ONCE (rsp->gp_state);/* GP_IDLE */ }extern void rcu_sync_init (struct rcu_sync *);
@@ -49,7 +48,6 @@ extern void rcu_sync_dtor(struct rcu_sync *); .gp_state = 0, \ .gp_count = 0, \ .gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait), \ - .cb_state = 0, \ }#define DEFINE_RCU_SYNC (name) \
kernel/rcu/sync.c +95 -70
232 lines not on this page · raw message
Signature · 1 line
2.17.1