Skip to contentSkip to conversation

RCU-sync updates for v5.3

PATCH New Paul E. McKenney 1 participant 5 messages May 30, 2019

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

cover RCU-sync updates for v5.3

Hello!

This series contains flavor-consolidation updates to RCU-sync:

1. Kill rcu_sync_type/gp_type, courtesy of Oleg Nesterov.

2. Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem in
uprobes, courtesy of Oleg Nesterov.

3. Add DEFINE_PERCPU_RWSEM(), use it to initialize
cgroup_threadgroup_rwsem in percpu-rwsem, courtesy of Oleg Nesterov.

4. Simplify the state machine, courtesy of Oleg Nesterov.

Thanx, Paul

------------------------------------------------------------------------

 include/linux/percpu-rwsem.h  |   10 +
 include/linux/rcu_sync.h      |   40 ++-----
 kernel/cgroup/cgroup.c        |    3 
 kernel/events/uprobes.c       |    4 
 kernel/locking/percpu-rwsem.c |    2 
 kernel/rcu/sync.c             |  220 +++++++++++++++++++-----------------------
 6 files changed, 121 insertions(+), 158 deletions(-)
#2 lore raw

PATCH 2/4 uprobes: Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem

From: Oleg Nesterov <[email protected]>

Use DEFINE_STATIC_PERCPU_RWSEM() to initialize dup_mmap_sem.

---
 kernel/events/uprobes.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/events/uprobes.c +1 -3
index 78f61bfc6b79..97c367f0a9aa 100644
@@ -46,7 +46,7 @@ static DEFINE_SPINLOCK(uprobes_treelock);	/* serialize rbtree access */
 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
 #define uprobes_mmap_hash(v)	(&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
 
-static struct percpu_rw_semaphore dup_mmap_sem;
+DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
 
 /* Have a copy of original instruction */
 #define UPROBE_COPY_INSN	0
@@ -2302,7 +2302,5 @@ void __init uprobes_init(void)
 	for (i = 0; i < UPROBES_HASH_SZ; i++)
 		mutex_init(&uprobes_mmap_mutex[i]);
 
-	BUG_ON(percpu_init_rwsem(&dup_mmap_sem));
-
 	BUG_ON(register_die_notifier(&uprobe_exception_nb));
 }
Signature · 1 line
2.17.1
#3 lore raw

PATCH 3/4 locking/percpu-rwsem: Add DEFINE_PERCPU_RWSEM(), use it to initialize cgroup_threadgroup_rwsem

In reply to Paul E. McKenney

From: Oleg Nesterov <[email protected]>

Turn DEFINE_STATIC_PERCPU_RWSEM() into __DEFINE_PERCPU_RWSEM() with the additional "is_static" argument to introduce DEFINE_PERCPU_RWSEM().

Change cgroup.c to use DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem).

---
 include/linux/percpu-rwsem.h | 8 ++++++--
 kernel/cgroup/cgroup.c       | 3 +--
 2 files changed, 7 insertions(+), 4 deletions(-)
include/linux/percpu-rwsem.h +6 -2
index 6887636ea169..2809b44cbbee 100644
@@ -17,14 +17,18 @@ struct percpu_rw_semaphore {
 	int			readers_block;
 };
 
-#define DEFINE_STATIC_PERCPU_RWSEM(name)				\
+#define __DEFINE_PERCPU_RWSEM(name, is_static)				\
 static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name);		\
-static struct percpu_rw_semaphore name = {				\
+is_static struct percpu_rw_semaphore name = {				\
 	.rss = __RCU_SYNC_INITIALIZER(name.rss),			\
 	.read_count = &__percpu_rwsem_rc_##name,			\
 	.rw_sem = __RWSEM_INITIALIZER(name.rw_sem),			\
 	.writer = __RCUWAIT_INITIALIZER(name.writer),			\
 }
+#define DEFINE_PERCPU_RWSEM(name)		\
+	__DEFINE_PERCPU_RWSEM(name, /* not static */)
+#define DEFINE_STATIC_PERCPU_RWSEM(name)	\
+	__DEFINE_PERCPU_RWSEM(name, static)
 
 extern int __percpu_down_read(struct percpu_rw_semaphore *, int);
 extern void __percpu_up_read(struct percpu_rw_semaphore *);
kernel/cgroup/cgroup.c +1 -2
index 217cec4e22c6..b112e93388dc 100644
@@ -101,7 +101,7 @@ static DEFINE_SPINLOCK(cgroup_idr_lock);
  */
 static DEFINE_SPINLOCK(cgroup_file_kn_lock);
 
-struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
+DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem);
 
 #define cgroup_assert_mutex_or_rcu_locked()				\
 	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
@@ -5616,7 +5616,6 @@ int __init cgroup_init(void)
 	int ssid;
 
 	BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16);
-	BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
 	BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
 	BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files));
 
Signature · 1 line
2.17.1
#4 lore raw

PATCH 4/4 rcu/sync: Simplify the state machine

In reply to Paul E. McKenney

From: Oleg Nesterov <[email protected]>

With this patch rcu_sync has a single state variable and the transition rules become really simple:

GP_IDLE - owned by the first rcu_sync_enter() which moves it to

GP_ENTER - owned by rcu-callback which moves it to

GP_PASSED - owned by the last rcu_sync_exit() which moves it to

GP_EXIT - and this is the only "nontrivial" state.

rcu-callback moves it back to GP_IDLE unless another enter() comes before a GP pass.

If rcu-callback is invoked before the next rcu_sync_exit() it must see gp_count incremented by that enter() and set GP_PASSED.

Otherwise, if the next rcu_sync_exit() wins the race, it will move it to

GP_REPLAY - owned by rcu-callback which moves it to GP_EXIT

[ paulmck: While here, apply READ_ONCE() and WRITE_ONCE() to ->gp_state. ]
[ paulmck: Tweaks to make htmldocs happy. (Reported by kbuild test robot.) ]
---
 include/linux/rcu_sync.h |   4 +-
 kernel/rcu/sync.c        | 165 ++++++++++++++++++++++-----------------
 2 files changed, 96 insertions(+), 73 deletions(-)
include/linux/rcu_sync.h +1 -3
index 87971e85519c..9b83865d24f9 100644
@@ -19,7 +19,6 @@ struct rcu_sync {
 	int			gp_count;
 	wait_queue_head_t	gp_wait;
 
-	int			cb_state;
 	struct rcu_head		cb_head;
 };
 
@@ -36,7 +35,7 @@ static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)
 			 !rcu_read_lock_bh_held() &&
 			 !rcu_read_lock_sched_held(),
 			 "suspicious rcu_sync_is_idle() usage");
-	return !rsp->gp_state; /* GP_IDLE */
+	return !READ_ONCE(rsp->gp_state); /* GP_IDLE */
 }
 
 extern void rcu_sync_init(struct rcu_sync *);
@@ -49,7 +48,6 @@ extern void rcu_sync_dtor(struct rcu_sync *);
 		.gp_state = 0,						\
 		.gp_count = 0,						\
 		.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),	\
-		.cb_state = 0,						\
 	}
 
 #define	DEFINE_RCU_SYNC(name)	\
kernel/rcu/sync.c +95 -70

232 lines not on this page · raw message

Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/4 rcu/sync: Kill rcu_sync_type/gp_type

In reply to Paul E. McKenney

From: Oleg Nesterov <[email protected]>

Now that the RCU flavors have been consolidated, rcu_sync_type makes no sense because none of internal update functions aside from .held() depend on gp_type. This commit therefore removes this field and consolidates the relevant code.

[ paulmck: Added RCU and RCU-bh checks to rcu_sync_is_idle(). ]
[ paulmck: And applied subsequent feedback from Oleg Nesterov. ]
---
 include/linux/percpu-rwsem.h  |  2 +-
 include/linux/rcu_sync.h      | 36 +++++++----------------
 kernel/locking/percpu-rwsem.c |  2 +-
 kernel/rcu/sync.c             | 55 ++++-------------------------------
 4 files changed, 17 insertions(+), 78 deletions(-)
include/linux/percpu-rwsem.h +1 -1
index 03cb4b6f842e..6887636ea169 100644
@@ -20,7 +20,7 @@ struct percpu_rw_semaphore {
 #define DEFINE_STATIC_PERCPU_RWSEM(name)				\
 static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name);		\
 static struct percpu_rw_semaphore name = {				\
-	.rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC),	\
+	.rss = __RCU_SYNC_INITIALIZER(name.rss),			\
 	.read_count = &__percpu_rwsem_rc_##name,			\
 	.rw_sem = __RWSEM_INITIALIZER(name.rw_sem),			\
 	.writer = __RCUWAIT_INITIALIZER(name.writer),			\
include/linux/rcu_sync.h +10 -26
index 6fc53a1345b3..87971e85519c 100644
@@ -13,8 +13,6 @@
 #include <linux/wait.h>
 #include <linux/rcupdate.h>
 
-enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
-
 /* Structure to mediate between updaters and fastpath-using readers.  */
 struct rcu_sync {
 	int			gp_state;
@@ -23,52 +21,38 @@ struct rcu_sync {
 
 	int			cb_state;
 	struct rcu_head		cb_head;
-
-	enum rcu_sync_type	gp_type;
 };
 
-extern void rcu_sync_lockdep_assert(struct rcu_sync *);
-
 /**
  * rcu_sync_is_idle() - Are readers permitted to use their fastpaths?
  * @rsp: Pointer to rcu_sync structure to use for synchronization
  *
- * Returns true if readers are permitted to use their fastpaths.
- * Must be invoked within an RCU read-side critical section whose
- * flavor matches that of the rcu_sync struture.
+ * Returns true if readers are permitted to use their fastpaths.  Must be
+ * invoked within some flavor of RCU read-side critical section.
  */
 static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)
 {
-#ifdef CONFIG_PROVE_RCU
-	rcu_sync_lockdep_assert(rsp);
-#endif
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&
+			 !rcu_read_lock_bh_held() &&
+			 !rcu_read_lock_sched_held(),
+			 "suspicious rcu_sync_is_idle() usage");
 	return !rsp->gp_state; /* GP_IDLE */
 }
 
-extern void rcu_sync_init(struct rcu_sync *, enum rcu_sync_type);
+extern void rcu_sync_init(struct rcu_sync *);
 extern void rcu_sync_enter_start(struct rcu_sync *);
 extern void rcu_sync_enter(struct rcu_sync *);
 extern void rcu_sync_exit(struct rcu_sync *);
 extern void rcu_sync_dtor(struct rcu_sync *);
 
-#define __RCU_SYNC_INITIALIZER(name, type) {				\
+#define __RCU_SYNC_INITIALIZER(name) {					\
 		.gp_state = 0,						\
 		.gp_count = 0,						\
 		.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),	\
 		.cb_state = 0,						\
-		.gp_type = type,					\
 	}
 
-#define	__DEFINE_RCU_SYNC(name, type)	\
-	struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type)
-
-#define DEFINE_RCU_SYNC(name)		\
-	__DEFINE_RCU_SYNC(name, RCU_SYNC)
-
-#define DEFINE_RCU_SCHED_SYNC(name)	\
-	__DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC)
-
-#define DEFINE_RCU_BH_SYNC(name)	\
-	__DEFINE_RCU_SYNC(name, RCU_BH_SYNC)
+#define	DEFINE_RCU_SYNC(name)	\
+	struct rcu_sync name = __RCU_SYNC_INITIALIZER(name)
 
 #endif /* _LINUX_RCU_SYNC_H_ */
kernel/locking/percpu-rwsem.c +1 -1
index f17dad99eec8..48cab93a47fd 100644
@@ -17,7 +17,7 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
 		return -ENOMEM;
 
 	/* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */
-	rcu_sync_init(&sem->rss, RCU_SCHED_SYNC);
+	rcu_sync_init(&sem->rss);
 	__init_rwsem(&sem->rw_sem, name, rwsem_key);
 	rcuwait_init(&sem->writer);
 	sem->readers_block = 0;
kernel/rcu/sync.c +5 -50

98 lines not on this page · raw message

Signature · 1 line
2.17.1