Skip to contentSkip to conversation

rcusync: validations + dtor + exclusive

PATCH In review Oleg Nesterov 5 participants 24 messages Oct 4, 2013 – Oct 8, 2013

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover rcusync: validations + dtor + exclusive

Hello.

On top of Peter's "[PATCH 0/3] Optimize the cpu hotplug locking" series.

It seems that Paul and Peter agree with 1-3, and they are simple.

4-5 should be ignored without the explicit ack. Peter seems to dislike the exclusive mode. I disagree but I am not going to argue. (well, at least right now ;)

Oleg.

 include/linux/rcusync.h |   80 ++++++++++++++++++++------------------
 kernel/cpu.c            |    2 +-
 kernel/rcusync.c        |   98 +++++++++++++++++++++++++++++++++-------------
 3 files changed, 113 insertions(+), 67 deletions(-)
#1 lore raw

PATCH 1/5 rcusync: introduce struct rcu_sync_ops

Add the new struct rcu_sync_ops which holds sync/call methods, and turn the function pointers in rcu_sync_struct into the single pointer to struct rcu_sync_ops.

This simplifies the "init" helpers, and this way it is simpler to add the new methods we need, especially ifdef'ed.

---
 include/linux/rcusync.h |   63 +++++++++++++++++++++--------------------------
 kernel/rcusync.c        |   40 ++++++++++++++---------------
 2 files changed, 47 insertions(+), 56 deletions(-)
include/linux/rcusync.h +28 -35
index 7858491..30c6037 100644
@@ -4,6 +4,11 @@
 #include <linux/wait.h>
 #include <linux/rcupdate.h>
 
+struct rcu_sync_ops {
+	void (*sync)(void);
+	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+};
+
 struct rcu_sync_struct {
 	int			gp_state;
 	int			gp_count;
@@ -12,43 +17,9 @@ struct rcu_sync_struct {
 	int			cb_state;
 	struct rcu_head		cb_head;
 
-	void (*sync)(void);
-	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+	struct rcu_sync_ops	*ops;
 };
 
-#define ___RCU_SYNC_INIT(name)						\
-	.gp_state = 0,							\
-	.gp_count = 0,							\
-	.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),		\
-	.cb_state = 0
-
-#define __RCU_SCHED_SYNC_INIT(name) {					\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_sched,					\
-	.call = call_rcu_sched,						\
-}
-
-#define __RCU_BH_SYNC_INIT(name) {					\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_rcu_bh,					\
-	.call = call_rcu_bh,						\
-}
-
-#define __RCU_SYNC_INIT(name) {						\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_rcu,					\
-	.call = call_rcu,						\
-}
-
-#define DEFINE_RCU_SCHED_SYNC(name)					\
-	struct rcu_sync_struct name = __RCU_SCHED_SYNC_INIT(name)
-
-#define DEFINE_RCU_BH_SYNC(name)					\
-	struct rcu_sync_struct name = __RCU_BH_SYNC_INIT(name)
-
-#define DEFINE_RCU_SYNC(name)						\
-	struct rcu_sync_struct name = __RCU_SYNC_INIT(name)
-
 static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
 {
 	return !rss->gp_state; /* GP_IDLE */
@@ -60,5 +31,27 @@ extern void rcu_sync_init(struct rcu_sync_struct *, enum rcu_sync_type);
 extern void rcu_sync_enter(struct rcu_sync_struct *);
 extern void rcu_sync_exit(struct rcu_sync_struct *);
 
+extern struct rcu_sync_ops rcu_sync_ops_array[];
+
+#define __RCU_SYNC_INITIALIZER(name, type) {				\
+		.gp_state = 0,						\
+		.gp_count = 0,						\
+		.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),	\
+		.cb_state = 0,						\
+		.ops = rcu_sync_ops_array + (type),			\
+	}
+
+#define	__DEFINE_RCU_SYNC(name, type)	\
+	struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type)
+
+#define DEFINE_RCU_SYNC(name)		\
+	__DEFINE_RCU_SYNC(name, RCU_SYNC)
+
+#define DEFINE_RCU_SCHED_SYNC(name)	\
+	__DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC)
+
+#define DEFINE_RCU_BH_SYNC(name)	\
+	__DEFINE_RCU_SYNC(name, RCU_BH_SYNC)
+
 #endif /* _LINUX_RCUSYNC_H_ */
 
kernel/rcusync.c +19 -21
index f84176a..1cefb83 100644
@@ -1,4 +1,3 @@
-
 #include <linux/rcusync.h>
 #include <linux/sched.h>
 
@@ -7,27 +6,26 @@ enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
 #define	rss_lock	gp_wait.lock
 
+struct rcu_sync_ops rcu_sync_ops_array[] = {
+	[RCU_SYNC] = {
+		.sync = synchronize_rcu,
+		.call = call_rcu,
+	},
+	[RCU_SCHED_SYNC] = {
+		.sync = synchronize_sched,
+		.call = call_rcu_sched,
+	},
+	[RCU_BH_SYNC] = {
+		.sync = synchronize_rcu_bh,
+		.call = call_rcu_bh,
+	},
+};
+
 void rcu_sync_init(struct rcu_sync_struct *rss, enum rcu_sync_type type)
 {
 	memset(rss, 0, sizeof(*rss));
 	init_waitqueue_head(&rss->gp_wait);
-
-	switch (type) {
-	case RCU_SYNC:
-		rss->sync = synchronize_rcu;
-		rss->call = call_rcu;
-		break;
-
-	case RCU_SCHED_SYNC:
-		rss->sync = synchronize_sched;
-		rss->call = call_rcu_sched;
-		break;
-
-	case RCU_BH_SYNC:
-		rss->sync = synchronize_rcu_bh;
-		rss->call = call_rcu_bh;
-		break;
-	}
+	rss->ops = rcu_sync_ops_array + type;
 }
 
 void rcu_sync_enter(struct rcu_sync_struct *rss)
@@ -44,7 +42,7 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 	BUG_ON(need_wait && need_sync);
 
 	if (need_sync) {
-		rss->sync();
+		rss->ops->sync();
 		rss->gp_state = GP_PASSED;
 		wake_up_all(&rss->gp_wait);
 	} else if (need_wait) {
@@ -81,7 +79,7 @@ static void rcu_sync_func(struct rcu_head *rcu)
 		 * to catch a later GP.
 		 */
 		rss->cb_state = CB_PENDING;
-		rss->call(&rss->cb_head, rcu_sync_func);
+		rss->ops->call(&rss->cb_head, rcu_sync_func);
 	} else {
 		/*
 		 * We're at least a GP after rcu_sync_exit(); eveybody will now
@@ -99,7 +97,7 @@ void rcu_sync_exit(struct rcu_sync_struct *rss)
 	if (!--rss->gp_count) {
 		if (rss->cb_state == CB_IDLE) {
 			rss->cb_state = CB_PENDING;
-			rss->call(&rss->cb_head, rcu_sync_func);
+			rss->ops->call(&rss->cb_head, rcu_sync_func);
 		} else if (rss->cb_state == CB_PENDING) {
 			rss->cb_state = CB_REPLAY;
 		}
Signature · 1 line
1.5.5.1
#2 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 13 lines · Oleg Nesterov

Add the new struct rcu_sync_ops which holds sync/call methods, and turn the function pointers in rcu_sync_struct into the single pointer to struct rcu_sync_ops.

include/linux/rcusync.h +5 -0
+struct rcu_sync_ops {
+       void (*sync)(void);
+       void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+};
+
 struct rcu_sync_struct {
        int                     gp_state;
        int                     gp_count;

include/linux/rcusync.h struct rcu_sync_struct { · line 20

Quoted 6 lines
include/linux/rcusync.h +1 -2
@@ -12,43 +17,9 @@ struct rcu_sync_struct {
        int                     cb_state;
        struct rcu_head         cb_head;
 
-       void (*sync)(void);
-       void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+       struct rcu_sync_ops     *ops;

Ugh.

This interface pretty much guarantees that a compiler can never do anything clever, like know that "hey, you used a static initializer on this thing, and the fields are const, so now know statically what the functions are, and I can just turn the indirect jumps into direct jumps".

I'm not sure gcc is actually that clever, but by making it this kind of ops pointer, I *guarantee* that gcc can never do it.

How about you make the rule be:

- get rid of the stupid "type" enum index thing

- get rid of the "init" thing that sets pointers in the dynamic data
structures. Get rid of the pointer too.

- instead, use a "static const" type descriptor for each type (it
approaches being your "rcu_sync_ops" structure). Pass this in as an argument to all the functions (use a #define per type or something, so that users don't need to do this by hand)

- now every single user passes in that type descriptor.

- together with using a few inline functions, suddenly the "indirect"
jumps through this type descriptor end up actually being nice direct compile-time constants: iow, they get turned into direct jumps.

Tadaa. You actually get good code generation, and you use *less* dynamic memory since you don't have to have this pointer to the descriptor.

Linus

#3 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 24 lines · Linus Torvalds
Quoted 13 lines · Oleg Nesterov

Add the new struct rcu_sync_ops which holds sync/call methods, and turn the function pointers in rcu_sync_struct into the single pointer to struct rcu_sync_ops.

include/linux/rcusync.h +5 -0
+struct rcu_sync_ops {
+       void (*sync)(void);
+       void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+};
+
 struct rcu_sync_struct {
        int                     gp_state;
        int                     gp_count;

include/linux/rcusync.h struct rcu_sync_struct { · line 20

Quoted 6 lines
include/linux/rcusync.h +1 -2
@@ -12,43 +17,9 @@ struct rcu_sync_struct {
        int                     cb_state;
        struct rcu_head         cb_head;
 
-       void (*sync)(void);
-       void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+       struct rcu_sync_ops     *ops;

Ugh.

Quoted 5 lines

This interface pretty much guarantees that a compiler can never do anything clever, like know that "hey, you used a static initializer on this thing, and the fields are const, so now know statically what the functions are, and I can just turn the indirect jumps into direct jumps".

But we do not care? rcu_sync_struct->ops is only used by the writer (slow path). In this case the simpler the better, I think.

Quoted 10 lines

- instead, use a "static const" type descriptor for each type (it
approaches being your "rcu_sync_ops" structure). Pass this in as an argument to all the functions (use a #define per type or something, so that users don't need to do this by hand)

- now every single user passes in that type descriptor.

- together with using a few inline functions, suddenly the "indirect"
jumps through this type descriptor end up actually being nice direct compile-time constants: iow, they get turned into direct jumps.

Hmm. Can't understand, sorry... Could you spell??

I assume you do not suggest to pass the "type" to, say, rcu_sync_enter?

Oleg.

#4 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

In reply to Linus Torvalds
Quoted 3 lines · On Fri, 4 Oct 2013 12:12:25 -0700 Linus Torvalds…

- together with using a few inline functions, suddenly the "indirect"
jumps through this type descriptor end up actually being nice direct compile-time constants: iow, they get turned into direct jumps.

As all the rcu_synchronization() methods (on non UP) are quite expensive, I doubt that this optimization is worth anything.

Quoted 4 lines

Tadaa. You actually get good code generation, and you use *less* dynamic memory since you don't have to have this pointer to the descriptor.

Getting rid of the extra dynamic memory and the pointer business, on the other hand, does make your suggestion worth doing.

-- Steve

#5 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 3 lines · Steven Rostedt

As all the rcu_synchronization() methods (on non UP) are quite expensive, I doubt that this optimization is worth anything.

Maybe. It just annoys me, because afaik, the function that gets called is always static per callsite.

Linus

maintainer
#6 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 7 lines · Linus Torvalds
Quoted 3 lines · Steven Rostedt

As all the rcu_synchronization() methods (on non UP) are quite expensive, I doubt that this optimization is worth anything.

Maybe. It just annoys me, because afaik, the function that gets called is always static per callsite.

Yes, very much so indeed. Worst is that we have no users of the regular RCU and RCU_BH variants and only included them for completeness since the general operation is just as valid for those.

#7 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 7 lines · Peter Zijlstra
Quoted 5 lines · Linus Torvalds
Quoted 3 lines · Steven Rostedt

As all the rcu_synchronization() methods (on non UP) are quite expensive, I doubt that this optimization is worth anything.

Quoted 6 lines
Quoted 2 lines

Maybe. It just annoys me, because afaik, the function that gets called is always static per callsite.

Yes, very much so indeed. Worst is that we have no users of the regular RCU and RCU_BH variants and only included them for completeness since the general operation is just as valid for those.

And personally I think we should keep type/ops for completeness anyway, even if we do not have RCU and RCU_BH users. But perhaps we can kill RCU_SYNC and RCU_BH_SYNC enums/entries until we have a user.

Oleg.

#8 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

In reply to Linus Torvalds
Quoted 8 lines · Linus Torvalds
Quoted 3 lines · Steven Rostedt

As all the rcu_synchronization() methods (on non UP) are quite expensive, I doubt that this optimization is worth anything.

Maybe. It just annoys me, because afaik, the function that gets called is always static per callsite.

I simply can't understand how we can improve the code generation. And more importantly, why does this matter at all in this case.

So unless you strongly object, I'd prefer to keep it as is. Except I forgot to add "const" to _array[], this needs v2.

To me the annoying part is that this patch exports rcu_sync_ops*. We can add "enum rcu_sync_type rcu_sync_struct->gp_type" instead, see the patch below. But then the next "add the CONFIG_PROVE_RCU checks" needs to uninline rcu_sync_is_idle() for CONFIG_PROVE_RCU.

Or do you still think we should do something else?

Oleg.
---

include/linux/rcusync.h +23 -37
index 7858491..988ec33 100644
@@ -4,6 +4,8 @@
 #include <linux/wait.h>
 #include <linux/rcupdate.h>
 
+enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
+
 struct rcu_sync_struct {
 	int			gp_state;
 	int			gp_count;
@@ -12,53 +14,37 @@ struct rcu_sync_struct {
 	int			cb_state;
 	struct rcu_head		cb_head;
 
-	void (*sync)(void);
-	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+	enum rcu_sync_type	gp_type;
 };
 
-#define ___RCU_SYNC_INIT(name)						\
-	.gp_state = 0,							\
-	.gp_count = 0,							\
-	.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),		\
-	.cb_state = 0
-
-#define __RCU_SCHED_SYNC_INIT(name) {					\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_sched,					\
-	.call = call_rcu_sched,						\
-}
-
-#define __RCU_BH_SYNC_INIT(name) {					\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_rcu_bh,					\
-	.call = call_rcu_bh,						\
-}
-
-#define __RCU_SYNC_INIT(name) {						\
-	___RCU_SYNC_INIT(name),						\
-	.sync = synchronize_rcu,					\
-	.call = call_rcu,						\
-}
-
-#define DEFINE_RCU_SCHED_SYNC(name)					\
-	struct rcu_sync_struct name = __RCU_SCHED_SYNC_INIT(name)
-
-#define DEFINE_RCU_BH_SYNC(name)					\
-	struct rcu_sync_struct name = __RCU_BH_SYNC_INIT(name)
-
-#define DEFINE_RCU_SYNC(name)						\
-	struct rcu_sync_struct name = __RCU_SYNC_INIT(name)
-
 static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
 {
 	return !rss->gp_state; /* GP_IDLE */
 }
 
-enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
-
 extern void rcu_sync_init(struct rcu_sync_struct *, enum rcu_sync_type);
 extern void rcu_sync_enter(struct rcu_sync_struct *);
 extern void rcu_sync_exit(struct rcu_sync_struct *);
 
+#define __RCU_SYNC_INITIALIZER(name, type) {				\
+		.gp_state = 0,						\
+		.gp_count = 0,						\
+		.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),	\
+		.cb_state = 0,						\
+		.gp_type = type,					\
+	}
+
+#define	__DEFINE_RCU_SYNC(name, type)	\
+	struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type)
+
+#define DEFINE_RCU_SYNC(name)		\
+	__DEFINE_RCU_SYNC(name, RCU_SYNC)
+
+#define DEFINE_RCU_SCHED_SYNC(name)	\
+	__DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC)
+
+#define DEFINE_RCU_BH_SYNC(name)	\
+	__DEFINE_RCU_SYNC(name, RCU_BH_SYNC)
+
 #endif /* _LINUX_RCUSYNC_H_ */
 
kernel/rcusync.c +22 -21

73 lines not on this page · raw message

maintainer
#9 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 27 lines · Oleg Nesterov

To me the annoying part is that this patch exports rcu_sync_ops*. We can add "enum rcu_sync_type rcu_sync_struct->gp_type" instead, see the patch below. But then the next "add the CONFIG_PROVE_RCU checks" needs to uninline rcu_sync_is_idle() for CONFIG_PROVE_RCU.

Or do you still think we should do something else?

kernel/rcusync.c +2 -2
@@ -44,7 +45,7 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 	BUG_ON(need_wait && need_sync);
 
 	if (need_sync) {
-		rss->sync();
+		gp_ops[rss->gp_type].sync();
 		rss->gp_state = GP_PASSED;
 		wake_up_all(&rss->gp_wait);
 	} else if (need_wait) {
@@ -81,7 +82,7 @@ static void rcu_sync_func(struct rcu_head *rcu)
 		 * to catch a later GP.
 		 */
 		rss->cb_state = CB_PENDING;
-		rss->call(&rss->cb_head, rcu_sync_func);
+		gp_ops[rss->gp_type].call(&rss->cb_head, rcu_sync_func);
 	} else {
 		/*
 		 * We're at least a GP after rcu_sync_exit(); eveybody will now

kernel/rcusync.c void rcu_sync_exit(struct rcu_sync_struct *rss) · line 106

Quoted 8 lines
kernel/rcusync.c +1 -1
@@ -99,7 +100,7 @@ void rcu_sync_exit(struct rcu_sync_struct *rss)
 	if (!--rss->gp_count) {
 		if (rss->cb_state == CB_IDLE) {
 			rss->cb_state = CB_PENDING;
-			rss->call(&rss->cb_head, rcu_sync_func);
+			gp_ops[rss->gp_type].call(&rss->cb_head, rcu_sync_func);
 		} else if (rss->cb_state == CB_PENDING) {
 			rss->cb_state = CB_REPLAY;
 		}

I think Linus meant to have rcu_sync_{enter,exit} as inlines with a const enum argument for the gp_type.

That said; yes that will generate better code, but also more code, and like Steven already argued performance isn't really an issue here since we're going to potentially sleep for a rather long time.

#10 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 7 lines · Peter Zijlstra

I think Linus meant to have rcu_sync_{enter,exit} as inlines with a const enum argument for the gp_type.

That said; yes that will generate better code, but also more code, and like Steven already argued performance isn't really an issue here since we're going to potentially sleep for a rather long time.

Yes, I do not think that we should make them "inline". Plus we need the non-inline rcu_sync_func() anyway.

Thanks again for the last series you sent.

Oleg.

#11 lore raw

Re: [PATCH 1/5] rcusync: introduce struct rcu_sync_ops

Quoted 11 lines · Oleg Nesterov
Quoted 7 lines · Peter Zijlstra

I think Linus meant to have rcu_sync_{enter,exit} as inlines with a const enum argument for the gp_type.

That said; yes that will generate better code, but also more code, and like Steven already argued performance isn't really an issue here since we're going to potentially sleep for a rather long time.

Yes, I do not think that we should make them "inline". Plus we need the non-inline rcu_sync_func() anyway.

Yep. Now if it was rcu_read_lock() rather than synchronize_rcu(), it would be different. ;-)

Thanx, Paul

Quoted 4 lines

Thanks again for the last series you sent.

Oleg.

#12 lore raw

PATCH 2/5 rcusync: add the CONFIG_PROVE_RCU checks

In reply to Oleg Nesterov

It would be nice to validate that the caller of rcu_sync_is_idle() holds the corresponding type of RCU read-side lock. Add the new rcu_sync_ops->held() method and change rcu_sync_is_idle() to WARN() if it returns false.

This obviously penalizes the readers (fast-path), but only if CONFIG_PROVE_RCU.

---
 include/linux/rcusync.h |    6 ++++++
 kernel/rcusync.c        |    9 +++++++++
 2 files changed, 15 insertions(+), 0 deletions(-)
include/linux/rcusync.h +6 -0
index 30c6037..ab787c1 100644
@@ -7,6 +7,9 @@
 struct rcu_sync_ops {
 	void (*sync)(void);
 	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+#ifdef CONFIG_PROVE_RCU
+	int (*held)(void);
+#endif
 };
 
 struct rcu_sync_struct {
@@ -22,6 +25,9 @@ struct rcu_sync_struct {
 
 static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
 {
+#ifdef CONFIG_PROVE_RCU
+	WARN_ON(!rss->ops->held());
+#endif
 	return !rss->gp_state; /* GP_IDLE */
 }
 
kernel/rcusync.c +9 -0
index 1cefb83..21cde9b 100644
@@ -6,18 +6,27 @@ enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
 #define	rss_lock	gp_wait.lock
 
+#ifdef CONFIG_PROVE_RCU
+#define __INIT_HELD(func)	.held = func,
+#else
+#define __INIT_HELD(func)
+#endif
+
 struct rcu_sync_ops rcu_sync_ops_array[] = {
 	[RCU_SYNC] = {
 		.sync = synchronize_rcu,
 		.call = call_rcu,
+		__INIT_HELD(rcu_read_lock_held)
 	},
 	[RCU_SCHED_SYNC] = {
 		.sync = synchronize_sched,
 		.call = call_rcu_sched,
+		__INIT_HELD(rcu_read_lock_sched_held)
 	},
 	[RCU_BH_SYNC] = {
 		.sync = synchronize_rcu_bh,
 		.call = call_rcu_bh,
+		__INIT_HELD(rcu_read_lock_bh_held)
 	},
 };
 
Signature · 1 line
1.5.5.1
#13 lore raw

PATCH 3/5 rcusync: introduce rcu_sync_dtor()

In reply to Oleg Nesterov

Add the new rcu_sync_ops->wait() method and the new helper, rcu_sync_dtor().

It is needed if you are going to, say, kfree(rcu_sync_object). It simply calls ops->wait() to "flush" the potentially pending rcu callback.

---
 include/linux/rcusync.h |    2 ++
 kernel/rcusync.c        |   21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 0 deletions(-)
include/linux/rcusync.h +2 -0
index ab787c1..33864a0 100644
@@ -7,6 +7,7 @@
 struct rcu_sync_ops {
 	void (*sync)(void);
 	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));
+	void (*wait)(void);
 #ifdef CONFIG_PROVE_RCU
 	int (*held)(void);
 #endif
@@ -36,6 +37,7 @@ enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
 extern void rcu_sync_init(struct rcu_sync_struct *, enum rcu_sync_type);
 extern void rcu_sync_enter(struct rcu_sync_struct *);
 extern void rcu_sync_exit(struct rcu_sync_struct *);
+extern void rcu_sync_dtor(struct rcu_sync_struct *);
 
 extern struct rcu_sync_ops rcu_sync_ops_array[];
 
kernel/rcusync.c +21 -0
index 21cde9b..bb311eb 100644
@@ -16,16 +16,19 @@ struct rcu_sync_ops rcu_sync_ops_array[] = {
 	[RCU_SYNC] = {
 		.sync = synchronize_rcu,
 		.call = call_rcu,
+		.wait = rcu_barrier,
 		__INIT_HELD(rcu_read_lock_held)
 	},
 	[RCU_SCHED_SYNC] = {
 		.sync = synchronize_sched,
 		.call = call_rcu_sched,
+		.wait = rcu_barrier_sched,
 		__INIT_HELD(rcu_read_lock_sched_held)
 	},
 	[RCU_BH_SYNC] = {
 		.sync = synchronize_rcu_bh,
 		.call = call_rcu_bh,
+		.wait = rcu_barrier_bh,
 		__INIT_HELD(rcu_read_lock_bh_held)
 	},
 };
@@ -113,3 +116,21 @@ void rcu_sync_exit(struct rcu_sync_struct *rss)
 	}
 	spin_unlock_irq(&rss->rss_lock);
 }
+
+void rcu_sync_dtor(struct rcu_sync_struct *rss)
+{
+	int cb_state;
+
+	BUG_ON(rss->gp_count);
+
+	spin_lock_irq(&rss->rss_lock);
+	if (rss->cb_state == CB_REPLAY)
+		rss->cb_state = CB_PENDING;
+	cb_state = rss->cb_state;
+	spin_unlock_irq(&rss->rss_lock);
+
+	if (cb_state != CB_IDLE) {
+		rss->ops->wait();
+		BUG_ON(rss->cb_state != CB_IDLE);
+	}
+}
Signature · 1 line
1.5.5.1
#14 lore raw

PATCH 4/5 rcusync: introduce rcu_sync_struct->exclusive mode

In reply to Oleg Nesterov

Add rcu_sync_struct->exclusive boolean set by rcu_sync_init(), it obviously controls the exclusiveness of rcu_sync_enter(). This is what percpu_down_write() actually wants.

We turn ->gp_wait into "struct completion gp_comp", it is used as a resource counter in "exclusive" mode. Otherwise we only use its completion->wait member for wait_event/wake_up_all. We never mix the completion/wait_queue_head_t operations.

Note: it would be more clean to do __complete_locked() under ->rss_lock in rcu_sync_exit() in the "else" branch, but we don't have this trivial helper.

---
 include/linux/rcusync.h |   29 ++++++++++++++++-------------
 kernel/cpu.c            |    2 +-
 kernel/rcusync.c        |   22 +++++++++++++++++-----
 3 files changed, 34 insertions(+), 19 deletions(-)
include/linux/rcusync.h +16 -13
index 33864a0..5689f24 100644
@@ -1,8 +1,8 @@
 #ifndef _LINUX_RCUSYNC_H_
 #define _LINUX_RCUSYNC_H_
 
-#include <linux/wait.h>
 #include <linux/rcupdate.h>
+#include <linux/completion.h>
 
 struct rcu_sync_ops {
 	void (*sync)(void);
@@ -16,11 +16,12 @@ struct rcu_sync_ops {
 struct rcu_sync_struct {
 	int			gp_state;
 	int			gp_count;
-	wait_queue_head_t	gp_wait;
+	struct completion	gp_comp;
 
 	int			cb_state;
 	struct rcu_head		cb_head;
 
+	bool			exclusive;
 	struct rcu_sync_ops	*ops;
 };
 
@@ -34,32 +35,34 @@ static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
 
 enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
 
-extern void rcu_sync_init(struct rcu_sync_struct *, enum rcu_sync_type);
+extern void rcu_sync_init(struct rcu_sync_struct *,
+				enum rcu_sync_type, bool excl);
 extern void rcu_sync_enter(struct rcu_sync_struct *);
 extern void rcu_sync_exit(struct rcu_sync_struct *);
 extern void rcu_sync_dtor(struct rcu_sync_struct *);
 
 extern struct rcu_sync_ops rcu_sync_ops_array[];
 
-#define __RCU_SYNC_INITIALIZER(name, type) {				\
+#define __RCU_SYNC_INITIALIZER(name, type, excl) {			\
 		.gp_state = 0,						\
 		.gp_count = 0,						\
-		.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),	\
+		.gp_comp = COMPLETION_INITIALIZER(name.gp_comp),	\
 		.cb_state = 0,						\
+		.exclusive = excl,					\
 		.ops = rcu_sync_ops_array + (type),			\
 	}
 
-#define	__DEFINE_RCU_SYNC(name, type)	\
-	struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type)
+#define	__DEFINE_RCU_SYNC(name, type, excl)	\
+	struct rcu_sync_struct name = __RCU_SYNC_INITIALIZER(name, type, excl)
 
-#define DEFINE_RCU_SYNC(name)		\
-	__DEFINE_RCU_SYNC(name, RCU_SYNC)
+#define DEFINE_RCU_SYNC(name, excl)		\
+	__DEFINE_RCU_SYNC(name, RCU_SYNC, excl)
 
-#define DEFINE_RCU_SCHED_SYNC(name)	\
-	__DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC)
+#define DEFINE_RCU_SCHED_SYNC(name, excl)	\
+	__DEFINE_RCU_SYNC(name, RCU_SCHED_SYNC, excl)
 
-#define DEFINE_RCU_BH_SYNC(name)	\
-	__DEFINE_RCU_SYNC(name, RCU_BH_SYNC)
+#define DEFINE_RCU_BH_SYNC(name, excl)		\
+	__DEFINE_RCU_SYNC(name, RCU_BH_SYNC, excl)
 
 #endif /* _LINUX_RCUSYNC_H_ */
 
kernel/cpu.c +1 -1
index d5f475a..fb1bdf0 100644
@@ -51,7 +51,7 @@ static int cpu_hotplug_disabled;
 
 enum { readers_slow, readers_block };
 
-DEFINE_RCU_SCHED_SYNC(__cpuhp_rss);
+DEFINE_RCU_SCHED_SYNC(__cpuhp_rss, false);
 EXPORT_SYMBOL_GPL(__cpuhp_rss);
 
 DEFINE_PER_CPU(unsigned int, __cpuhp_refcount);
kernel/rcusync.c +17 -5
index bb311eb..667eb7d 100644
@@ -4,7 +4,7 @@
 enum { GP_IDLE = 0, GP_PENDING, GP_PASSED };
 enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
-#define	rss_lock	gp_wait.lock
+#define	rss_lock	gp_comp.wait.lock
 
 #ifdef CONFIG_PROVE_RCU
 #define __INIT_HELD(func)	.held = func,
@@ -33,11 +33,13 @@ struct rcu_sync_ops rcu_sync_ops_array[] = {
 	},
 };
 
-void rcu_sync_init(struct rcu_sync_struct *rss, enum rcu_sync_type type)
+void rcu_sync_init(struct rcu_sync_struct *rss,
+			enum rcu_sync_type type, bool excl)
 {
 	memset(rss, 0, sizeof(*rss));
-	init_waitqueue_head(&rss->gp_wait);
+	init_completion(&rss->gp_comp);
 	rss->ops = rcu_sync_ops_array + type;
+	rss->exclusive = excl;
 }
 
 void rcu_sync_enter(struct rcu_sync_struct *rss)
@@ -56,9 +58,13 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 	if (need_sync) {
 		rss->ops->sync();
 		rss->gp_state = GP_PASSED;
-		wake_up_all(&rss->gp_wait);
+		if (!rss->exclusive)
+			wake_up_all(&rss->gp_comp.wait);
 	} else if (need_wait) {
-		wait_event(rss->gp_wait, rss->gp_state == GP_PASSED);
+		if (!rss->exclusive)
+			wait_event(rss->gp_comp.wait, rss->gp_state == GP_PASSED);
+		else
+			wait_for_completion(&rss->gp_comp);
 	} else {
 		/*
 		 * Possible when there's a pending CB from a rcu_sync_exit().
@@ -105,6 +111,8 @@ static void rcu_sync_func(struct rcu_head *rcu)
 
 void rcu_sync_exit(struct rcu_sync_struct *rss)
 {
+	bool wakeup_excl = rss->exclusive;
+
 	spin_lock_irq(&rss->rss_lock);
 	if (!--rss->gp_count) {
 		if (rss->cb_state == CB_IDLE) {
@@ -113,8 +121,12 @@ void rcu_sync_exit(struct rcu_sync_struct *rss)
 		} else if (rss->cb_state == CB_PENDING) {
 			rss->cb_state = CB_REPLAY;
 		}
+		wakeup_excl = false;
 	}
 	spin_unlock_irq(&rss->rss_lock);
+
+	if (wakeup_excl)
+		complete(&rss->gp_comp);
 }
 
 void rcu_sync_dtor(struct rcu_sync_struct *rss)
Signature · 1 line
1.5.5.1
maintainer
#15 lore raw

Re: [PATCH 4/5] rcusync: introduce rcu_sync_struct->exclusive mode

Quoted 4 lines · Oleg Nesterov

Add rcu_sync_struct->exclusive boolean set by rcu_sync_init(), it obviously controls the exclusiveness of rcu_sync_enter(). This is what percpu_down_write() actually wants.

Quoted 8 lines

We turn ->gp_wait into "struct completion gp_comp", it is used as a resource counter in "exclusive" mode. Otherwise we only use its completion->wait member for wait_event/wake_up_all. We never mix the completion/wait_queue_head_t operations.

Note: it would be more clean to do __complete_locked() under ->rss_lock in rcu_sync_exit() in the "else" branch, but we don't have this trivial helper.

Something equivalent in available functions would be:

	rss->gp_comp.done++;
	__wake_up_locked_key(&rss->gp_comp.wait, TASK_NORMAL, NULL);

Review

Quoted 12 lines

struct rcu_sync_struct {

(quoted diff) +2 -1
 	int			gp_state;
 	int			gp_count;
-	wait_queue_head_t	gp_wait;
+	struct completion	gp_comp;
 
 	int			cb_state;
 	struct rcu_head		cb_head;
 
+	bool			exclusive;
 	struct rcu_sync_ops	*ops;
 };

I suppose we have a hole before or after cb_state to fit exclusive in., now it looks like we're going to create another hole before the *ops pointer.

Review line 7

Quoted 6 lines
(quoted diff) +1 -1
@@ -4,7 +4,7 @@
 enum { GP_IDLE = 0, GP_PENDING, GP_PASSED };
 enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
-#define	rss_lock	gp_wait.lock
+#define	rss_lock	gp_comp.wait.lock

Should we, for convenience, also do:

#define rss_wait gp_comp.wait

Quoted 21 lines
(quoted diff) +4 -2
 #ifdef CONFIG_PROVE_RCU
 #define __INIT_HELD(func)	.held = func,
@@ -33,11 +33,13 @@ struct rcu_sync_ops rcu_sync_ops_array[] = {
 	},
 };
 
-void rcu_sync_init(struct rcu_sync_struct *rss, enum rcu_sync_type type)
+void rcu_sync_init(struct rcu_sync_struct *rss,
+			enum rcu_sync_type type, bool excl)
 {
 	memset(rss, 0, sizeof(*rss));
-	init_waitqueue_head(&rss->gp_wait);
+	init_completion(&rss->gp_comp);
 	rss->ops = rcu_sync_ops_array + type;
+	rss->exclusive = excl;
 }
 
 void rcu_sync_enter(struct rcu_sync_struct *rss)
@@ -56,9 +58,13 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 	if (need_sync) {
 		rss->ops->sync();

Review void rcu_sync_enter(struct rcu_sync_struct *rss) · line 67

Quoted 10 lines
(quoted diff) +6 -2
 		rss->gp_state = GP_PASSED;
-		wake_up_all(&rss->gp_wait);
+		if (!rss->exclusive)
+			wake_up_all(&rss->gp_comp.wait);
 	} else if (need_wait) {
-		wait_event(rss->gp_wait, rss->gp_state == GP_PASSED);
+		if (!rss->exclusive)
+			wait_event(rss->gp_comp.wait, rss->gp_state == GP_PASSED);
+		else
+			wait_for_completion(&rss->gp_comp);

I'm still not entirely sure why we need the completion; we already have the gp_count variable and a waitqueue; together those should be able to implement the condition/semaphore variable, no?

wait_for_completion:

	spin_lock_irq(&rss->rss_lock);
	if (rss->gp_count > 0) {
		__wait_event_locked(rss->gp_wait, (rss->gp_count > 0),
					TASK_UNINTERRUPTIBLE, 1, 0);
	}
	spin_unlock_irq(&rss->rss_lock);

complete:

	bool excl = rss->excl;

	spin_lock_irq(&rss->rss_lock);
	if (!--rss_gp_count) {
		__wake_up_locked_key(&rss->gp_comp.wait, TASK_NORMAL, NULL);
	}
	spin_unlock_irq(&rss->rss_lock);

All we would need would be something like:

include/linux/wait.h +5 -3
@@ -422,7 +422,7 @@ do {									\
 })
 
 
-#define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
+#define __wait_event_locked(wq, condition, state, exclusive, irq) 	\
 ({									\
 	int __ret = 0;							\
 	DEFINE_WAIT(__wait);						\
@@ -431,8 +431,8 @@ do {									\
 	do {								\
 		if (likely(list_empty(&__wait.task_list)))		\
 			__add_wait_queue_tail(&(wq), &__wait);		\
-		set_current_state(TASK_INTERRUPTIBLE);			\
-		if (signal_pending(current)) {				\
+		set_current_state(state);				\
+		if (___wait_signal_pending(state)) {			\
 			__ret = -ERESTARTSYS;				\
 			break;						\
 		}							\
@@ -451,6 +451,8 @@ do {									\
 	__ret;								\
 })
 
+#define __wait_event_interruptible_locked(wq, condition, exclusive, irq)\
+	__wait_event_locked(wq, condition, TASK_INTERRUPTIBLE, exclusive, irq)
 
 /**
  * wait_event_interruptible_locked - sleep until a condition gets true
#16 lore raw

Re: [PATCH 4/5] rcusync: introduce rcu_sync_struct->exclusive mode

Quoted 11 lines · Peter Zijlstra
Quoted 3 lines · Oleg Nesterov

Note: it would be more clean to do __complete_locked() under ->rss_lock in rcu_sync_exit() in the "else" branch, but we don't have this trivial helper.

Something equivalent in available functions would be:

	rss->gp_comp.done++;
	__wake_up_locked_key(&rss->gp_comp.wait, TASK_NORMAL, NULL);

Or __wake_up_locked(&rss->gp_comp.wait, TASK_NORMAL, 1).

Sure, this is what I had in mind. Just I thought that you also dislike the idea to use/add the new helper ;) (and I think it would be better to add the new helper even if we are not going to export it).

Quoted 9 lines
Quoted 9 lines

struct rcu_sync_struct {

(quoted diff) +1 -1
 	int			gp_state;
 	int			gp_count;
-	wait_queue_head_t	gp_wait;
+	struct completion	gp_comp;
 
 	int			cb_state;
 	struct rcu_head		cb_head;
Quoted 7 lines

Review

Quoted 3 lines
(quoted diff) +1 -0
+	bool			exclusive;
 	struct rcu_sync_ops	*ops;
 };

I suppose we have a hole before or after cb_state to fit exclusive in., now it looks like we're going to create another hole before the *ops pointer.

Yes, it probably makes sense to rearrange the members. And, for example, gp_state and cb_state can be "char" and packed together.

Quoted 10 lines

Review line 7

Quoted 6 lines
(quoted diff) +1 -1
@@ -4,7 +4,7 @@
 enum { GP_IDLE = 0, GP_PENDING, GP_PASSED };
 enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };
 
-#define	rss_lock	gp_wait.lock
+#define	rss_lock	gp_comp.wait.lock

Should we, for convenience, also do:

#define rss_wait gp_comp.wait

Yes, I considered this too. OK, will do.

Quoted 9 lines
Quoted 9 lines
(quoted diff) +2 -1
 void rcu_sync_enter(struct rcu_sync_struct *rss)
@@ -56,9 +58,13 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 	if (need_sync) {
 		rss->ops->sync();
 		rss->gp_state = GP_PASSED;
-		wake_up_all(&rss->gp_wait);
+		if (!rss->exclusive)
+			wake_up_all(&rss->gp_comp.wait);
 	} else if (need_wait) {
Quoted 8 lines

Review void rcu_sync_enter(struct rcu_sync_struct *rss) · line 67

Quoted 5 lines
(quoted diff) +4 -1
-		wait_event(rss->gp_wait, rss->gp_state == GP_PASSED);
+		if (!rss->exclusive)
+			wait_event(rss->gp_comp.wait, rss->gp_state == GP_PASSED);
+		else
+			wait_for_completion(&rss->gp_comp);

I'm still not entirely sure why we need the completion; we already have the gp_count variable and a waitqueue;

and we also need the resource counter (like completion->done).

Quoted 8 lines

together those should be able to
implement the condition/semaphore variable, no?

wait_for_completion:

	spin_lock_irq(&rss->rss_lock);
	if (rss->gp_count > 0) {
		__wait_event_locked(rss->gp_wait, (rss->gp_count > 0),

How? I do not even understand what did you mean ;) both conditions are "gp_count > 0".

We simply can not define the CONDITION for wait_event() here, without the additional accounting.

Hmm. perhaps you meant that this should be done before rcu_sync_enter() increments ->gp_count. Perhaps this can work, but the code will be more complex and this way rcu_sync_exit() will always schedule the callback? And again, we do want to increment ->gp_count asap to disable this cb if it is already pending.

Oleg.

maintainer
#17 lore raw

Re: [PATCH 4/5] rcusync: introduce rcu_sync_struct->exclusive mode

Quoted 3 lines · Oleg Nesterov

Hmm. perhaps you meant that this should be done before rcu_sync_enter() increments ->gp_count. Perhaps this can work, but the code will be more complex and this way rcu_sync_exit() will always schedule the callback?

Yah, however see below.

Quoted 2 lines

And again, we do want to increment ->gp_count asap to disable this cb if it is already pending.

Ah indeed, I forgot about that. We'd have to wait until we'd get scheduled to increment gp_count again. However I think we can fix this and the above problem by introduction of rcu_sync_busy() which checks for eiter gp_count or pending waiters.

But yes, slightly more complex code :/

That would yield something like so I suppose:

void rcu_sync_enter(struct rcu_sync_struct *rss)

{
	bool need_wait, need_sync;

	spin_lock_irq(&rss->rss_lock);
	if (rss->exclusive && rss->gp_count) {
		__wait_event_locked(rss->gp_wait, rss->gp_count);
		rss->gp_count++;
		need_wait = need_sync = false;
	} else {
		need_wait = rss->gp_count++;
		need_sync = rss->gp_state == GP_IDLE;
		if (need_sync)
			rss->gp_state = GP_PENDING;
	}
	spin_unlock_irq(&rss->lock);

	if (need_sync) {
		rss->sync();
		rss->gp_state = GP_PASSED;
		wake_up_all(&rss->gp_wait);
	} else if (need_wait) {
		wait_event(rss->gp_wait, rss->gp_state == GP_PASSED);
	} else {
		BUG_ON(rss->gp_state != GP_PASSED);
	}
}

static bool rcu_sync_busy(struct rcu_sync_struct *rss)

{
	return rss->gp_count ||
		(rss->exclusive && waitqueue_active(&rss->gp_wait));
}

static void rcu_sync_func(struct rcu_head *rcu)

{
	struct rcu_sync_struct *rss =
		container_of(rcu, struct rcu_sync_struct, cb_head);
	unsigned long flags;

	BUG_ON(rss->gp_state != GP_PASSED);
	BUG_ON(rss->cb_state == CB_IDLE);

	spin_lock_irqsave(&rss->rss_lock, flags);
	if (rcu_sync_busy(rss)) {
		/*
		 * A new rcu_sync_begin() has happened; drop the callback.
		 */
		rss->cb_state = CB_IDLE;
	} else if (rss->cb_state == CB_REPLAY) {
		/*
		 * A new rcu_sync_exit() has happened; requeue the callback
		 * to catch a later GP.
		 */
		rss->cb_state = CB_PENDING;
		rss->call(&rss->cb_head, rcu_sync_func);
	} else {
		/*
		 * We're at least a GP after rcu_sync_exit(); eveybody will now
		 * have observed the write side critical section. Let 'em rip!.
		 */
		rss->cb_state = CB_IDLE;
		rss->gp_state = GP_IDLE;
	}
	spin_unlock_irqrestore(&rss->rss_lock, flags);
}

void rcu_sync_exit(struct rcu_sync_struct *rss)

{
	spin_lock_irq(&rss->rss_lock);
	if (!--rss->gp_count) {
		if (!rcu_sync_busy(rss)) {
			if (rss->cb_state == CB_IDLE) {
				rss->cb_state = CB_PENDING;
				rss->call(&rss->cb_head, rcu_sync_func);
			} else if (rss->cb_state == CB_PENDING) {
				rss->cb_state = CB_REPLAY;
			}
		} else {
			__wake_up_locked(&rss->gp_wait, TASK_NORMAL, 1);
		}
	}
	spin_unlock_irq(&rss->rss_lock);
}
#18 lore raw

Re: [PATCH 4/5] rcusync: introduce rcu_sync_struct->exclusive mode

Quoted 4 lines · Peter Zijlstra

On Fri, Oct 04, 2013 at 09:56:23PM +0200, Oleg Nesterov wrote:

But yes, slightly more complex code :/

Yes. rcu_sync_busy() adds more obscurity and we need to implement the logic which wait_for_completion already does.

Quoted 9 lines

That would yield something like so I suppose:

void rcu_sync_enter(struct rcu_sync_struct *rss)

{
	bool need_wait, need_sync;

	spin_lock_irq(&rss->rss_lock);
	if (rss->exclusive && rss->gp_count) {
		__wait_event_locked(rss->gp_wait, rss->gp_count);
                                                  ^^^^^^^^^^^^^

I guess you meant !rss->gp_count.

Quoted 12 lines
		rss->gp_count++;
		need_wait = need_sync = false;
	} else {
		need_wait = rss->gp_count++;
		need_sync = rss->gp_state == GP_IDLE;
		if (need_sync)
			rss->gp_state = GP_PENDING;
	}
	spin_unlock_irq(&rss->lock);

	if (need_sync) {
		rss->sync();
Quoted 8 lines
		rss->gp_state = GP_PASSED;
		wake_up_all(&rss->gp_wait);
	} else if (need_wait) {
		wait_event(rss->gp_wait, rss->gp_state == GP_PASSED);
	} else {
		BUG_ON(rss->gp_state != GP_PASSED);
	}
}

I am obviously biased, but imho the code looks worse this way. I like the current simple "need_wait" and "gp_count != 0" logic.

And afaics this is racy,

Quoted 7 lines

static bool rcu_sync_busy(struct rcu_sync_struct *rss)

{
	return rss->gp_count ||
		(rss->exclusive && waitqueue_active(&rss->gp_wait));
}

static void rcu_sync_func(struct rcu_head *rcu)

Quoted 14 lines
{
	struct rcu_sync_struct *rss =
		container_of(rcu, struct rcu_sync_struct, cb_head);
	unsigned long flags;

	BUG_ON(rss->gp_state != GP_PASSED);
	BUG_ON(rss->cb_state == CB_IDLE);

	spin_lock_irqsave(&rss->rss_lock, flags);
	if (rcu_sync_busy(rss)) {
		/*
		 * A new rcu_sync_begin() has happened; drop the callback.
		 */
		rss->cb_state = CB_IDLE;

Yes, but if rcu_sync_exit() does __wake_up_locked(), then autoremove_wake_function() makes waitqueue_active() == F. If the pending rcu_sync_func() takes ->rss_lock first we have a problem.

Easy to fix, but needs more complications.

Or we can simply ignore the fact that rcu_sync_func() can race with wakeup. This can lead to unnecessary sched_sync() but this case is unlikely. IOW,

	spin_lock_irq(&rss->rss_lock);
	if (rss->exclusive)
		wait_event_locked(rss->gp_wait, !rss->gp_count);
	need_wait = rss->gp_count++;
	need_sync = rss->gp_state == GP_IDLE;
	if (need_sync)
		rss->gp_state = GP_PENDING;
	spin_unlock_irq(&rss->lock);

But still I don't like the (imho) unnecessary complications. And the fact we can race with rcu_sync_func() even if this is very unlikely, this just doesn't look good.

Oleg.

maintainer
#19 lore raw

Re: [PATCH 4/5] rcusync: introduce rcu_sync_struct->exclusive mode

Quoted 7 lines · Oleg Nesterov
Quoted 6 lines · Peter Zijlstra

That would yield something like so I suppose:

void rcu_sync_enter(struct rcu_sync_struct *rss)

{
	bool need_wait, need_sync;
Quoted 5 lines
Quoted 3 lines
	spin_lock_irq(&rss->rss_lock);
	if (rss->exclusive && rss->gp_count) {
		__wait_event_locked(rss->gp_wait, rss->gp_count);
                                                  ^^^^^^^^^^^^^

I guess you meant !rss->gp_count.

Uh yes, obviously :-)

Quoted 2 lines

I am obviously biased, but imho the code looks worse this way. I like the current simple "need_wait" and "gp_count != 0" logic.

Yeah, I know.. but it doesn't add the extra variable and doesn't play games with the completion implementation.

Quoted 5 lines

And afaics this is racy,

Yes, but if rcu_sync_exit() does __wake_up_locked(), then autoremove_wake_function() makes waitqueue_active() == F. If the pending rcu_sync_func() takes ->rss_lock first we have a problem.

Ah indeed, it seems I got confused between DECLARE_WAITQUEUE and DEFINE_WAIT; there's too damn many variants there :/

Quoted 15 lines

Easy to fix, but needs more complications.

Or we can simply ignore the fact that rcu_sync_func() can race with wakeup. This can lead to unnecessary sched_sync() but this case is unlikely. IOW,

	spin_lock_irq(&rss->rss_lock);
	if (rss->exclusive)
		wait_event_locked(rss->gp_wait, !rss->gp_count);
	need_wait = rss->gp_count++;
	need_sync = rss->gp_state == GP_IDLE;
	if (need_sync)
		rss->gp_state = GP_PENDING;
	spin_unlock_irq(&rss->lock);
Quoted 3 lines

But still I don't like the (imho) unnecessary complications. And the fact we can race with rcu_sync_func() even if this is very unlikely, this just doesn't look good.

OK.. I'll give up trying to wreck this stuff ;-)

#20 lore raw

PATCH 5/5 rcusync: make rcu_sync_enter() return "bool"

In reply to Oleg Nesterov

Change rcu_sync_enter() to return "need_sync" to let the caller know whether we did the FAST -> SLOW transition or not.

This is particularly useful in exclusive mode, for example percpu_down_write() can avoid clear_fast_ctr() if rcu_sync_enter() returns F.

---
 include/linux/rcusync.h |    2 +-
 kernel/rcusync.c        |    4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)
include/linux/rcusync.h +1 -1
index 5689f24..6f4c75b 100644
@@ -37,7 +37,7 @@ enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };
 
 extern void rcu_sync_init(struct rcu_sync_struct *,
 				enum rcu_sync_type, bool excl);
-extern void rcu_sync_enter(struct rcu_sync_struct *);
+extern bool rcu_sync_enter(struct rcu_sync_struct *);
 extern void rcu_sync_exit(struct rcu_sync_struct *);
 extern void rcu_sync_dtor(struct rcu_sync_struct *);
 
kernel/rcusync.c +3 -1
index 667eb7d..d03d8e5 100644
@@ -42,7 +42,7 @@ void rcu_sync_init(struct rcu_sync_struct *rss,
 	rss->exclusive = excl;
 }
 
-void rcu_sync_enter(struct rcu_sync_struct *rss)
+bool rcu_sync_enter(struct rcu_sync_struct *rss)
 {
 	bool need_wait, need_sync;
 
@@ -73,6 +73,8 @@ void rcu_sync_enter(struct rcu_sync_struct *rss)
 		 */
 		BUG_ON(rss->gp_state != GP_PASSED);
 	}
+
+	return need_sync;
 }
 
 static void rcu_sync_func(struct rcu_head *rcu)
Signature · 1 line
1.5.5.1
maintainer
#21 lore raw
In reply to Oleg Nesterov
Quoted 6 lines · Oleg Nesterov

Hello.

On top of Peter's "[PATCH 0/3] Optimize the cpu hotplug locking" series.

It seems that Paul and Peter agree with 1-3, and they are simple.

Yeah, ACK on those.. shall I pick them up and stuff them through tip with the rest of the lot?

Quoted 3 lines

4-5 should be ignored without the explicit ack. Peter seems to dislike the exclusive mode. I disagree but I am not going to argue. (well, at least right now ;)

Lets have another few rounds of email on that one; maybe I'll get used to it :-)

#22 lore raw
Quoted 10 lines · Peter Zijlstra
Quoted 6 lines · Oleg Nesterov

Hello.

On top of Peter's "[PATCH 0/3] Optimize the cpu hotplug locking" series.

It seems that Paul and Peter agree with 1-3, and they are simple.

Yeah, ACK on those.. shall I pick them up and stuff them through tip with the rest of the lot?

I am good with that.

Quoted 6 lines
Quoted 3 lines

4-5 should be ignored without the explicit ack. Peter seems to dislike the exclusive mode. I disagree but I am not going to argue. (well, at least right now ;)

Lets have another few rounds of email on that one; maybe I'll get used to it :-)

;-) ;-) ;-)

Thanx, Paul

#23 lore raw
In reply to Peter Zijlstra
Quoted 3 lines · Peter Zijlstra

shall I pick them up and stuff them through tip
with the rest of the lot?

This would be great, thanks.

Quoted 6 lines
Quoted 3 lines

4-5 should be ignored without the explicit ack. Peter seems to dislike the exclusive mode. I disagree but I am not going to argue. (well, at least right now ;)

Lets have another few rounds of email on that one; maybe I'll get used to it :-)

OK ;) I'll send v2 anyway.

Oleg.