jrcu: support lazy / not-lazy end-of-batch recognition

PATCH In review Joe Korty 1 participant 1 message Mar 14, 2011 Scheduler

lore

Share & tools

lore

Joe Korty

author

2011-03-14 03:52 UTC #0 lore raw

PATCH 7/9 jrcu: support lazy / not-lazy end-of-batch recognition

jrcu: make lazy/not-lazy end-of-batch recognition a config option.

Some mb() are not needed for correct operation, they just make JRCU recognize end-of-batch at the earliest possible moment. Mark those semi-optional mb's with an #ifdef.

Signed-off-by Joe Korty <[email protected]>

init/Kconfig +17 -0

Index: b/init/Kconfig
===================================================================
@@ -434,6 +434,23 @@ config JRCU_DAEMON
 	  Required. The context switch when leaving the daemon is needed
 	  to get the CPU to reliably participate in end-of-batch processing.
 
+config JRCU_LAZY
+	bool "Should JRCU be lazy recognizing end-of-batch"
+	depends on JRCU
+	default n
+	help
+	  If you say Y here, JRCU will on occasion fail to recognize
+	  end-of-batch for an rcu period or two.
+
+	  If you say N here, JRCU will be more aggressive; in fact it
+	  will always recognize end-of-batch at the earliest possible time.
+
+	  Being lazy should be fractionally more efficient in that JRCU
+	  inserts fewer memory barriers along some high performance kernel
+	  code paths.
+
+	  If unsure, say N.
+
 config PREEMPT_COUNT_CPU
 	# bool "Let one CPU look at another CPUs preemption count"
 	bool

include/linux/preempt.h +26 -2

Index: b/include/linux/preempt.h
===================================================================
@@ -10,12 +10,36 @@
 #include <linux/linkage.h>
 #include <linux/list.h>
 
+/* cannot include rcupdate.h here, so open-code this */
+
+#if defined(CONFIG_JRCU)
+# define __add_preempt_count(val) do { \
+	int newval = (preempt_count() += (val)); \
+	if (newval == (val)) \
+		smp_wmb(); \
+} while (0)
+#else
+# define __add_preempt_count(val) do { preempt_count() += (val); } while (0)
+#endif
+
+#if defined(CONFIG_JRCU_LAZY) || !defined(CONFIG_JRCU)
+# define __sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
+#else
+# define __sub_preempt_count(val) do { \
+	int newval = (preempt_count() -= (val)); \
+	if (newval == 0) { \
+		/* race with preemption OK, preempt will do the mb for us */ \
+		smp_wmb(); \
+	} \
+} while (0)
+#endif
+
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
   extern void add_preempt_count(int val);
   extern void sub_preempt_count(int val);
 #else
-# define add_preempt_count(val)	do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val)	do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val) __add_preempt_count(val)
+# define sub_preempt_count(val) __sub_preempt_count(val)
 #endif
 
 #define inc_preempt_count() add_preempt_count(1)

kernel/jrcu.c +1 -3

Index: b/kernel/jrcu.c
===================================================================
@@ -154,9 +154,7 @@ static inline void rcu_eob(int cpu)
 	struct rcu_data *rd = &rcu_data[cpu];
 	if (unlikely(rd->wait)) {
 		rd->wait = 0;
-#ifdef CONFIG_RCU_PARANOID
-		/* not needed, we can tolerate some fuzziness on exactly
-		 * when other CPUs see the above write insn. */
+#ifndef CONFIG_JRCU_LAZY
 		smp_wmb();
 #endif
 	}

kernel/sched.c +2 -2

Index: b/kernel/sched.c
===================================================================
@@ -3829,7 +3829,7 @@ void __kprobes add_preempt_count(int val
 	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
 		return;
 #endif
-	preempt_count() += val;
+	__add_preempt_count(val);
 #ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Spinlock count overflowing soon?

@@ -3860,7 +3860,7 @@ void __kprobes sub_preempt_count(int val
 
 	if (preempt_count() == val)
 		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
-	preempt_count() -= val;
+	__sub_preempt_count(val);
 }
 EXPORT_SYMBOL(sub_preempt_count);