Skip to contentSkip to conversation

preempt: A few headers cleanups and preempt_schedule*() optimizations

RFC PATCH In review Frederic Weisbecker 2 participants 11 messages May 11, 2015

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover preempt: A few headers cleanups and preempt_schedule*() optimizations

Hi,

Optimizing the preempt_count operations (PREEMPT_ACTIVE + preempt_disable()) in a single operation from __schedule() callers was an idea of Linus a few month ago. Here is a re-iteration, with a few header cleanups along the way.

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/preempt

HEAD: c8bda0e6afc163e7e2804d30b9e4301652dc679c

Thanks,
Frederic
---

Frederic Weisbecker (7):
preempt: Merge preempt_mask.h into preempt.h
preempt: Rearrange a few symbols after headers merge
preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
preempt: Disable preemption from preempt_schedule*() callers
sched: Pull preemption disablement duty up to __schedule() callers
preempt: Fix out of date comment
preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()

 arch/m68k/include/asm/irqflags.h |   3 -
 include/linux/bottom_half.h      |   1 -
 include/linux/hardirq.h          |   2 +-
 include/linux/preempt.h          | 123 ++++++++++++++++++++++++++++++++++++++-
 include/linux/preempt_mask.h     | 117 -------------------------------------
 include/linux/sched.h            |   2 +-
 kernel/sched/core.c              |  29 +++------
 lib/radix-tree.c                 |   2 +-
 8 files changed, 133 insertions(+), 146 deletions(-)
#1 lore raw

PATCH 1/7 preempt: Merge preempt_mask.h into preempt.h

preempt_mask.h defines all the preempt_count semantics and related
symbols: preempt, softirq, hardirq, nmi, preempt active, need resched, etc...

preempt.h defines the accessors and mutators of preempt_count.

But there is a messy dependency game around those two header files:

* preempt_mask.h includes preempt.h in order to access preempt_count()

* preempt_mask.h defines all preempt_count semantic and symbols except PREEMPT_NEED_RESCHED that is needed by asm/preempt.h Thus we need to define it from preempt.h, right before including asm/preempt.h, instead of defining it to preempt_mask.h with the other preempt_count symbols. Therefore the preempt_count semantics happen to be spread out.

* We plan to introduce preempt_active_[enter,exit]() to consolidate preempt_schedule*() code. But we'll need to access both preempt_count mutators (preempt_count_add()) and preempt_count symbols (PREEMPT_ACTIVE, PREEMPT_OFFSET). The usual place to define preempt operations is in preempt.h but then we'll need symbols in preempt_mask.h which already includes preempt.h. So we end up with a ressource circle dependency.

Lets merge preempt_mask.h into preempt.h to solve these dependency issues. This way we gather semantic symbols and operation definition of preempt_count in a single file.

This is a dumb copy-paste merge. Further merge re-arrangments are performed in a subsequent patch to ease review.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 arch/m68k/include/asm/irqflags.h |   3 -
 include/linux/bottom_half.h      |   1 -
 include/linux/hardirq.h          |   2 +-
 include/linux/preempt.h          | 111 +++++++++++++++++++++++++++++++++++++
 include/linux/preempt_mask.h     | 117 ---------------------------------------
 include/linux/sched.h            |   2 +-
 lib/radix-tree.c                 |   2 +-
 7 files changed, 114 insertions(+), 124 deletions(-)
 delete mode 100644 include/linux/preempt_mask.h
arch/m68k/include/asm/irqflags.h +0 -3
index a823cd7..b594181 100644
@@ -2,9 +2,6 @@
 #define _M68K_IRQFLAGS_H
 
 #include <linux/types.h>
-#ifdef CONFIG_MMU
-#include <linux/preempt_mask.h>
-#endif
 #include <linux/preempt.h>
 #include <asm/thread_info.h>
 #include <asm/entry.h>
include/linux/bottom_half.h +0 -1
index 86c12c9..8fdcb78 100644
@@ -2,7 +2,6 @@
 #define _LINUX_BH_H
 
 #include <linux/preempt.h>
-#include <linux/preempt_mask.h>
 
 #ifdef CONFIG_TRACE_IRQFLAGS
 extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
include/linux/hardirq.h +1 -1
index cba442e..4e971fa 100644
@@ -1,7 +1,7 @@
 #ifndef LINUX_HARDIRQ_H
 #define LINUX_HARDIRQ_H
 
-#include <linux/preempt_mask.h>
+#include <linux/preempt.h>
 #include <linux/lockdep.h>
 #include <linux/ftrace_irq.h>
 #include <linux/vtime.h>
include/linux/preempt.h +111 -0

117 lines not on this page · raw message

include/linux/preempt_mask.h deleted +0 -117

117 lines not on this page · raw message

include/linux/sched.h +1 -1
index fdca05c..5c13be9 100644
@@ -25,7 +25,7 @@ struct sched_param {
 #include <linux/errno.h>
 #include <linux/nodemask.h>
 #include <linux/mm_types.h>
-#include <linux/preempt_mask.h>
+#include <linux/preempt.h>
 
 #include <asm/page.h>
 #include <asm/ptrace.h>
lib/radix-tree.c +1 -1
index 3d2aa27..061550d 100644
@@ -33,7 +33,7 @@
 #include <linux/string.h>
 #include <linux/bitops.h>
 #include <linux/rcupdate.h>
-#include <linux/preempt_mask.h>		/* in_interrupt() */
+#include <linux/preempt.h>		/* in_interrupt() */
 
 
 /*
Signature · 1 line
2.1.4
#2 lore raw

PATCH 2/7 preempt: Rearrange a few symbols after headers merge

In reply to Frederic Weisbecker

Adjust a few comments, and further integrate a few definitions after the dumb headers copy.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)
include/linux/preempt.h +15 -19
index 8cc0338..37974cd 100644
@@ -10,14 +10,6 @@
 #include <linux/list.h>
 
 /*
- * We use the MSB mostly because its available; see <linux/preempt_mask.h> for
- * the other bits -- can't include that header due to inclusion hell.
- */
-#define PREEMPT_NEED_RESCHED	0x80000000
-
-#include <asm/preempt.h>
-
-/*
  * We put the hardirq and softirq counter into the preemption
  * counter. The bitmask has the following meaning:
  *
@@ -30,11 +22,12 @@
  * there are a few palaeontologic drivers which reenable interrupts in
  * the handler, so we need more than one bit here.
  *
- * PREEMPT_MASK:	0x000000ff
- * SOFTIRQ_MASK:	0x0000ff00
- * HARDIRQ_MASK:	0x000f0000
- *     NMI_MASK:	0x00100000
- * PREEMPT_ACTIVE:	0x00200000
+ *         PREEMPT_MASK:	0x000000ff
+ *         SOFTIRQ_MASK:	0x0000ff00
+ *         HARDIRQ_MASK:	0x000f0000
+ *             NMI_MASK:	0x00100000
+ *       PREEMPT_ACTIVE:	0x00200000
+ * PREEMPT_NEED_RESCHED:	0x80000000
  */
 #define PREEMPT_BITS	8
 #define SOFTIRQ_BITS	8
@@ -64,6 +57,12 @@
 #define PREEMPT_ACTIVE_SHIFT	(NMI_SHIFT + NMI_BITS)
 #define PREEMPT_ACTIVE	(__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
 
+/* We use the MSB mostly because its available */
+#define PREEMPT_NEED_RESCHED	0x80000000
+
+/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
+#include <asm/preempt.h>
+
 #define hardirq_count()	(preempt_count() & HARDIRQ_MASK)
 #define softirq_count()	(preempt_count() & SOFTIRQ_MASK)
 #define irq_count()	(preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
@@ -122,12 +121,6 @@
 #define in_atomic_preempt_off() \
 		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
 
-#ifdef CONFIG_PREEMPT_COUNT
-# define preemptible()	(preempt_count() == 0 && !irqs_disabled())
-#else
-# define preemptible()	0
-#endif
-
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
 extern void preempt_count_add(int val);
 extern void preempt_count_sub(int val);
@@ -160,6 +153,8 @@ do { \
 
 #define preempt_enable_no_resched() sched_preempt_enable_no_resched()
 
+#define preemptible()	(preempt_count() == 0 && !irqs_disabled())
+
 #ifdef CONFIG_PREEMPT
 #define preempt_enable() \
 do { \
@@ -232,6 +227,7 @@ do { \
 #define preempt_disable_notrace()		barrier()
 #define preempt_enable_no_resched_notrace()	barrier()
 #define preempt_enable_notrace()		barrier()
+#define preemptible()				0
 
 #endif /* CONFIG_PREEMPT_COUNT */
 
Signature · 1 line
2.1.4
#3 lore raw

PATCH 3/7 preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET

In reply to Frederic Weisbecker

"CHECK" suggests it's only used as a comparison mask. But now it's used further as a config-conditional preempt disabler offset. Lets disambiguate this name.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
include/linux/preempt.h +4 -4
index 37974cd..4689ef2 100644
@@ -85,9 +85,9 @@
 #define in_nmi()	(preempt_count() & NMI_MASK)
 
 #if defined(CONFIG_PREEMPT_COUNT)
-# define PREEMPT_CHECK_OFFSET 1
+# define PREEMPT_DISABLE_OFFSET 1
 #else
-# define PREEMPT_CHECK_OFFSET 0
+# define PREEMPT_DISABLE_OFFSET 0
 #endif
 
 /*
@@ -103,7 +103,7 @@
  *
  * Work as expected.
  */
-#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_CHECK_OFFSET)
+#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_DISABLE_OFFSET)
 
 /*
  * Are we running in atomic context?  WARNING: this macro cannot
@@ -119,7 +119,7 @@
  * (used by the scheduler, *after* releasing the kernel lock)
  */
 #define in_atomic_preempt_off() \
-		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
+		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
 
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
 extern void preempt_count_add(int val);
Signature · 1 line
2.1.4
#4 lore raw

PATCH 4/7 preempt: Disable preemption from preempt_schedule*() callers

In reply to Frederic Weisbecker

Lets gather the preempt operations (set PREEMPT_ACTIVE and disable preemption) in a single operation. This way we prepare to remove the preemption disablement in __schedule() in order to omptimize this duty on the caller.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 12 ++++++++++++
 kernel/sched/core.c     | 20 ++++++--------------
 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/preempt.h +12 -0
index 4689ef2..45da394 100644
@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);
 #define preempt_count_inc() preempt_count_add(1)
 #define preempt_count_dec() preempt_count_sub(1)
 
+#define preempt_active_enter() \
+do { \
+	preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \
+	barrier(); \
+} while (0)
+
+#define preempt_active_exit() \
+do { \
+	barrier(); \
+	preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \
+} while (0)
+
 #ifdef CONFIG_PREEMPT_COUNT
 
 #define preempt_disable() \
kernel/sched/core.c +6 -14
index 8027cfd..182127a 100644
@@ -2859,15 +2859,14 @@ void __sched schedule_preempt_disabled(void)
 static void __sched notrace preempt_schedule_common(void)
 {
 	do {
-		__preempt_count_add(PREEMPT_ACTIVE);
+		preempt_active_enter();
 		__schedule();
-		__preempt_count_sub(PREEMPT_ACTIVE);
+		preempt_active_exit();
 
 		/*
 		 * Check again in case we missed a preemption opportunity
 		 * between schedule and now.
 		 */
-		barrier();
 	} while (need_resched());
 }
 
@@ -2914,7 +2913,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)
 		return;
 
 	do {
-		__preempt_count_add(PREEMPT_ACTIVE);
+		preempt_active_enter();
 		/*
 		 * Needs preempt disabled in case user_exit() is traced
 		 * and the tracer calls preempt_enable_notrace() causing
@@ -2924,8 +2923,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)
 		__schedule();
 		exception_exit(prev_ctx);
 
-		__preempt_count_sub(PREEMPT_ACTIVE);
-		barrier();
+		preempt_active_exit();
 	} while (need_resched());
 }
 EXPORT_SYMBOL_GPL(preempt_schedule_context);
@@ -2949,17 +2947,11 @@ asmlinkage __visible void __sched preempt_schedule_irq(void)
 	prev_state = exception_enter();
 
 	do {
-		__preempt_count_add(PREEMPT_ACTIVE);
+		preempt_active_enter();
 		local_irq_enable();
 		__schedule();
 		local_irq_disable();
-		__preempt_count_sub(PREEMPT_ACTIVE);
-
-		/*
-		 * Check again in case we missed a preemption opportunity
-		 * between schedule and now.
-		 */
-		barrier();
+		preempt_active_exit();
 	} while (need_resched());
 
 	exception_exit(prev_state);
Signature · 1 line
2.1.4
maintainer
#5 lore raw

Re: [PATCH 4/7] preempt: Disable preemption from preempt_schedule*() callers

Quoted 50 lines · Frederic Weisbecker

Lets gather the preempt operations (set PREEMPT_ACTIVE and disable preemption) in a single operation. This way we prepare to remove the preemption disablement in __schedule() in order to omptimize this duty on the caller.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 12 ++++++++++++
 kernel/sched/core.c     | 20 ++++++--------------
 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/preempt.h +12 -0
index 4689ef2..45da394 100644
@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);
 #define preempt_count_inc() preempt_count_add(1)
 #define preempt_count_dec() preempt_count_sub(1)
 
+#define preempt_active_enter() \
+do { \
+	preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \
+	barrier(); \
+} while (0)
+
+#define preempt_active_exit() \
+do { \
+	barrier(); \
+	preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \
+} while (0)
+
 #ifdef CONFIG_PREEMPT_COUNT
 
 #define preempt_disable() \
kernel/sched/core.c +1 -1
index 8027cfd..182127a 100644
@@ -2859,15 +2859,14 @@ void __sched schedule_preempt_disabled(void)
 static void __sched notrace preempt_schedule_common(void)
 {
 	do {
-		__preempt_count_add(PREEMPT_ACTIVE);
+		preempt_active_enter();
 		__schedule();

kernel/sched/core.c void __sched schedule_preempt_disabled(void) · line 2871

Quoted 10 lines
kernel/sched/core.c +1 -2
-		__preempt_count_sub(PREEMPT_ACTIVE);
+		preempt_active_exit();
 
 		/*
 		 * Check again in case we missed a preemption opportunity
 		 * between schedule and now.
 		 */
-		barrier();
 	} while (need_resched());
 }

So this patch adds a level of preempt_disable; I suspect the goal is to remove the preempt_disable() inside __schedule(), but as it stands this patch is broken, no?

#6 lore raw

PATCH 5/7 sched: Pull preemption disablement duty up to __schedule() callers

In reply to Frederic Weisbecker

Disable preemption from the last caller of __schedule() that needed to carry this duty. This way we can remove this responsability from __scheduler() so to optimize the preempt_count() operations to a single place on preempt_schedule*() functions.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 kernel/sched/core.c | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)
kernel/sched/core.c +3 -6
index 182127a..b23def2 100644
@@ -2723,9 +2723,7 @@ again:
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
  *
- * WARNING: all callers must re-check need_resched() afterward and reschedule
- * accordingly in case an event triggered the need for rescheduling (such as
- * an interrupt waking up a task) while preemption was disabled in __schedule().
+ * WARNING: must be called with preemption disabled!
  */
 static void __sched __schedule(void)
 {
@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)
 	struct rq *rq;
 	int cpu;
 
-	preempt_disable();
 	cpu = smp_processor_id();
 	rq = cpu_rq(cpu);
 	rcu_note_context_switch();
@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)
 		raw_spin_unlock_irq(&rq->lock);
 
 	post_schedule(rq);
-
-	sched_preempt_enable_no_resched();
 }
 
 static inline void sched_submit_work(struct task_struct *tsk)
@@ -2820,7 +2815,9 @@ asmlinkage __visible void __sched schedule(void)
 
 	sched_submit_work(tsk);
 	do {
+		preempt_disable();
 		__schedule();
+		sched_preempt_enable_no_resched();
 	} while (need_resched());
 }
 EXPORT_SYMBOL(schedule);
Signature · 1 line
2.1.4
maintainer
#7 lore raw

Re: [PATCH 5/7] sched: Pull preemption disablement duty up to __schedule() callers

Quoted 21 lines · Frederic Weisbecker

+++ b/kernel/sched/core.c

kernel/sched/core.c +1 -4
@@ -2723,9 +2723,7 @@ again:
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
  *
- * WARNING: all callers must re-check need_resched() afterward and reschedule
- * accordingly in case an event triggered the need for rescheduling (such as
- * an interrupt waking up a task) while preemption was disabled in __schedule().
+ * WARNING: must be called with preemption disabled!
  */
 static void __sched __schedule(void)
 {
@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)
 	struct rq *rq;
 	int cpu;
 
-	preempt_disable();
 	cpu = smp_processor_id();
 	rq = cpu_rq(cpu);
 	rcu_note_context_switch();

kernel/sched/core.c static void __sched __schedule(void) · line 2798

Quoted 6 lines
kernel/sched/core.c +0 -2
@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)
 		raw_spin_unlock_irq(&rq->lock);
 
 	post_schedule(rq);
-
-	sched_preempt_enable_no_resched();
 }

Ah, see, you need to fold this patch into the previous one.

#8 lore raw

Re: [PATCH 5/7] sched: Pull preemption disablement duty up to __schedule() callers

Quoted 24 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker

+++ b/kernel/sched/core.c

kernel/sched/core.c +1 -4
@@ -2723,9 +2723,7 @@ again:
  *          - return from syscall or exception to user-space
  *          - return from interrupt-handler to user-space
  *
- * WARNING: all callers must re-check need_resched() afterward and reschedule
- * accordingly in case an event triggered the need for rescheduling (such as
- * an interrupt waking up a task) while preemption was disabled in __schedule().
+ * WARNING: must be called with preemption disabled!
  */
 static void __sched __schedule(void)
 {
@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)
 	struct rq *rq;
 	int cpu;
 
-	preempt_disable();
 	cpu = smp_processor_id();
 	rq = cpu_rq(cpu);
 	rcu_note_context_switch();
@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)
 		raw_spin_unlock_irq(&rq->lock);
 
Quoted 6 lines

kernel/sched/core.c static void __sched __schedule(void) · line 2798

Quoted 4 lines
kernel/sched/core.c +0 -2
 	post_schedule(rq);
-
-	sched_preempt_enable_no_resched();
 }

Ah, see, you need to fold this patch into the previous one.

Yeah I hesitated. I'm going to fold then.

Thanks.

#9 lore raw

PATCH 6/7 preempt: Fix out of date comment

In reply to Frederic Weisbecker
Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
index 45da394..4057696 100644
@@ -116,7 +116,7 @@
 
 /*
  * Check whether we were atomic before we did preempt_disable():
- * (used by the scheduler, *after* releasing the kernel lock)
+ * (used by the scheduler)
  */
 #define in_atomic_preempt_off() \
 		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
Signature · 1 line
2.1.4
#10 lore raw

PATCH 7/7 preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()

In reply to Frederic Weisbecker

Now that PREEMPT_ACTIVE implies PREEMPT_DISABLE_OFFSET, ignoring PREEMPT_ACTIVE from in_atomic() check isn't useful anymore.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Linus Torvalds <[email protected]>
---
 include/linux/preempt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
index 4057696..a1a00e1 100644
@@ -112,7 +112,7 @@
  * used in the general case to determine whether sleeping is possible.
  * Do not use in_atomic() in driver code.
  */
-#define in_atomic()	((preempt_count() & ~PREEMPT_ACTIVE) != 0)
+#define in_atomic()	(preempt_count() != 0)
 
 /*
  * Check whether we were atomic before we did preempt_disable():
Signature · 1 line
2.1.4