preempt: A few headers cleanups and preempt_schedule*() optimizations
Subsystems (3)
cover preempt: A few headers cleanups and preempt_schedule*() optimizations
Hi,
Optimizing the preempt_count operations (PREEMPT_ACTIVE + preempt_disable()) in a single operation from __schedule() callers was an idea of Linus a few month ago. Here is a re-iteration, with a few header cleanups along the way.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/preempt
HEAD: c8bda0e6afc163e7e2804d30b9e4301652dc679c
Thanks,
Frederic
---
Frederic Weisbecker (7):
preempt: Merge preempt_mask.h into preempt.h
preempt: Rearrange a few symbols after headers merge
preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
preempt: Disable preemption from preempt_schedule*() callers
sched: Pull preemption disablement duty up to __schedule() callers
preempt: Fix out of date comment
preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
arch/m68k/include/asm/irqflags.h | 3 - include/linux/bottom_half.h | 1 - include/linux/hardirq.h | 2 +- include/linux/preempt.h | 123 ++++++++++++++++++++++++++++++++++++++- include/linux/preempt_mask.h | 117 ------------------------------------- include/linux/sched.h | 2 +- kernel/sched/core.c | 29 +++------ lib/radix-tree.c | 2 +- 8 files changed, 133 insertions(+), 146 deletions(-)
PATCH 1/7 preempt: Merge preempt_mask.h into preempt.h
preempt_mask.h defines all the preempt_count semantics and related
symbols: preempt, softirq, hardirq, nmi, preempt active, need resched, etc...
preempt.h defines the accessors and mutators of preempt_count.
But there is a messy dependency game around those two header files:
* preempt_mask.h includes preempt.h in order to access preempt_count()
* preempt_mask.h defines all preempt_count semantic and symbols except PREEMPT_NEED_RESCHED that is needed by asm/preempt.h Thus we need to define it from preempt.h, right before including asm/preempt.h, instead of defining it to preempt_mask.h with the other preempt_count symbols. Therefore the preempt_count semantics happen to be spread out.
* We plan to introduce preempt_active_[enter,exit]() to consolidate preempt_schedule*() code. But we'll need to access both preempt_count mutators (preempt_count_add()) and preempt_count symbols (PREEMPT_ACTIVE, PREEMPT_OFFSET). The usual place to define preempt operations is in preempt.h but then we'll need symbols in preempt_mask.h which already includes preempt.h. So we end up with a ressource circle dependency.
Lets merge preempt_mask.h into preempt.h to solve these dependency issues. This way we gather semantic symbols and operation definition of preempt_count in a single file.
This is a dumb copy-paste merge. Further merge re-arrangments are performed in a subsequent patch to ease review.
--- arch/m68k/include/asm/irqflags.h | 3 - include/linux/bottom_half.h | 1 - include/linux/hardirq.h | 2 +- include/linux/preempt.h | 111 +++++++++++++++++++++++++++++++++++++ include/linux/preempt_mask.h | 117 --------------------------------------- include/linux/sched.h | 2 +- lib/radix-tree.c | 2 +- 7 files changed, 114 insertions(+), 124 deletions(-) delete mode 100644 include/linux/preempt_mask.h
arch/m68k/include/asm/irqflags.h +0 -3
@@ -2,9 +2,6 @@#define _M68K_IRQFLAGS_H#include <linux/types.h> -#ifdef CONFIG_MMU -#include <linux/preempt_mask.h> -#endif #include <linux/preempt.h> #include <asm/thread_info.h> #include <asm/entry.h>
include/linux/bottom_half.h +0 -1
@@ -2,7 +2,6 @@#define _LINUX_BH_H#include <linux/preempt.h> -#include <linux/preempt_mask.h> #ifdef CONFIG_TRACE_IRQFLAGS extern void __local_bh_disable_ip (unsigned long ip,unsigned int cnt);
include/linux/hardirq.h +1 -1
@@ -1,7 +1,7 @@#ifndef LINUX_HARDIRQ_H #define LINUX_HARDIRQ_H -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <linux/lockdep.h> #include <linux/ftrace_irq.h> #include <linux/vtime.h>
include/linux/preempt.h +111 -0
117 lines not on this page · raw message
include/linux/preempt_mask.h deleted +0 -117
117 lines not on this page · raw message
include/linux/sched.h +1 -1
@@ -25,7 +25,7 @@ struct sched_param {#include <linux/errno.h> #include <linux/nodemask.h> #include <linux/mm_types.h> -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <asm/page.h> #include <asm/ptrace.h>
lib/radix-tree.c +1 -1
@@ -33,7 +33,7 @@#include <linux/string.h> #include <linux/bitops.h> #include <linux/rcupdate.h> -#include <linux/preempt_mask.h> /* in_interrupt() */ +#include <linux/preempt.h> /* in_interrupt() */ /*
Signature · 1 line
2.1.4
PATCH 2/7 preempt: Rearrange a few symbols after headers merge
In reply to Frederic WeisbeckerAdjust a few comments, and further integrate a few definitions after the dumb headers copy.
--- include/linux/preempt.h | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-)
include/linux/preempt.h +15 -19
@@ -10,14 +10,6 @@#include <linux/list.h> /* -* We use the MSB mostly because its available; see <linux/preempt_mask.h> for -* the other bits -- can't include that header due to inclusion hell. -*/ -#define PREEMPT_NEED_RESCHED 0x80000000 - -#include <asm/preempt.h> - -/* * We put the hardirq and softirq counter into the preemption * counter. The bitmask has the following meaning: *
@@ -30,11 +22,12 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 -* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8
@@ -64,6 +57,12 @@#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) +/* We use the MSB mostly because its available */ +#define PREEMPT_NEED_RESCHED 0x80000000 + +/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */ +#include <asm/preempt.h> +#define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)#define irq_count () (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
@@ -122,12 +121,6 @@#define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) -#ifdef CONFIG_PREEMPT_COUNT -# define preemptible () (preempt_count() == 0 && !irqs_disabled()) -#else -# define preemptible () 0 -#endif -#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);extern void preempt_count_sub (int val);
@@ -160,6 +153,8 @@ do { \#define preempt_enable_no_resched () sched_preempt_enable_no_resched() +#define preemptible () (preempt_count() == 0 && !irqs_disabled()) +#ifdef CONFIG_PREEMPT #define preempt_enable () \ do { \
@@ -232,6 +227,7 @@ do { \#define preempt_disable_notrace () barrier()#define preempt_enable_no_resched_notrace () barrier()#define preempt_enable_notrace () barrier() +#define preemptible () 0#endif /* CONFIG_PREEMPT_COUNT */
Signature · 1 line
2.1.4
PATCH 3/7 preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
In reply to Frederic Weisbecker"CHECK" suggests it's only used as a comparison mask. But now it's used further as a config-conditional preempt disabler offset. Lets disambiguate this name.
--- include/linux/preempt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
include/linux/preempt.h +4 -4
@@ -85,9 +85,9 @@#define in_nmi () (preempt_count() & NMI_MASK)#if defined(CONFIG_PREEMPT_COUNT ) -# define PREEMPT_CHECK_OFFSET 1 +# define PREEMPT_DISABLE_OFFSET 1#else -# define PREEMPT_CHECK_OFFSET 0 +# define PREEMPT_DISABLE_OFFSET 0#endif /*
@@ -103,7 +103,7 @@* * Work as expected. */ -#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_CHECK_OFFSET) +#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_DISABLE_OFFSET)/* * Are we running in atomic context? WARNING: this macro cannot
@@ -119,7 +119,7 @@* (used by the scheduler, *after* releasing the kernel lock) */ #define in_atomic_preempt_off () \ - ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) + ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);
Signature · 1 line
2.1.4
PATCH 4/7 preempt: Disable preemption from preempt_schedule*() callers
In reply to Frederic WeisbeckerLets gather the preempt operations (set PREEMPT_ACTIVE and disable preemption) in a single operation. This way we prepare to remove the preemption disablement in __schedule() in order to omptimize this duty on the caller.
--- include/linux/preempt.h | 12 ++++++++++++ kernel/sched/core.c | 20 ++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/preempt.h +12 -0
@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);#define preempt_count_inc () preempt_count_add(1)#define preempt_count_dec () preempt_count_sub(1) +#define preempt_active_enter () \ +do { \ + preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + barrier(); \ +} while (0) + +#define preempt_active_exit () \ +do { \ + barrier(); \ + preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ +} while (0) +#ifdef CONFIG_PREEMPT_COUNT #define preempt_disable () \
kernel/sched/core.c +6 -14
@@ -2859,15 +2859,14 @@ void __sched schedule_preempt_disabled(void) static void __sched notrace preempt_schedule_common(void) {do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); +preempt_active_exit ();/* * Check again in case we missed a preemption opportunity * between schedule and now. */ -barrier (); }while (need_resched ()); }
@@ -2914,7 +2913,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)return ; do { - __preempt_count_add(PREEMPT_ACTIVE); + preempt_active_enter(); /* * Needs preempt disabled in case user_exit() is traced * and the tracer calls preempt_enable_notrace() causing
@@ -2924,8 +2923,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)__schedule ();exception_exit (prev_ctx); -__preempt_count_sub (PREEMPT_ACTIVE ); -barrier (); +preempt_active_exit (); }while (need_resched ()); }EXPORT_SYMBOL_GPL (preempt_schedule_context);
@@ -2949,17 +2947,11 @@ asmlinkage __visible void __sched preempt_schedule_irq(void) prev_state =exception_enter ();do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();local_irq_enable ();__schedule ();local_irq_disable (); -__preempt_count_sub (PREEMPT_ACTIVE ); - -/* -* Check again in case we missed a preemption opportunity -* between schedule and now. -*/ -barrier (); +preempt_active_exit (); }while (need_resched ());exception_exit (prev_state);
Signature · 1 line
2.1.4
Peter Zijlstra
maintainerRe: [PATCH 4/7] preempt: Disable preemption from preempt_schedule*() callers
Quoted 50 lines · Frederic Weisbecker
Lets gather the preempt operations (set
PREEMPT_ACTIVEand disable preemption) in a single operation. This way we prepare to remove the preemption disablement in__schedule()in order to omptimize this duty on the caller.Suggested-by Linus Torvalds <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Linus Torvalds <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- include/linux/preempt.h | 12 ++++++++++++ kernel/sched/core.c | 20 ++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-)include/linux/preempt.h +12
-0@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);#define preempt_count_inc () preempt_count_add(1)#define preempt_count_dec () preempt_count_sub(1) +#define preempt_active_enter () \ +do { \ + preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + barrier(); \ +} while (0) + +#define preempt_active_exit () \ +do { \ + barrier(); \ + preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ +} while (0) +#ifdef CONFIG_PREEMPT_COUNT #define preempt_disable () \kernel/sched/core.c +1
-1@@ -2859,15 +2859,14 @@ void __sched schedule_preempt_disabled(void) static void __sched notrace preempt_schedule_common(void) { do { - __preempt_count_add(PREEMPT_ACTIVE); + preempt_active_enter(); __schedule();
kernel/sched/core.c void __sched schedule_preempt_disabled(void) · line 2871
Quoted 10 lines
kernel/sched/core.c +1
-2-__preempt_count_sub (PREEMPT_ACTIVE ); +preempt_active_exit ();/* * Check again in case we missed a preemption opportunity * between schedule and now. */ -barrier (); }while (need_resched ()); }
So this patch adds a level of preempt_disable; I suspect the goal is to remove the preempt_disable() inside __schedule(), but as it stands this patch is broken, no?
PATCH 5/7 sched: Pull preemption disablement duty up to __schedule() callers
In reply to Frederic WeisbeckerDisable preemption from the last caller of __schedule() that needed to carry this duty. This way we can remove this responsability from __scheduler() so to optimize the preempt_count() operations to a single place on preempt_schedule*() functions.
--- kernel/sched/core.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
kernel/sched/core.c +3 -6
@@ -2723,9 +2723,7 @@ again:* - return from syscall or exception to user-space * - return from interrupt-handler to user-space * -* WARNING: all callers must re-check need_resched() afterward and reschedule -* accordingly in case an event triggered the need for rescheduling (such as -* an interrupt waking up a task) while preemption was disabled in __schedule(). +* WARNING: must be called with preemption disabled! */ static void __sched __schedule(void) {
@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)struct rq *rq;int cpu; -preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_note_context_switch ();
@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)raw_spin_unlock_irq (&rq->lock);post_schedule (rq); - -sched_preempt_enable_no_resched (); }static inline void sched_submit_work (struct task_struct *tsk)
@@ -2820,7 +2815,9 @@ asmlinkage __visible void __sched schedule(void)sched_submit_work (tsk);do { +preempt_disable ();__schedule (); +sched_preempt_enable_no_resched (); }while (need_resched ()); }EXPORT_SYMBOL (schedule);
Signature · 1 line
2.1.4
Peter Zijlstra
maintainerRe: [PATCH 5/7] sched: Pull preemption disablement duty up to __schedule() callers
Quoted 21 lines · Frederic Weisbecker
+++
b/kernel/sched/core.ckernel/sched/core.c +1
-4@@ -2723,9 +2723,7 @@ again:* - return from syscall or exception to user-space * - return from interrupt-handler to user-space * -* WARNING: all callers must re-check need_resched() afterward and reschedule -* accordingly in case an event triggered the need for rescheduling (such as -* an interrupt waking up a task) while preemption was disabled in __schedule(). +* WARNING: must be called with preemption disabled! */ static void __sched __schedule(void) {@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)struct rq *rq;int cpu; -preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_note_context_switch ();
kernel/sched/core.c static void __sched __schedule(void) · line 2798
Quoted 6 lines
kernel/sched/core.c +0
-2@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)raw_spin_unlock_irq (&rq->lock);post_schedule (rq); - -sched_preempt_enable_no_resched (); }
Ah, see, you need to fold this patch into the previous one.
Re: [PATCH 5/7] sched: Pull preemption disablement duty up to __schedule() callers
Quoted 24 lines · Peter Zijlstra
Quoted 23 lines · Frederic Weisbecker
+++
b/kernel/sched/core.ckernel/sched/core.c +1
-4@@ -2723,9 +2723,7 @@ again:* - return from syscall or exception to user-space * - return from interrupt-handler to user-space * -* WARNING: all callers must re-check need_resched() afterward and reschedule -* accordingly in case an event triggered the need for rescheduling (such as -* an interrupt waking up a task) while preemption was disabled in __schedule(). +* WARNING: must be called with preemption disabled! */ static void __sched __schedule(void) {@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)struct rq *rq;int cpu; -preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_note_context_switch ();@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)raw_spin_unlock_irq (&rq->lock);
Quoted 6 lines
kernel/sched/core.c static void __sched __schedule(void) · line 2798
Quoted 4 lines
kernel/sched/core.c +0
-2post_schedule (rq); - -sched_preempt_enable_no_resched (); }Ah, see, you need to fold this patch into the previous one.
Yeah I hesitated. I'm going to fold then.
Thanks.
PATCH 6/7 preempt: Fix out of date comment
In reply to Frederic Weisbecker--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -116,7 +116,7 @@/* * Check whether we were atomic before we did preempt_disable(): -* (used by the scheduler, *after* releasing the kernel lock) +* (used by the scheduler) */ #define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
Signature · 1 line
2.1.4
PATCH 7/7 preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
In reply to Frederic WeisbeckerNow that PREEMPT_ACTIVE implies PREEMPT_DISABLE_OFFSET, ignoring PREEMPT_ACTIVE from in_atomic() check isn't useful anymore.
--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -112,7 +112,7 @@* used in the general case to determine whether sleeping is possible. * Do not use in_atomic() in driver code. */ -#define in_atomic () ((preempt_count() & ~PREEMPT_ACTIVE) != 0) +#define in_atomic () (preempt_count() != 0)/* * Check whether we were atomic before we did preempt_disable():
Signature · 1 line
2.1.4