preempt: A few headers cleanups and preempt_schedule*() optimizations v2
Subsystems (3)
cover preempt: A few headers cleanups and preempt_schedule*() optimizations v2
Hi,
Changes from v1:
* Merge the two commits that optimize preempt count operations on __schedule() caller as per Peterz suggestion.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/preempt-v2
HEAD: d87df756db90d364b8497e01aaa636fffda04a7d
Thanks,
Frederic
---
Frederic Weisbecker (6):
preempt: Merge preempt_mask.h into preempt.h
preempt: Rearrange a few symbols after headers merge
preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
sched: Optimize preemption operations on __schedule() callers
preempt: Fix out of date comment
preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
arch/m68k/include/asm/irqflags.h | 3 - include/linux/bottom_half.h | 1 - include/linux/hardirq.h | 2 +- include/linux/preempt.h | 123 ++++++++++++++++++++++++++++++++++++++- include/linux/preempt_mask.h | 117 ------------------------------------- include/linux/sched.h | 2 +- kernel/sched/core.c | 29 +++------ lib/radix-tree.c | 2 +- 8 files changed, 133 insertions(+), 146 deletions(-)
PATCH 1/6 preempt: Merge preempt_mask.h into preempt.h
preempt_mask.h defines all the preempt_count semantics and related
symbols: preempt, softirq, hardirq, nmi, preempt active, need resched, etc...
preempt.h defines the accessors and mutators of preempt_count.
But there is a messy dependency game around those two header files:
* preempt_mask.h includes preempt.h in order to access preempt_count()
* preempt_mask.h defines all preempt_count semantic and symbols except PREEMPT_NEED_RESCHED that is needed by asm/preempt.h Thus we need to define it from preempt.h, right before including asm/preempt.h, instead of defining it to preempt_mask.h with the other preempt_count symbols. Therefore the preempt_count semantics happen to be spread out.
* We plan to introduce preempt_active_[enter,exit]() to consolidate preempt_schedule*() code. But we'll need to access both preempt_count mutators (preempt_count_add()) and preempt_count symbols (PREEMPT_ACTIVE, PREEMPT_OFFSET). The usual place to define preempt operations is in preempt.h but then we'll need symbols in preempt_mask.h which already includes preempt.h. So we end up with a ressource circle dependency.
Lets merge preempt_mask.h into preempt.h to solve these dependency issues. This way we gather semantic symbols and operation definition of preempt_count in a single file.
This is a dumb copy-paste merge. Further merge re-arrangments are performed in a subsequent patch to ease review.
--- arch/m68k/include/asm/irqflags.h | 3 - include/linux/bottom_half.h | 1 - include/linux/hardirq.h | 2 +- include/linux/preempt.h | 111 +++++++++++++++++++++++++++++++++++++ include/linux/preempt_mask.h | 117 --------------------------------------- include/linux/sched.h | 2 +- lib/radix-tree.c | 2 +- 7 files changed, 114 insertions(+), 124 deletions(-) delete mode 100644 include/linux/preempt_mask.h
arch/m68k/include/asm/irqflags.h +0 -3
@@ -2,9 +2,6 @@#define _M68K_IRQFLAGS_H#include <linux/types.h> -#ifdef CONFIG_MMU -#include <linux/preempt_mask.h> -#endif #include <linux/preempt.h> #include <asm/thread_info.h> #include <asm/entry.h>
include/linux/bottom_half.h +0 -1
@@ -2,7 +2,6 @@#define _LINUX_BH_H#include <linux/preempt.h> -#include <linux/preempt_mask.h> #ifdef CONFIG_TRACE_IRQFLAGS extern void __local_bh_disable_ip (unsigned long ip,unsigned int cnt);
include/linux/hardirq.h +1 -1
@@ -1,7 +1,7 @@#ifndef LINUX_HARDIRQ_H #define LINUX_HARDIRQ_H -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <linux/lockdep.h> #include <linux/ftrace_irq.h> #include <linux/vtime.h>
include/linux/preempt.h +111 -0
117 lines not on this page · raw message
include/linux/preempt_mask.h deleted +0 -117
117 lines not on this page · raw message
include/linux/sched.h +1 -1
@@ -25,7 +25,7 @@ struct sched_param {#include <linux/errno.h> #include <linux/nodemask.h> #include <linux/mm_types.h> -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <asm/page.h> #include <asm/ptrace.h>
lib/radix-tree.c +1 -1
@@ -33,7 +33,7 @@#include <linux/string.h> #include <linux/bitops.h> #include <linux/rcupdate.h> -#include <linux/preempt_mask.h> /* in_interrupt() */ +#include <linux/preempt.h> /* in_interrupt() */ /*
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Merge preempt_mask.h into preempt.h
sched/preempt: Merge preempt_mask.h into preempt.h
Commit-ID: 92cf211874e954027b8e91cc9a15485a50b58d6b Gitweb: http://git.kernel.org/tip/92cf211874e954027b8e91cc9a15485a50b58d6b Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:46 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:11 +0200
sched/preempt: Merge preempt_mask.h into preempt.h
preempt_mask.h defines all the preempt_count semantics and related
symbols: preempt, softirq, hardirq, nmi, preempt active, need resched, etc...
preempt.h defines the accessors and mutators of preempt_count.
But there is a messy dependency game around those two header files:
* preempt_mask.h includes preempt.h in order to access preempt_count()
* preempt_mask.h defines all preempt_count semantic and symbols except PREEMPT_NEED_RESCHED that is needed by asm/preempt.h Thus we need to define it from preempt.h, right before including asm/preempt.h, instead of defining it to preempt_mask.h with the other preempt_count symbols. Therefore the preempt_count semantics happen to be spread out.
* We plan to introduce preempt_active_[enter,exit]() to consolidate preempt_schedule*() code. But we'll need to access both preempt_count mutators (preempt_count_add()) and preempt_count symbols (PREEMPT_ACTIVE, PREEMPT_OFFSET). The usual place to define preempt operations is in preempt.h but then we'll need symbols in preempt_mask.h which already includes preempt.h. So we end up with a ressource circle dependency.
Lets merge preempt_mask.h into preempt.h to solve these dependency issues. This way we gather semantic symbols and operation definition of preempt_count in a single file.
This is a dumb copy-paste merge. Further merge re-arrangments are performed in a subsequent patch to ease review.
--- arch/m68k/include/asm/irqflags.h | 3 - include/linux/bottom_half.h | 1 - include/linux/hardirq.h | 2 +- include/linux/preempt.h | 111 +++++++++++++++++++++++++++++++++++++ include/linux/preempt_mask.h | 117 --------------------------------------- include/linux/sched.h | 2 +- lib/radix-tree.c | 2 +- 7 files changed, 114 insertions(+), 124 deletions(-)
arch/m68k/include/asm/irqflags.h +0 -3
@@ -2,9 +2,6 @@#define _M68K_IRQFLAGS_H#include <linux/types.h> -#ifdef CONFIG_MMU -#include <linux/preempt_mask.h> -#endif #include <linux/preempt.h> #include <asm/thread_info.h> #include <asm/entry.h>
include/linux/bottom_half.h +0 -1
@@ -2,7 +2,6 @@#define _LINUX_BH_H#include <linux/preempt.h> -#include <linux/preempt_mask.h> #ifdef CONFIG_TRACE_IRQFLAGS extern void __local_bh_disable_ip (unsigned long ip,unsigned int cnt);
include/linux/hardirq.h +1 -1
@@ -1,7 +1,7 @@#ifndef LINUX_HARDIRQ_H #define LINUX_HARDIRQ_H -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <linux/lockdep.h> #include <linux/ftrace_irq.h> #include <linux/vtime.h>
include/linux/preempt.h +111 -0
117 lines not on this page · raw message
include/linux/preempt_mask.h deleted +0 -117
117 lines not on this page · raw message
include/linux/sched.h +1 -1
@@ -25,7 +25,7 @@ struct sched_param {#include <linux/errno.h> #include <linux/nodemask.h> #include <linux/mm_types.h> -#include <linux/preempt_mask.h> +#include <linux/preempt.h> #include <asm/page.h> #include <asm/ptrace.h>
lib/radix-tree.c +1 -1
@@ -33,7 +33,7 @@#include <linux/string.h> #include <linux/bitops.h> #include <linux/rcupdate.h> -#include <linux/preempt_mask.h> /* in_interrupt() */ +#include <linux/preempt.h> /* in_interrupt() */ /*
PATCH 2/6 preempt: Rearrange a few symbols after headers merge
In reply to Frederic WeisbeckerAdjust a few comments, and further integrate a few definitions after the dumb headers copy.
--- include/linux/preempt.h | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-)
include/linux/preempt.h +15 -19
@@ -10,14 +10,6 @@#include <linux/list.h> /* -* We use the MSB mostly because its available; see <linux/preempt_mask.h> for -* the other bits -- can't include that header due to inclusion hell. -*/ -#define PREEMPT_NEED_RESCHED 0x80000000 - -#include <asm/preempt.h> - -/* * We put the hardirq and softirq counter into the preemption * counter. The bitmask has the following meaning: *
@@ -30,11 +22,12 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 -* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8
@@ -64,6 +57,12 @@#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) +/* We use the MSB mostly because its available */ +#define PREEMPT_NEED_RESCHED 0x80000000 + +/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */ +#include <asm/preempt.h> +#define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)#define irq_count () (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
@@ -122,12 +121,6 @@#define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) -#ifdef CONFIG_PREEMPT_COUNT -# define preemptible () (preempt_count() == 0 && !irqs_disabled()) -#else -# define preemptible () 0 -#endif -#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);extern void preempt_count_sub (int val);
@@ -160,6 +153,8 @@ do { \#define preempt_enable_no_resched () sched_preempt_enable_no_resched() +#define preemptible () (preempt_count() == 0 && !irqs_disabled()) +#ifdef CONFIG_PREEMPT #define preempt_enable () \ do { \
@@ -232,6 +227,7 @@ do { \#define preempt_disable_notrace () barrier()#define preempt_enable_no_resched_notrace () barrier()#define preempt_enable_notrace () barrier() +#define preemptible () 0#endif /* CONFIG_PREEMPT_COUNT */
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Rearrange a few symbols after headers merge
sched/preempt: Rearrange a few symbols after headers merge
Commit-ID: 2e10e71ce88e3eaccfd09a045ae6ecebe657ba09 Gitweb: http://git.kernel.org/tip/2e10e71ce88e3eaccfd09a045ae6ecebe657ba09 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:47 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200
sched/preempt: Rearrange a few symbols after headers merge
Adjust a few comments, and further integrate a few definitions after the dumb headers copy.
--- include/linux/preempt.h | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-)
include/linux/preempt.h +15 -19
@@ -10,14 +10,6 @@#include <linux/list.h> /* -* We use the MSB mostly because its available; see <linux/preempt_mask.h> for -* the other bits -- can't include that header due to inclusion hell. -*/ -#define PREEMPT_NEED_RESCHED 0x80000000 - -#include <asm/preempt.h> - -/* * We put the hardirq and softirq counter into the preemption * counter. The bitmask has the following meaning: *
@@ -30,11 +22,12 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 -* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 +* PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8
@@ -64,6 +57,12 @@#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) +/* We use the MSB mostly because its available */ +#define PREEMPT_NEED_RESCHED 0x80000000 + +/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */ +#include <asm/preempt.h> +#define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)#define irq_count () (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
@@ -122,12 +121,6 @@#define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) -#ifdef CONFIG_PREEMPT_COUNT -# define preemptible () (preempt_count() == 0 && !irqs_disabled()) -#else -# define preemptible () 0 -#endif -#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);extern void preempt_count_sub (int val);
@@ -160,6 +153,8 @@ do { \#define preempt_enable_no_resched () sched_preempt_enable_no_resched() +#define preemptible () (preempt_count() == 0 && !irqs_disabled()) +#ifdef CONFIG_PREEMPT #define preempt_enable () \ do { \
@@ -232,6 +227,7 @@ do { \#define preempt_disable_notrace () barrier()#define preempt_enable_no_resched_notrace () barrier()#define preempt_enable_notrace () barrier() +#define preemptible () 0#endif /* CONFIG_PREEMPT_COUNT */
PATCH 3/6 preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
In reply to Frederic Weisbecker"CHECK" suggests it's only used as a comparison mask. But now it's used further as a config-conditional preempt disabler offset. Lets disambiguate this name.
--- include/linux/preempt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
include/linux/preempt.h +4 -4
@@ -85,9 +85,9 @@#define in_nmi () (preempt_count() & NMI_MASK)#if defined(CONFIG_PREEMPT_COUNT ) -# define PREEMPT_CHECK_OFFSET 1 +# define PREEMPT_DISABLE_OFFSET 1#else -# define PREEMPT_CHECK_OFFSET 0 +# define PREEMPT_DISABLE_OFFSET 0#endif /*
@@ -103,7 +103,7 @@* * Work as expected. */ -#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_CHECK_OFFSET) +#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_DISABLE_OFFSET)/* * Are we running in atomic context? WARNING: this macro cannot
@@ -119,7 +119,7 @@* (used by the scheduler, *after* releasing the kernel lock) */ #define in_atomic_preempt_off () \ - ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) + ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
sched/preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
Commit-ID: 90b62b5129d5cb50f62f40e684de7a1961e57197 Gitweb: http://git.kernel.org/tip/90b62b5129d5cb50f62f40e684de7a1961e57197 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:48 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200
sched/preempt: Rename PREEMPT_CHECK_OFFSET to PREEMPT_DISABLE_OFFSET
"CHECK" suggests it's only used as a comparison mask. But now it's used further as a config-conditional preempt disabler offset. Lets disambiguate this name.
--- include/linux/preempt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
include/linux/preempt.h +4 -4
@@ -85,9 +85,9 @@#define in_nmi () (preempt_count() & NMI_MASK)#if defined(CONFIG_PREEMPT_COUNT ) -# define PREEMPT_CHECK_OFFSET 1 +# define PREEMPT_DISABLE_OFFSET 1#else -# define PREEMPT_CHECK_OFFSET 0 +# define PREEMPT_DISABLE_OFFSET 0#endif /*
@@ -103,7 +103,7 @@* * Work as expected. */ -#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_CHECK_OFFSET) +#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_DISABLE_OFFSET)/* * Are we running in atomic context? WARNING: this macro cannot
@@ -119,7 +119,7 @@* (used by the scheduler, *after* releasing the kernel lock) */ #define in_atomic_preempt_off () \ - ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) + ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void preempt_count_add (int val);
PATCH 4/6 sched: Optimize preemption operations on __schedule() callers
In reply to Frederic Weisbecker__schedule() disables preemption and some of its callers (the preempt_schedule*() family) also set PREEMPT_ACTIVE.
So we have two preempt_count() modifications that could be performed at once.
Lets remove the preemption disablement from __schedule() and pull this responsibility to its callers in order to optimize preempt_count() operations in a single place.
--- include/linux/preempt.h | 12 ++++++++++++ kernel/sched/core.c | 29 +++++++++-------------------- 2 files changed, 21 insertions(+), 20 deletions(-)
include/linux/preempt.h +12 -0
@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);#define preempt_count_inc () preempt_count_add(1)#define preempt_count_dec () preempt_count_sub(1) +#define preempt_active_enter () \ +do { \ + preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + barrier(); \ +} while (0) + +#define preempt_active_exit () \ +do { \ + barrier(); \ + preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ +} while (0) +#ifdef CONFIG_PREEMPT_COUNT #define preempt_disable () \
kernel/sched/core.c +9 -20
@@ -2723,9 +2723,7 @@ again:* - return from syscall or exception to user-space * - return from interrupt-handler to user-space * -* WARNING: all callers must re-check need_resched() afterward and reschedule -* accordingly in case an event triggered the need for rescheduling (such as -* an interrupt waking up a task) while preemption was disabled in __schedule(). +* WARNING: must be called with preemption disabled! */ static void __sched __schedule(void) {
@@ -2734,7 +2732,6 @@ static void __sched __schedule(void)struct rq *rq;int cpu; -preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_note_context_switch ();
@@ -2798,8 +2795,6 @@ static void __sched __schedule(void)raw_spin_unlock_irq (&rq->lock);post_schedule (rq); - -sched_preempt_enable_no_resched (); }static inline void sched_submit_work (struct task_struct *tsk)
@@ -2820,7 +2815,9 @@ asmlinkage __visible void __sched schedule(void)sched_submit_work (tsk);do { +preempt_disable ();__schedule (); +sched_preempt_enable_no_resched (); }while (need_resched ()); }EXPORT_SYMBOL (schedule);
@@ -2859,15 +2856,14 @@ void __sched schedule_preempt_disabled(void) static void __sched notrace preempt_schedule_common(void) {do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); +preempt_active_exit ();/* * Check again in case we missed a preemption opportunity * between schedule and now. */ -barrier (); }while (need_resched ()); }
@@ -2914,7 +2910,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)return ; do { - __preempt_count_add(PREEMPT_ACTIVE); + preempt_active_enter(); /* * Needs preempt disabled in case user_exit() is traced * and the tracer calls preempt_enable_notrace() causing
@@ -2924,8 +2920,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)__schedule ();exception_exit (prev_ctx); -__preempt_count_sub (PREEMPT_ACTIVE ); -barrier (); +preempt_active_exit (); }while (need_resched ()); }EXPORT_SYMBOL_GPL (preempt_schedule_context);
@@ -2949,17 +2944,11 @@ asmlinkage __visible void __sched preempt_schedule_irq(void) prev_state =exception_enter ();do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();local_irq_enable ();__schedule ();local_irq_disable (); -__preempt_count_sub (PREEMPT_ACTIVE ); - -/* -* Check again in case we missed a preemption opportunity -* between schedule and now. -*/ -barrier (); +preempt_active_exit (); }while (need_resched ());exception_exit (prev_state);
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Optimize preemption operations on __schedule() callers
sched/preempt: Optimize preemption operations on __schedule() callers
Commit-ID: b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Gitweb: http://git.kernel.org/tip/b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:49 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200
sched/preempt: Optimize preemption operations on __schedule() callers
__schedule() disables preemption and some of its callers (the preempt_schedule*() family) also set PREEMPT_ACTIVE.
So we have two preempt_count() modifications that could be performed at once.
Lets remove the preemption disablement from __schedule() and pull this responsibility to its callers in order to optimize preempt_count() operations in a single place.
--- include/linux/preempt.h | 12 ++++++++++++ kernel/sched/core.c | 29 +++++++++-------------------- 2 files changed, 21 insertions(+), 20 deletions(-)
include/linux/preempt.h +12 -0
@@ -137,6 +137,18 @@ extern void preempt_count_sub(int val);#define preempt_count_inc () preempt_count_add(1)#define preempt_count_dec () preempt_count_sub(1) +#define preempt_active_enter () \ +do { \ + preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + barrier(); \ +} while (0) + +#define preempt_active_exit () \ +do { \ + barrier(); \ + preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ +} while (0) +#ifdef CONFIG_PREEMPT_COUNT #define preempt_disable () \
kernel/sched/core.c +9 -20
@@ -2773,9 +2773,7 @@ again:* - return from syscall or exception to user-space * - return from interrupt-handler to user-space * -* WARNING: all callers must re-check need_resched() afterward and reschedule -* accordingly in case an event triggered the need for rescheduling (such as -* an interrupt waking up a task) while preemption was disabled in __schedule(). +* WARNING: must be called with preemption disabled! */ static void __sched __schedule(void) {
@@ -2784,7 +2782,6 @@ static void __sched __schedule(void)struct rq *rq;int cpu; -preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_note_context_switch ();
@@ -2848,8 +2845,6 @@ static void __sched __schedule(void)raw_spin_unlock_irq (&rq->lock);post_schedule (rq); - -sched_preempt_enable_no_resched (); }static inline void sched_submit_work (struct task_struct *tsk)
@@ -2870,7 +2865,9 @@ asmlinkage __visible void __sched schedule(void)sched_submit_work (tsk);do { +preempt_disable ();__schedule (); +sched_preempt_enable_no_resched (); }while (need_resched ()); }EXPORT_SYMBOL (schedule);
@@ -2909,15 +2906,14 @@ void __sched schedule_preempt_disabled(void) static void __sched notrace preempt_schedule_common(void) {do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();__schedule (); -__preempt_count_sub (PREEMPT_ACTIVE ); +preempt_active_exit ();/* * Check again in case we missed a preemption opportunity * between schedule and now. */ -barrier (); }while (need_resched ()); }
@@ -2964,7 +2960,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)return ; do { - __preempt_count_add(PREEMPT_ACTIVE); + preempt_active_enter(); /* * Needs preempt disabled in case user_exit() is traced * and the tracer calls preempt_enable_notrace() causing
@@ -2974,8 +2970,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_context(void)__schedule ();exception_exit (prev_ctx); -__preempt_count_sub (PREEMPT_ACTIVE ); -barrier (); +preempt_active_exit (); }while (need_resched ()); }EXPORT_SYMBOL_GPL (preempt_schedule_context);
@@ -2999,17 +2994,11 @@ asmlinkage __visible void __sched preempt_schedule_irq(void) prev_state =exception_enter ();do { -__preempt_count_add (PREEMPT_ACTIVE ); +preempt_active_enter ();local_irq_enable ();__schedule ();local_irq_disable (); -__preempt_count_sub (PREEMPT_ACTIVE ); - -/* -* Check again in case we missed a preemption opportunity -* between schedule and now. -*/ -barrier (); +preempt_active_exit (); }while (need_resched ());exception_exit (prev_state);
Re: [tip:sched/core] sched/preempt: Optimize preemption operations on __schedule() callers
Quoted 19 lines · tip-bot for Frederic Weisbecker
Commit-ID: b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Gitweb: http://git.kernel.org/tip/b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:49 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200sched/preempt: Optimize preemption operations on
__schedule()callers
__schedule()disables preemption and some of its callers (the preempt_schedule*() family) also setPREEMPT_ACTIVE.So we have two
preempt_count()modifications that could be performed at once.Lets remove the preemption disablement from
__schedule()and pull this responsibility to its callers in order to optimizepreempt_count()operations in a single place.
Quoted 7 lines
Suggested-by Linus Torvalds <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Ingo Molnar <[email protected]>
Hi all,
I was seeing RCU stall warnings that appeared to have different backtrace each time, but would reliably trigger when fuzzing and looked something like this:
[4394886.414687] INFO: rcu_preempt detected stalls on CPUs/tasks: [4394886.416717] Tasks blocked on level-0 rcu_node (CPUs 0-23): P13356 [4394886.418699] (detected by 2, t=20502 jiffies, g=3548, c=3547, q=120) [4394886.420712] trinity-c42 R running task 26936 13356 9574 0x10000000 [4394886.422466] ffff8807d1c7fbe8 ffff8807d1c7fb88 0000000000000022 ffff8801081e1bb0 [4394886.423711] ffff8801081e1b88 ffff8807d1eb8780 ffff8801081e11d8 ffff8807e2564000 [4394886.424956] ffff8807d1eb8000 ffff8807d1c7fbd8 ffff8807d1c78000 0000000000000000 [4394886.426196] Call Trace: [4394886.426627] preempt_schedule_irq (./arch/x86/include/asm/paravirt.h:807 kernel/sched/core.c:3218) [4394886.427582] ? lockdep_reset_lock (kernel/locking/lockdep.c:3105) [4394886.428548] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.429438] retint_kernel (arch/x86/entry/entry_64.S:578) [4394886.430286] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.431176] ? native_restore_fl (./arch/x86/include/asm/irqflags.h:35) [4394886.432376] lock_is_held (kernel/locking/lockdep.c:3661) [4394886.433241] ? kill_pid_info (include/linux/rcupdate.h:914 kernel/signal.c:1344) [4394886.434135] rcu_read_lock_held (kernel/rcu/update.c:275) [4394886.435012] pid_task (kernel/pid.c:440 (discriminator 5)) [4394886.435656] kill_pid_info (kernel/signal.c:1341) [4394886.436351] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.437065] SYSC_kill (kernel/signal.c:1426 kernel/signal.c:2903) [4394886.437726] ? SYSC_kill (include/linux/rcupdate.h:857 kernel/signal.c:1425 kernel/signal.c:2903) [4394886.438419] ? find_get_pid (include/linux/rcupdate.h:914 kernel/pid.c:494) [4394886.439134] ? kill_pid (kernel/signal.c:2894) [4394886.439782] ? find_get_pid (kernel/pid.c:497) [4394886.440492] ? find_get_pid (kernel/pid.c:489) [4394886.441183] ? lock_is_held (kernel/locking/lockdep.c:3661) [4394886.441929] ? rcu_read_lock_sched_held (kernel/rcu/update.c:109) [4394886.442772] ? syscall_trace_enter_phase2 (arch/x86/entry/common.c:196) [4394886.443632] SyS_kill (kernel/signal.c:2893) [4394886.444255] tracesys_phase2 (arch/x86/entry/entry_64.S:270)
I worked with Paul to rule out RCU as the cause.
I've noticed that all traces had one thing in common: being stuck in preempt_schedule_irq(), so I've looked at recent changes there and noticed this commit.
I've tried testing the commit before that, and the problem went away. Checking out this commit the problem reappeared.
Thanks,
Sasha
Re: [tip:sched/core] sched/preempt: Optimize preemption operations on __schedule() callers
Quoted 65 lines · Sasha Levin
Quoted 19 lines · tip-bot for Frederic Weisbecker
Commit-ID: b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Gitweb: http://git.kernel.org/tip/b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:49 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200sched/preempt: Optimize preemption operations on
__schedule()callers
__schedule()disables preemption and some of its callers (the preempt_schedule*() family) also setPREEMPT_ACTIVE.So we have two
preempt_count()modifications that could be performed at once.Lets remove the preemption disablement from
__schedule()and pull this responsibility to its callers in order to optimizepreempt_count()operations in a single place.Quoted 7 lines
Suggested-by Linus Torvalds <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Ingo Molnar <[email protected]>Hi all,
I was seeing RCU stall warnings that appeared to have different backtrace each time, but would reliably trigger when fuzzing and looked something like this:
[4394886.414687] INFO: rcu_preempt detected stalls on CPUs/tasks: [4394886.416717] Tasks blocked on level-0 rcu_node (CPUs 0-23): P13356 [4394886.418699] (detected by 2, t=20502 jiffies, g=3548, c=3547, q=120) [4394886.420712] trinity-c42 R running task 26936 13356 9574 0x10000000 [4394886.422466] ffff8807d1c7fbe8 ffff8807d1c7fb88 0000000000000022 ffff8801081e1bb0 [4394886.423711] ffff8801081e1b88 ffff8807d1eb8780 ffff8801081e11d8 ffff8807e2564000 [4394886.424956] ffff8807d1eb8000 ffff8807d1c7fbd8 ffff8807d1c78000 0000000000000000 [4394886.426196] Call Trace: [4394886.426627] preempt_schedule_irq (./arch/x86/include/asm/paravirt.h:807 kernel/sched/core.c:3218) [4394886.427582] ? lockdep_reset_lock (kernel/locking/lockdep.c:3105) [4394886.428548] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.429438] retint_kernel (arch/x86/entry/entry_64.S:578) [4394886.430286] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.431176] ? native_restore_fl (./arch/x86/include/asm/irqflags.h:35) [4394886.432376] lock_is_held (kernel/locking/lockdep.c:3661) [4394886.433241] ? kill_pid_info (include/linux/rcupdate.h:914 kernel/signal.c:1344) [4394886.434135] rcu_read_lock_held (kernel/rcu/update.c:275) [4394886.435012] pid_task (kernel/pid.c:440 (discriminator 5)) [4394886.435656] kill_pid_info (kernel/signal.c:1341) [4394886.436351] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.437065] SYSC_kill (kernel/signal.c:1426 kernel/signal.c:2903) [4394886.437726] ? SYSC_kill (include/linux/rcupdate.h:857 kernel/signal.c:1425 kernel/signal.c:2903) [4394886.438419] ? find_get_pid (include/linux/rcupdate.h:914 kernel/pid.c:494) [4394886.439134] ? kill_pid (kernel/signal.c:2894) [4394886.439782] ? find_get_pid (kernel/pid.c:497) [4394886.440492] ? find_get_pid (kernel/pid.c:489) [4394886.441183] ? lock_is_held (kernel/locking/lockdep.c:3661) [4394886.441929] ? rcu_read_lock_sched_held (kernel/rcu/update.c:109) [4394886.442772] ? syscall_trace_enter_phase2 (arch/x86/entry/common.c:196) [4394886.443632] SyS_kill (kernel/signal.c:2893) [4394886.444255] tracesys_phase2 (arch/x86/entry/entry_64.S:270)
Quoted 7 lines
I worked with Paul to rule out RCU as the cause.
I've noticed that all traces had one thing in common: being stuck in
preempt_schedule_irq(), so I've looked at recent changes there and noticed this commit.I've tried testing the commit before that, and the problem went away. Checking out this commit the problem reappeared.
Do you have any idea how to reproduce that?
Thanks.
Re: [tip:sched/core] sched/preempt: Optimize preemption operations on __schedule() callers
Quoted 66 lines · Frederic Weisbecker
Quoted 65 lines · Sasha Levin
Quoted 65 lines · tip-bot for Frederic Weisbecker
Quoted 19 lines
Quoted 19 lines
Commit-ID: b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Gitweb: http://git.kernel.org/tip/b30f0e3ffedfa52b1d67a302ae5860c49998e5e2 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:49 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:12 +0200sched/preempt: Optimize preemption operations on
__schedule()callers
__schedule()disables preemption and some of its callers (the preempt_schedule*() family) also setPREEMPT_ACTIVE.So we have two
preempt_count()modifications that could be performed at once.Lets remove the preemption disablement from
__schedule()and pull this responsibility to its callers in order to optimizepreempt_count()operations in a single place.Quoted 7 lines
Quoted 7 lines
Suggested-by Linus Torvalds <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Ingo Molnar <[email protected]>Hi all,
I was seeing RCU stall warnings that appeared to have different backtrace each time, but would reliably trigger when fuzzing and looked something like this:
[4394886.414687] INFO: rcu_preempt detected stalls on CPUs/tasks: [4394886.416717] Tasks blocked on level-0 rcu_node (CPUs 0-23): P13356 [4394886.418699] (detected by 2, t=20502 jiffies, g=3548, c=3547, q=120) [4394886.420712] trinity-c42 R running task 26936 13356 9574 0x10000000 [4394886.422466] ffff8807d1c7fbe8 ffff8807d1c7fb88 0000000000000022 ffff8801081e1bb0 [4394886.423711] ffff8801081e1b88 ffff8807d1eb8780 ffff8801081e11d8 ffff8807e2564000 [4394886.424956] ffff8807d1eb8000 ffff8807d1c7fbd8 ffff8807d1c78000 0000000000000000 [4394886.426196] Call Trace: [4394886.426627] preempt_schedule_irq (./arch/x86/include/asm/paravirt.h:807 kernel/sched/core.c:3218) [4394886.427582] ? lockdep_reset_lock (kernel/locking/lockdep.c:3105) [4394886.428548] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.429438] retint_kernel (arch/x86/entry/entry_64.S:578) [4394886.430286] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.431176] ? native_restore_fl (./arch/x86/include/asm/irqflags.h:35) [4394886.432376] lock_is_held (kernel/locking/lockdep.c:3661) [4394886.433241] ? kill_pid_info (include/linux/rcupdate.h:914 kernel/signal.c:1344) [4394886.434135] rcu_read_lock_held (kernel/rcu/update.c:275) [4394886.435012] pid_task (kernel/pid.c:440 (discriminator 5)) [4394886.435656] kill_pid_info (kernel/signal.c:1341) [4394886.436351] ? kill_pid_info (include/linux/rcupdate.h:857 kernel/signal.c:1340) [4394886.437065] SYSC_kill (kernel/signal.c:1426 kernel/signal.c:2903) [4394886.437726] ? SYSC_kill (include/linux/rcupdate.h:857 kernel/signal.c:1425 kernel/signal.c:2903) [4394886.438419] ? find_get_pid (include/linux/rcupdate.h:914 kernel/pid.c:494) [4394886.439134] ? kill_pid (kernel/signal.c:2894) [4394886.439782] ? find_get_pid (kernel/pid.c:497) [4394886.440492] ? find_get_pid (kernel/pid.c:489) [4394886.441183] ? lock_is_held (kernel/locking/lockdep.c:3661) [4394886.441929] ? rcu_read_lock_sched_held (kernel/rcu/update.c:109) [4394886.442772] ? syscall_trace_enter_phase2 (arch/x86/entry/common.c:196) [4394886.443632] SyS_kill (kernel/signal.c:2893) [4394886.444255] tracesys_phase2 (arch/x86/entry/entry_64.S:270)
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
I worked with Paul to rule out RCU as the cause.
I've noticed that all traces had one thing in common: being stuck in
preempt_schedule_irq(), so I've looked at recent changes there and noticed this commit.I've tried testing the commit before that, and the problem went away. Checking out this commit the problem reappeared.
Do you have any idea how to reproduce that?
Beyond running trinity, no. That's why I couldn't bisect it to begin with.
Thanks,
Sasha
Peter Zijlstra
maintainerRe: [tip:sched/core] sched/preempt: Optimize preemption operations on __schedule() callers
In reply to Sasha LevinQuoted 10 lines · Sasha Levin
Quoted 4 lines · tip-bot for Frederic Weisbecker
Commit-ID:
b30f0e3ffedfa52b1d67a302ae5860c49998e5e2sched/preempt: Optimize preemption operations on
__schedule()callersI was seeing RCU stall warnings that appeared to have different backtrace each time, but would reliably trigger when fuzzing and looked something like this:
[4394886.426627] preempt_schedule_irq (./arch/x86/include/asm/paravirt.h:807 kernel/sched/core.c:3218)
The only 'obvious' difference there seems to be something like the below. Prior to the patch preempt_schedule_irq() used __preempt_count_{add,sub}() which are the !tracing versions.
Does this make it go again? If not, can you provide a .config?
--- include/linux/preempt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/preempt.h +2 -2
@@ -139,14 +139,14 @@ extern void preempt_count_sub(int val);#define preempt_active_enter () \ do { \ - preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + __preempt_count_add(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ barrier(); \ } while (0)#define preempt_active_exit () \ do { \ barrier(); \ - preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ + __preempt_count_sub(PREEMPT_ACTIVE + PREEMPT_DISABLE_OFFSET); \ } while (0)#ifdef CONFIG_PREEMPT_COUNT
PATCH 5/6 preempt: Fix out of date comment
In reply to Frederic Weisbecker--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -116,7 +116,7 @@/* * Check whether we were atomic before we did preempt_disable(): -* (used by the scheduler, *after* releasing the kernel lock) +* (used by the scheduler) */ #define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Fix out of date comment
sched/preempt: Fix out of date comment
Commit-ID: e017cf21ae82e0b36f026b22083a8ae67926f465 Gitweb: http://git.kernel.org/tip/e017cf21ae82e0b36f026b22083a8ae67926f465 Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:50 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:13 +0200
sched/preempt: Fix out of date comment
--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -116,7 +116,7 @@/* * Check whether we were atomic before we did preempt_disable(): -* (used by the scheduler, *after* releasing the kernel lock) +* (used by the scheduler) */ #define in_atomic_preempt_off () \ ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_DISABLE_OFFSET)
PATCH 6/6 preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
In reply to Frederic WeisbeckerNow that PREEMPT_ACTIVE implies PREEMPT_DISABLE_OFFSET, ignoring PREEMPT_ACTIVE from in_atomic() check isn't useful anymore.
--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -112,7 +112,7 @@* used in the general case to determine whether sleeping is possible. * Do not use in_atomic() in driver code. */ -#define in_atomic () ((preempt_count() & ~PREEMPT_ACTIVE) != 0) +#define in_atomic () (preempt_count() != 0)/* * Check whether we were atomic before we did preempt_disable():
Signature · 1 line
2.1.4
tip-bot for Frederic Weisbecker
merge bot[tip:sched/core] sched/preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
sched/preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
Commit-ID: 3e51f3c4004c9b01f66da03214a3e206f5ed627b Gitweb: http://git.kernel.org/tip/3e51f3c4004c9b01f66da03214a3e206f5ed627b Author: Frederic Weisbecker <[email protected]> AuthorDate: Tue, 12 May 2015 16:41:51 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2015 08:39:13 +0200
sched/preempt: Remove PREEMPT_ACTIVE unmasking off in_atomic()
Now that PREEMPT_ACTIVE implies PREEMPT_DISABLE_OFFSET, ignoring PREEMPT_ACTIVE from in_atomic() check isn't useful anymore.
--- include/linux/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/preempt.h +1 -1
@@ -112,7 +112,7 @@* used in the general case to determine whether sleeping is possible. * Do not use in_atomic() in driver code. */ -#define in_atomic () ((preempt_count() & ~PREEMPT_ACTIVE) != 0) +#define in_atomic () (preempt_count() != 0)/* * Check whether we were atomic before we did preempt_disable():