preempt_count rework -v3
Subsystems (26)
Peter Zijlstra
maintainercover preempt_count rework -v3
These patches optimize preempt_enable by firstly folding the preempt and need_resched tests into one -- this should work for all architectures. And secondly by providing per-arch preempt_count implementations; with x86 using per-cpu preempt_count for fastest access.
These patches have been boot tested on CONFIG_PREEMPT=y x86_64 and survive building a x86_64-defconfig kernel.
text data bss filename
11387014 1454776 1187840 defconfig-build/vmlinux.before 11352294 1454776 1187840 defconfig-build/vmlinux.after
void kick_process(struct task_struct *p)
{
int cpu;
preempt_disable();
cpu = task_cpu(p);
if ((cpu != smp_processor_id()) && task_curr(p))
smp_send_reschedule(cpu);
preempt_enable();
}before:
ffffffff81073f00 <kick_process>:
ffffffff81073f00: 55 push %rbp I ffffffff81073f01: 65 48 8b 04 25 f0 b7 mov %gs:0xb7f0,%rax ffffffff81073f08: 00 00 I ffffffff81073f0a: 83 80 44 e0 ff ff 01 addl $0x1,-0x1fbc(%rax) ffffffff81073f11: 48 89 e5 mov %rsp,%rbp ffffffff81073f14: 48 8b 47 08 mov 0x8(%rdi),%rax ffffffff81073f18: 8b 50 18 mov 0x18(%rax),%edx ffffffff81073f1b: 65 8b 04 25 1c b0 00 mov %gs:0xb01c,%eax ffffffff81073f22: 00 ffffffff81073f23: 39 c2 cmp %eax,%edx ffffffff81073f25: 74 1b je ffffffff81073f42 <kick_process+0x42> ffffffff81073f27: 89 d1 mov %edx,%ecx ffffffff81073f29: 48 c7 c0 40 2c 01 00 mov $0x12c40,%rax ffffffff81073f30: 48 8b 0c cd 00 25 cc mov -0x7e33db00(,%rcx,8),%rcx ffffffff81073f37: 81 ffffffff81073f38: 48 3b bc 08 00 08 00 cmp 0x800(%rax,%rcx,1),%rdi ffffffff81073f3f: 00 ffffffff81073f40: 74 2e je ffffffff81073f70 <kick_process+0x70> D ffffffff81073f42: 65 48 8b 04 25 f0 b7 mov %gs:0xb7f0,%rax ffffffff81073f49: 00 00 D ffffffff81073f4b: 83 a8 44 e0 ff ff 01 subl $0x1,-0x1fbc(%rax) D ffffffff81073f52: 48 8b 80 38 e0 ff ff mov -0x1fc8(%rax),%rax D ffffffff81073f59: a8 08 test $0x8,%al D ffffffff81073f5b: 75 02 jne ffffffff81073f5f <kick_process+0x5f> ffffffff81073f5d: 5d pop %rbp ffffffff81073f5e: c3 retq D ffffffff81073f5f: e8 9c 6f 75 00 callq ffffffff817caf00 <preempt_schedule> ffffffff81073f64: 5d pop %rbp ffffffff81073f65: c3 retq ffffffff81073f66: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1) ffffffff81073f6d: 00 00 00 ffffffff81073f70: 89 d7 mov %edx,%edi ffffffff81073f72: ff 15 a8 99 ba 00 callq *0xba99a8(%rip) # ffffffff81c1d920 <smp_ops+0x20> ffffffff81073f78: eb c8 jmp ffffffff81073f42 <kick_process+0x42> ffffffff81073f7a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
after:
ffffffff8106f5e0 <kick_process>:
ffffffff8106f5e0: 55 push %rbp I ffffffff8106f5e1: 65 ff 04 25 e0 b7 00 incl %gs:0xb7e0 ffffffff8106f5e8: 00 ffffffff8106f5e9: 48 89 e5 mov %rsp,%rbp ffffffff8106f5ec: 48 8b 47 08 mov 0x8(%rdi),%rax ffffffff8106f5f0: 8b 50 18 mov 0x18(%rax),%edx ffffffff8106f5f3: 65 8b 04 25 1c b0 00 mov %gs:0xb01c,%eax ffffffff8106f5fa: 00 ffffffff8106f5fb: 39 c2 cmp %eax,%edx ffffffff8106f5fd: 74 1b je ffffffff8106f61a <kick_process+0x3a> ffffffff8106f5ff: 89 d1 mov %edx,%ecx ffffffff8106f601: 48 c7 c0 40 2c 01 00 mov $0x12c40,%rax ffffffff8106f608: 48 8b 0c cd 00 25 cc mov -0x7e33db00(,%rcx,8),%rcx ffffffff8106f60f: 81 ffffffff8106f610: 48 3b bc 08 00 08 00 cmp 0x800(%rax,%rcx,1),%rdi ffffffff8106f617: 00 ffffffff8106f618: 74 26 je ffffffff8106f640 <kick_process+0x60> D ffffffff8106f61a: 65 ff 0c 25 e0 b7 00 decl %gs:0xb7e0 ffffffff8106f621: 00 D ffffffff8106f622: 74 0c je ffffffff8106f630 <kick_process+0x50> ffffffff8106f624: 5d pop %rbp ffffffff8106f625: c3 retq ffffffff8106f626: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1) ffffffff8106f62d: 00 00 00 D ffffffff8106f630: e8 fb b4 f9 ff callq ffffffff8100ab30 <___preempt_schedule> ffffffff8106f635: 5d pop %rbp ffffffff8106f636: c3 retq ffffffff8106f637: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) ffffffff8106f63e: 00 00 ffffffff8106f640: 89 d7 mov %edx,%edi ffffffff8106f642: ff 15 d8 e2 ba 00 callq *0xbae2d8(%rip) # ffffffff81c1d920 <smp_ops+0x20> ffffffff8106f648: eb d0 jmp ffffffff8106f61a <kick_process+0x3a> ffffffff8106f64a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
Peter Zijlstra
maintainerPATCH 1/11 x86: Use asm goto to implement better modify_and_test() functions
Linus suggested using asm goto to get rid of the typical SETcc + TEST instruction pair -- which also clobbers an extra register -- for our typical modify_and_test() functions.
Because asm goto doesn't allow output fields it has to include an unconditinal memory clobber when it changes a memory variable to force a reload.
Luckily all atomic ops already imply a compiler barrier to go along with their memory barrier semantics.
--- arch/x86/include/asm/atomic.h | 29 ++++---------------------- arch/x86/include/asm/atomic64_64.h | 28 +++---------------------- arch/x86/include/asm/bitops.h | 24 +++------------------ arch/x86/include/asm/local.h | 28 +++---------------------- arch/x86/include/asm/rmwcc.h | 41 +++++++++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 92 deletions(-)
arch/x86/include/asm/atomic.h +5 -24
@@ -6,6 +6,7 @@#include <asm/processor.h> #include <asm/alternative.h> #include <asm/cmpxchg.h> +#include <asm/rmwcc.h> /* * Atomic operations that C can't guarantee us. Useful for
@@ -76,12 +77,7 @@ static inline void atomic_sub(int i, ato*/ static inline int atomic_sub_and_test (int i,atomic_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "subl %2,%0; sete %1" - :"+m" (v->counter),"=qm" (c) - :"ir" (i) :"memory" ); -return c; +GEN_BINARY_RMWcc (LOCK_PREFIX "subl" , v->counter, i,"%0" ,"e" ); }/**
@@ -118,12 +114,7 @@ static inline void atomic_dec(atomic_t **/ static inline int atomic_dec_and_test (atomic_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "decl %0; sete %1" - :"+m" (v->counter),"=qm" (c) - : :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (LOCK_PREFIX "decl" , v->counter,"%0" ,"e" ); }/**
@@ -136,12 +127,7 @@ static inline int atomic_dec_and_test(at*/ static inline int atomic_inc_and_test (atomic_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "incl %0; sete %1" - :"+m" (v->counter),"=qm" (c) - : :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (LOCK_PREFIX "incl" , v->counter,"%0" ,"e" ); }/**
@@ -155,12 +141,7 @@ static inline int atomic_inc_and_test(at*/ static inline int atomic_add_negative (int i,atomic_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "addl %2,%0; sets %1" - :"+m" (v->counter),"=qm" (c) - :"ir" (i) :"memory" ); -return c; +GEN_BINARY_RMWcc (LOCK_PREFIX "addl" , v->counter, i,"%0" ,"s" ); }/**
arch/x86/include/asm/atomic64_64.h +4 -24
@@ -72,12 +72,7 @@ static inline void atomic64_sub(long i,*/ static inline int atomic64_sub_and_test (long i,atomic64_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "subq %2,%0; sete %1" - :"=m" (v->counter),"=qm" (c) - :"er" (i),"m" (v->counter) :"memory" ); -return c; +GEN_BINARY_RMWcc (LOCK_PREFIX "subq" , v->counter, i,"%0" ,"e" ); }/**
@@ -116,12 +111,7 @@ static inline void atomic64_dec(atomic64*/ static inline int atomic64_dec_and_test (atomic64_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "decq %0; sete %1" - :"=m" (v->counter),"=qm" (c) - :"m" (v->counter) :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (LOCK_PREFIX "decq" , v->counter,"%0" ,"e" ); }/**
@@ -134,12 +124,7 @@ static inline int atomic64_dec_and_test(*/ static inline int atomic64_inc_and_test (atomic64_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "incq %0; sete %1" - :"=m" (v->counter),"=qm" (c) - :"m" (v->counter) :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (LOCK_PREFIX "incq" , v->counter,"%0" ,"e" ); }/**
@@ -153,12 +138,7 @@ static inline int atomic64_inc_and_test(*/ static inline int atomic64_add_negative (long i,atomic64_t *v) { -unsigned char c; - - asmvolatile (LOCK_PREFIX "addq %2,%0; sets %1" - :"=m" (v->counter),"=qm" (c) - :"er" (i),"m" (v->counter) :"memory" ); -return c; +GEN_BINARY_RMWcc (LOCK_PREFIX "addq" , v->counter, i,"%0" ,"s" ); }/**
arch/x86/include/asm/bitops.h +4 -20
@@ -14,6 +14,7 @@#include <linux/compiler.h> #include <asm/alternative.h> +#include <asm/rmwcc.h> #if BITS_PER_LONG ==32 # define _BITOPS_LONG_SHIFT 5
@@ -204,12 +205,7 @@ static inline void change_bit(long nr, v*/ static inline int test_and_set_bit (long nr,volatile unsigned long *addr) { -int oldbit; - - asmvolatile (LOCK_PREFIX "bts %2,%1\n\t" - "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); - -return oldbit; +GEN_BINARY_RMWcc (LOCK_PREFIX "bts" , *addr, nr,"%0" ,"c" ); }/**
@@ -255,13 +251,7 @@ static inline int __test_and_set_bit(lon*/ static inline int test_and_clear_bit (long nr,volatile unsigned long *addr) { -int oldbit; - - asmvolatile (LOCK_PREFIX "btr %2,%1\n\t" -"sbb %0,%0" - : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); - -return oldbit; +GEN_BINARY_RMWcc (LOCK_PREFIX "btr" , *addr, nr,"%0" ,"c" ); }/**
@@ -314,13 +304,7 @@ static inline int __test_and_change_bit(*/ static inline int test_and_change_bit (long nr,volatile unsigned long *addr) { -int oldbit; - - asmvolatile (LOCK_PREFIX "btc %2,%1\n\t" -"sbb %0,%0" - : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); - -return oldbit; +GEN_BINARY_RMWcc (LOCK_PREFIX "btc" , *addr, nr,"%0" ,"c" ); } static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr)
arch/x86/include/asm/local.h +4 -24
@@ -52,12 +52,7 @@ static inline void local_sub(long i, loc*/ static inline int local_sub_and_test (long i,local_t *l) { -unsigned char c; - - asmvolatile (_ASM_SUB"%2,%0; sete %1" - :"+m" (l->a.counter),"=qm" (c) - :"ir" (i) :"memory" ); -return c; +GEN_BINARY_RMWcc (_ASM_SUB, l->a.counter, i,"%0" ,"e" ); }/**
@@ -70,12 +65,7 @@ static inline int local_sub_and_test(lon*/ static inline int local_dec_and_test (local_t *l) { -unsigned char c; - - asmvolatile (_ASM_DEC"%0; sete %1" - :"+m" (l->a.counter),"=qm" (c) - : :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (_ASM_DEC, l->a.counter,"%0" ,"e" ); }/**
@@ -88,12 +78,7 @@ static inline int local_dec_and_test(loc*/ static inline int local_inc_and_test (local_t *l) { -unsigned char c; - - asmvolatile (_ASM_INC"%0; sete %1" - :"+m" (l->a.counter),"=qm" (c) - : :"memory" ); -return c !=0 ; +GEN_UNARY_RMWcc (_ASM_INC, l->a.counter,"%0" ,"e" ); }/**
@@ -107,12 +92,7 @@ static inline int local_inc_and_test(loc*/ static inline int local_add_negative (long i,local_t *l) { -unsigned char c; - - asmvolatile (_ASM_ADD"%2,%0; sets %1" - :"+m" (l->a.counter),"=qm" (c) - :"ir" (i) :"memory" ); -return c; +GEN_BINARY_RMWcc (_ASM_ADD, l->a.counter, i,"%0" ,"s" ); }/**
arch/x86/include/asm/rmwcc.h new file +41 -0
41 lines not on this page · raw message
Re: [PATCH 01/11] x86: Use asm goto to implement better modify_and_test() functions
Quoted 3 lines · Peter Zijlstra
Linus suggested using asm goto to get rid of the typical SETcc + TEST instruction pair -- which also clobbers an extra register -- for our typical
modify_and_test()functions.
Thinking about this, we actually have another place in x86 low-level code where "asm goto" makes a lot of sense: exception handling for __put_user_asm().
I'd love to use it for __get_user_asm() too, but it doesn't work there because "asm goto" cannot have outputs (and a get_user obviously needs an output - the value it gets). But for put_user(), it seems to be a very good match.
The attached patch is ENTIRELY untested, but I did check some of the generated assembly language. And the output is absolutely beautiful, because now gcc sees the error case directly, so the straight-line code is just he single "mov" instruction, no tests, no nothing. The exception case will just jump to the local label directly.
Of course, the STAC/CLAC noise is there, and we really should try to come up with a better model for that (so that the code that uses __put_user() because it wants to do many of them in one go after having done one access_ok() check) but that's a separate issue.
hpa, comments? Are you looking at perhaps moving the stac/clac instructions out? With this, "filldir()" ends up lookng something like
...
data32 xchg %ax,%ax # stac
mov %rcx,0x8(%rax)
data32 xchg %ax,%ax # clac
mov 0x10(%rbx),%r13
data32 xchg %ax,%ax # stac
mov %r8,0x0(%r13)
data32 xchg %ax,%ax # clac
data32 xchg %ax,%ax # stac
mov %r12w,0x10(%r13)
data32 xchg %ax,%ax # clac
...
which is a bit sad, since the code really is almost perfect aside from the tons of extra nops/stac/clac instructions...
Linus
arch/x86/include/asm/uaccess.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+)
arch/x86/include/asm/uaccess.h +19 -0
@@ -411,6 +411,24 @@ struct __large_struct { unsigned long buf[100]; };* we do not write to any memory gcc knows about, so there are no * aliasing issues. */ +#ifdef CC_HAVE_ASM_GOTO +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ +do { __label__ error_label; \ + asm goto(ASM_STAC "\n" \ + "1: mov"itype" %"rtype"0,%1\n" \ + " " ASM_CLAC "\n" \ + _ASM_EXTABLE(1b,%l[error_label]) \ + : /* no outputs */ \ + : ltype(x), "m" (__m(addr)) \ + : /* no clobbers */ \ + : error_label); \ + err = 0; \ + break; \ +error_label: \ + asm volatile(ASM_CLAC); \ + err = errret; \ +} while (0) +#else #define __put_user_asm (x, addr, err, itype, rtype, ltype, errret) \ asm volatile(ASM_STAC "\n" \ "1: mov"itype" %"rtype"1,%2\n" \
@@ -422,6 +440,7 @@ struct __large_struct { unsigned long buf[100]; }; _ASM_EXTABLE(1b, 3b) \ : "=r"(err) \ : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) +#endif #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ asm volatile("1: mov"itype" %"rtype"0,%1\n" \
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 01/11] x86: Use asm goto to implement better modify_and_test() functions
Quoted 3 lines · H. Peter Anvin
Yes, a bit sad. We allow bracketing with the get/put_user_try/catch blocks, but that is x86-specific.
I don't think a generic option is possible without compiler support, but it might be possible to do better than we do know.
Letting the compiler do it is a bit risky, because it may open it up for really large blocks, thus defeating the security advantages.
-Andi
Re: [PATCH 01/11] x86: Use asm goto to implement better modify_and_test() functions
Quoted 9 lines · Andi Kleen
Quoted 4 lines · H. Peter Anvin
Yes, a bit sad. We allow bracketing with the get/put_user_try/catch blocks, but that is x86-specific. I don't think a generic option is possible without compiler support, but it might be possible to do better than we do know.
Letting the compiler do it is a bit risky, because it may open it up for really large blocks, thus defeating the security advantages.
Yeah, the compiler could cover other pointer dereferences in the put_user block and that won't result in any visible breakage, so it's difficult to prevent the compiler doing it accidentally or even intentionally.
Then again the many repeated STAC/CLAC sequences are really not nice.
So maybe we could add some macro magic to generate better assembly here - if we coded up a __put_user_2field() primitive then we could already optimize the filldir() case:
before:
if (__put_user(d_ino, &dirent->d_ino))
goto efault;
if (__put_user(reclen, &dirent->d_reclen))
goto efault;
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;
if (__put_user(0, dirent->d_name + namlen))
goto efault;
if (__put_user(d_type, (char __user *) dirent + reclen - 1))
goto efault;after:
if (__put_user_2field(d_ino, &dirent->d_ino, reclen, &dirent->d_reclen))
goto efault;
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;
if (__put_user_2field(0, dirent->d_name + namlen, d_type, (char __user *) dirent + reclen - 1)))
goto efault;That cuts down the inlined STAC/CLAC pairs from 4 to 2.
__put_user_2field() would be some truly disgusting (but hidden from most people) macro and assembly magic.
We could also add __put_user_4field() and slightly reorder filldir():
if (__put_user_4field( d_ino, &dirent->d_ino,
reclen, &dirent->d_reclen,
0, dirent->d_name + namlen,
d_type, (char __user *) dirent + reclen - 1)))
goto efault;
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;That would reduce the inlined STAC/CLAC pairs to a minimal 1 (only one of which would be visible in the filldir() disassembly).
In theory we could do something generic:
if (__put_user_fields( 4,
d_ino, &dirent->d_ino,
reclen, &dirent->d_reclen,
0, dirent->d_name + namlen,
d_type, (char __user *)dirent + reclen-1 ))
goto efault;
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;and implement it up to 4 or so. It will be some truly disgusting lowlevel code (especially due to the size variations which could make it explode combinatorically), with some generic header fallback that utilizes existing put_user primitives.
But it's solvable IMO, if we want to solve it. On the high level it's also more readable in a fashion and hence perhaps a bit less fragile than our usual __put_user() patterns.
Btw., while at it we could also maybe fix the assignment ordering and use copy_to_user() naming:
if (__copy_to_user_fields(4,
&dirent->d_ino, d_ino,
&dirent->d_reclen, reclen,
dirent->d_name + namlen, 0,
(char __user *)dirent + reclen-1, d_type ))
goto efault;
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;That would make it even more readable.
(Thinking about the macro tricks needed for something like this gave me a bad headache though.)
Thanks,
Ingo
H. Peter Anvin
reviewerRe: [PATCH 01/11] x86: Use asm goto to implement better modify_and_test() functions
In reply to Andi KleenQuoted 10 lines · Andi Kleen
Quoted 3 lines · H. Peter Anvin
Yes, a bit sad. We allow bracketing with the get/put_user_try/catch blocks, but that is x86-specific.
I don't think a generic option is possible without compiler support, but it might be possible to do better than we do know.
Letting the compiler do it is a bit risky, because it may open it up for really large blocks, thus defeating the security advantages.
-Andi
The compiler support. if done right, should avoid unannotated memory references. This is a pretty big job in the compiler, though.
-hpa
Peter Zijlstra
maintainerPATCH 2/11 sched, rcu: Make RCU use resched_cpu()
In reply to Peter ZijlstraWe're going to deprecate and remove set_need_resched() for it will do the wrong thing. Make an exception for RCU and allow it to use resched_cpu() which will do the right thing.
--- kernel/rcutree.c | 15 ++++++++++++++- kernel/sched/core.c | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-)
kernel/rcutree.c +14 -1
@@ -898,6 +898,12 @@ static void print_other_cpu_stall(structforce_quiescent_state (rsp);/* Kick them all. */ } +/* +* This function really isn't for public consumption, but RCU is special in +* that context switches can allow the state machine to make progress. +*/ +extern void resched_cpu (int cpu); +static void print_cpu_stall (struct rcu_state *rsp) {int cpu;
@@ -927,7 +933,14 @@ static void print_cpu_stall(struct rcu_s3 *rcu_jiffies_till_stall_check () +3 ;raw_spin_unlock_irqrestore (&rnp->lock, flags); -set_need_resched ();/* kick ourselves to get things going. */ +/* +* Attempt to revive the RCU machinery by forcing a context switch. +* +* A context switch would normally allow the RCU state machine to make +* progress and it could be we're stuck in kernel space without context +* switches for an entirely unreasonable amount of time. +*/ +resched_cpu (smp_processor_id ()); }static void check_cpu_stall (struct rcu_state *rsp,struct rcu_data *rdp)
kernel/sched/core.c +2 -2
@@ -513,12 +513,11 @@ static inline void init_hrtick(void)* might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu; -assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;
@@ -546,6 +545,7 @@ void resched_cpu(int cpu)raw_spin_unlock_irqrestore (&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers
Peter Zijlstra
maintainerRe: [PATCH 02/11] sched, rcu: Make RCU use resched_cpu()
Thomas Meyer reported a UP build fail, should be fixed.
---
Subject: sched, rcu: Make RCU use resched_cpu()
From: Peter Zijlstra <[email protected]>
Date: Tue Sep 17 09:30:55 CEST 2013
We're going to deprecate and remove set_need_resched() for it will do the wrong thing. Make an exception for RCU and allow it to use resched_cpu() which will do the right thing.
--- kernel/rcutree.c | 15 ++++++++++++++- kernel/sched/core.c | 10 ++-------- 2 files changed, 16 insertions(+), 9 deletions(-)
kernel/rcutree.c +14 -1
@@ -898,6 +898,12 @@ static void print_other_cpu_stall(structforce_quiescent_state (rsp);/* Kick them all. */ } +/* +* This function really isn't for public consumption, but RCU is special in +* that context switches can allow the state machine to make progress. +*/ +extern void resched_cpu (int cpu); +static void print_cpu_stall (struct rcu_state *rsp) {int cpu;
@@ -927,7 +933,14 @@ static void print_cpu_stall(struct rcu_s3 *rcu_jiffies_till_stall_check () +3 ;raw_spin_unlock_irqrestore (&rnp->lock, flags); -set_need_resched ();/* kick ourselves to get things going. */ +/* +* Attempt to revive the RCU machinery by forcing a context switch. +* +* A context switch would normally allow the RCU state machine to make +* progress and it could be we're stuck in kernel space without context +* switches for an entirely unreasonable amount of time. +*/ +resched_cpu (smp_processor_id ()); }static void check_cpu_stall (struct rcu_state *rsp,struct rcu_data *rdp)
kernel/sched/core.c +2 -8
@@ -513,12 +513,11 @@ static inline void init_hrtick(void)* might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu; -assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;
@@ -546,6 +545,7 @@ void resched_cpu(int cpu)raw_spin_unlock_irqrestore (&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers
@@ -693,12 +693,6 @@ void sched_avg_update(struct rq *rq) } } -#else /* !CONFIG_SMP */ -void resched_task (struct task_struct *p) -{ -assert_raw_spin_locked (&task_rq (p)->lock); -set_tsk_need_resched (p); -}#endif /* CONFIG_SMP */ #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
Paul E. McKenney
AppliedRe: [PATCH 02/11] sched, rcu: Make RCU use resched_cpu()
Quoted 7 lines · Peter Zijlstra
Thomas Meyer reported a UP build fail, should be fixed.
---
Subject: sched, rcu: Make RCU useresched_cpu()
From: Peter Zijlstra <[email protected]>
Date: Tue Sep 17 09:30:55 CEST 2013
Quoted 6 lines
We're going to deprecate and remove
set_need_resched()for it will do the wrong thing. Make an exception for RCU and allow it to useresched_cpu()which will do the right thing.Cc Paul McKenney <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>
Queued for 3.13, thank you!
Thanx, Paul
Quoted 75 lines
--- kernel/rcutree.c | 15 ++++++++++++++- kernel/sched/core.c | 10 ++-------- 2 files changed, 16 insertions(+), 9 deletions(-)kernel/rcutree.c +14
-1@@ -898,6 +898,12 @@ static void print_other_cpu_stall(structforce_quiescent_state (rsp);/* Kick them all. */ } +/* +* This function really isn't for public consumption, but RCU is special in +* that context switches can allow the state machine to make progress. +*/ +extern void resched_cpu (int cpu); +static void print_cpu_stall (struct rcu_state *rsp) {int cpu;@@ -927,7 +933,14 @@ static void print_cpu_stall(struct rcu_s3 *rcu_jiffies_till_stall_check () +3 ;raw_spin_unlock_irqrestore (&rnp->lock, flags); -set_need_resched ();/* kick ourselves to get things going. */ +/* +* Attempt to revive the RCU machinery by forcing a context switch. +* +* A context switch would normally allow the RCU state machine to make +* progress and it could be we're stuck in kernel space without context +* switches for an entirely unreasonable amount of time. +*/ +resched_cpu (smp_processor_id ()); }static void check_cpu_stall (struct rcu_state *rsp,struct rcu_data *rdp)kernel/sched/core.c +2
-8@@ -513,12 +513,11 @@ static inline void init_hrtick(void)* might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu; -assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;@@ -546,6 +545,7 @@ void resched_cpu(int cpu)raw_spin_unlock_irqrestore (&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers @@ -693,12 +693,6 @@ void sched_avg_update(struct rq *rq) } } -#else /* !CONFIG_SMP */ -void resched_task (struct task_struct *p) -{ -assert_raw_spin_locked (&task_rq (p)->lock); -set_tsk_need_resched (p); -}#endif /* CONFIG_SMP */ #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
Re: [PATCH 02/11] sched, rcu: Make RCU use resched_cpu()
Quoted 8 lines · Paul E. McKenney
Quoted 7 lines · Peter Zijlstra
Thomas Meyer reported a UP build fail, should be fixed.
---
Subject: sched, rcu: Make RCU useresched_cpu()
From: Peter Zijlstra <[email protected]>
Date: Tue Sep 17 09:30:55 CEST 2013
Quoted 8 lines
Quoted 6 lines
We're going to deprecate and remove
set_need_resched()for it will do the wrong thing. Make an exception for RCU and allow it to useresched_cpu()which will do the right thing.Cc Paul McKenney <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>Queued for 3.13, thank you!
Passes light testing, but I am confused about why the change to resched_task() when only resched_cpu() is invoked elsewhere in the patch. Enlightenment?
Thanx, Paul
Quoted 75 lines
Quoted 75 lines
--- kernel/rcutree.c | 15 ++++++++++++++- kernel/sched/core.c | 10 ++-------- 2 files changed, 16 insertions(+), 9 deletions(-)kernel/rcutree.c +14
-1@@ -898,6 +898,12 @@ static void print_other_cpu_stall(structforce_quiescent_state (rsp);/* Kick them all. */ } +/* +* This function really isn't for public consumption, but RCU is special in +* that context switches can allow the state machine to make progress. +*/ +extern void resched_cpu (int cpu); +static void print_cpu_stall (struct rcu_state *rsp) {int cpu;@@ -927,7 +933,14 @@ static void print_cpu_stall(struct rcu_s3 *rcu_jiffies_till_stall_check () +3 ;raw_spin_unlock_irqrestore (&rnp->lock, flags); -set_need_resched ();/* kick ourselves to get things going. */ +/* +* Attempt to revive the RCU machinery by forcing a context switch. +* +* A context switch would normally allow the RCU state machine to make +* progress and it could be we're stuck in kernel space without context +* switches for an entirely unreasonable amount of time. +*/ +resched_cpu (smp_processor_id ()); }static void check_cpu_stall (struct rcu_state *rsp,struct rcu_data *rdp)kernel/sched/core.c +2
-8@@ -513,12 +513,11 @@ static inline void init_hrtick(void)* might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu; -assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;@@ -546,6 +545,7 @@ void resched_cpu(int cpu)raw_spin_unlock_irqrestore (&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers @@ -693,12 +693,6 @@ void sched_avg_update(struct rq *rq) } } -#else /* !CONFIG_SMP */ -void resched_task (struct task_struct *p) -{ -assert_raw_spin_locked (&task_rq (p)->lock); -set_tsk_need_resched (p); -}#endif /* CONFIG_SMP */ #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
Peter Zijlstra
maintainerRe: [PATCH 02/11] sched, rcu: Make RCU use resched_cpu()
Quoted 8 lines · Paul E. McKenney
Passes light testing, but I am confused about why the change to
resched_task()when onlyresched_cpu()is invoked elsewhere in the patch. Enlightenment?Quoted 5 lines
Quoted 5 lines
kernel/sched/core.c +0
-1-#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu;
Quoted 5 lines
Quoted 5 lines
kernel/sched/core.c static inline void init_hrtick(void) · line 523
Quoted 5 lines
kernel/sched/core.c +1
-1-assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;
That one? Fly by fixup I suppose.. the lockdep check is cheaper as it compiles away for !lockdep kernels, also its more correct.
Re: [PATCH 02/11] sched, rcu: Make RCU use resched_cpu()
Quoted 9 lines · Peter Zijlstra
Quoted 8 lines · Paul E. McKenney
Passes light testing, but I am confused about why the change to
resched_task()when onlyresched_cpu()is invoked elsewhere in the patch. Enlightenment?Quoted 5 lines
Quoted 5 lines
kernel/sched/core.c +0
-1-#ifdef CONFIG_SMP void resched_task (struct task_struct *p) {int cpu;
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
kernel/sched/core.c static inline void init_hrtick(void) · line 523
Quoted 5 lines
kernel/sched/core.c +1
-1-assert_raw_spin_locked (&task_rq (p)->lock); +lockdep_assert_held (&task_rq (p)->lock);if (test_tsk_need_resched (p))return ;That one? Fly by fixup I suppose.. the lockdep check is cheaper as it compiles away for !lockdep kernels, also its more correct.
OK, happy to carry it, just wanted to make sure it was supposed to be there.
Thanx, Paul
Peter Zijlstra
maintainerPATCH 3/11 sched: Remove {set,clear}_need_resched
In reply to Peter ZijlstraPreemption semantics are going to change which mandate a change.
All DRM usage sites are already broken and will not be affected (much) by this change. DRM people are aware and will remove the last few stragglers.
For now, leave an empty stub that generates a warning, once all users are gone we can remove this.
--- include/linux/thread_info.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-)
include/linux/thread_info.h +13 -2
@@ -104,8 +104,19 @@ static inline int test_ti_thread_flag(st#define test_thread_flag (flag) \ test_ti_thread_flag(current_thread_info(), flag) -#define set_need_resched () set_thread_flag(TIF_NEED_RESCHED) -#define clear_need_resched () clear_thread_flag(TIF_NEED_RESCHED) +static inline __deprecated void set_need_resched (void ) +{ +/* +* Use of this function in deprecated. +* +* As of this writing there are only a few users in the DRM tree left +* all of which are wrong and can be removed without causing too much +* grief. +* +* The DRM people are aware and are working on removing the last few +* instances. +*/ +}#if definedTIF_RESTORE_SIGMASK && !definedHAVE_SET_RESTORE_SIGMASK /*
Peter Zijlstra
maintainerPATCH 4/11 sched, idle: Fix the idle polling state logic
In reply to Peter ZijlstraMike reported that commit 7d1a9417 ("x86: Use generic idle loop") regressed several workloads and caused excessive reschedule interrupts.
The patch in question failed to notice that the x86 code had an inverted sense of the polling state versus the new generic code (x86:
default polling, generic: default !polling).
Fix the two prominent x86 mwait based idle drivers and introduce a few new generic polling helpers (fixing the wrong smp_mb__after_clear_bit usage).
Also switch the idle routines to using tif_need_resched() which is an immediate TIF_NEED_RESCHED test as opposed to need_resched which will end up being slightly different.
--- arch/x86/kernel/process.c | 6 +-- drivers/acpi/processor_idle.c | 46 +++++------------------- drivers/idle/intel_idle.c | 2 - include/linux/sched.h | 78 ++++++++++++++++++++++++++++++++++++++---- include/linux/thread_info.h | 2 + kernel/cpu/idle.c | 9 ++-- 6 files changed, 91 insertions(+), 52 deletions(-)
arch/x86/kernel/process.c +3 -3
@@ -391,9 +391,9 @@ static void amd_e400_idle(void)* The switch back from broadcast mode needs to be * called with interrupts disabled. */ -local_irq_disable (); -clockevents_notify (CLOCK_EVT_NOTIFY_BROADCAST_EXIT , &cpu); -local_irq_enable (); +local_irq_disable (); +clockevents_notify (CLOCK_EVT_NOTIFY_BROADCAST_EXIT , &cpu); +local_irq_enable (); }else default_idle (); }
drivers/acpi/processor_idle.c +10 -36
87 lines not on this page · raw message
drivers/idle/intel_idle.c +1 -1
@@ -359,7 +359,7 @@ static int intel_idle(struct cpuidle_devif (!(lapic_timer_reliable_states & (1 << (cstate))))clockevents_notify (CLOCK_EVT_NOTIFY_BROADCAST_ENTER , &cpu); -if (!need_resched ()) { +if (!current_set_polling_and_test ()) {__monitor ((void *)¤t_thread_info ()->flags,0 ,0 );smp_mb ();
include/linux/sched.h +71 -7
105 lines not on this page · raw message
include/linux/thread_info.h +2 -0
@@ -118,6 +118,8 @@ static inline __deprecated void set_need*/ } +#define tif_need_resched () test_thread_flag(TIF_NEED_RESCHED) +#if definedTIF_RESTORE_SIGMASK && !definedHAVE_SET_RESTORE_SIGMASK /* * An arch can define its own version of set_restore_sigmask() to get the
kernel/cpu/idle.c +4 -5
@@ -44,7 +44,7 @@ static inline int cpu_idle_poll(void)rcu_idle_enter ();trace_cpu_idle_rcuidle (0 ,smp_processor_id ());local_irq_enable (); -while (!need_resched ()) +while (!tif_need_resched ())cpu_relax ();trace_cpu_idle_rcuidle (PWR_EVENT_EXIT ,smp_processor_id ());rcu_idle_exit ();
@@ -92,8 +92,7 @@ static void cpu_idle_loop(void)if (cpu_idle_force_poll ||tick_check_broadcast_expired ()) {cpu_idle_poll (); }else { -current_clr_polling (); -if (!need_resched ()) { +if (!current_clr_polling_and_test ()) {stop_critical_timings ();rcu_idle_enter ();arch_cpu_idle ();
@@ -103,7 +102,7 @@ static void cpu_idle_loop(void) }else {local_irq_enable (); } -current_set_polling (); +__current_set_polling (); }arch_cpu_idle_exit (); }
@@ -129,7 +128,7 @@ void cpu_startup_entry(enum cpuhp_state*/ boot_init_stack_canary ();#endif -current_set_polling (); +__current_set_polling ();arch_cpu_idle_prepare ();cpu_idle_loop (); }
Peter Zijlstra
maintainerPATCH 5/11 sched: Introduce preempt_count accessor functions
In reply to Peter ZijlstraReplace the single preempt_count() 'function' that's an lvalue with two proper functions:
preempt_count() - returns the preempt_count value as rvalue preempt_count_ptr() - returns a pointer to the preempt_count
Then change all sites that modify the preempt count to use preempt_count_ptr().
--- include/linux/preempt.h | 20 ++++++++++++++------ init/main.c | 2 +- kernel/sched/core.c | 4 ++-- kernel/softirq.c | 4 ++-- kernel/timer.c | 8 ++++---- lib/smp_processor_id.c | 3 +-- 6 files changed, 24 insertions(+), 17 deletions(-)
include/linux/preempt.h +14 -6
@@ -10,19 +10,27 @@#include <linux/linkage.h> #include <linux/list.h> +static __always_inline int preempt_count(void) +{ +return current_thread_info ()->preempt_count; +} + +static __always_inline int *preempt_count_ptr(void) +{ +return ¤t_thread_info ()->preempt_count; +} +#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void add_preempt_count (int val);extern void sub_preempt_count (int val);#else -# define add_preempt_count (val) do { preempt_count() += (val); } while (0) -# define sub_preempt_count (val) do { preempt_count() -= (val); } while (0) +# define add_preempt_count (val) do { *preempt_count_ptr() += (val); } while (0) +# define sub_preempt_count (val) do { *preempt_count_ptr() -= (val); } while (0)#endif #define inc_preempt_count () add_preempt_count(1)#define dec_preempt_count () sub_preempt_count(1) -#define preempt_count () (current_thread_info()->preempt_count) -#ifdef CONFIG_PREEMPT asmlinkage void preempt_schedule(void);
@@ -81,9 +89,9 @@ do { \/* For debugging and tracer internals only! */ #define add_preempt_count_notrace (val) \ - do { preempt_count() += (val); } while (0) + do { *preempt_count_ptr() += (val); } while (0)#define sub_preempt_count_notrace (val) \ - do { preempt_count() -= (val); } while (0) + do { *preempt_count_ptr() -= (val); } while (0)#define inc_preempt_count_notrace () add_preempt_count_notrace(1)#define dec_preempt_count_notrace () sub_preempt_count_notrace(1)
init/main.c +1 -1
@@ -690,7 +690,7 @@ int __init_or_module do_one_initcall(iniif (preempt_count () != count) {sprintf (msgbuf,"preemption imbalance " ); -preempt_count () = count; + *preempt_count_ptr () = count; }if (irqs_disabled ()) {strlcat (msgbuf,"disabled interrupts " ,sizeof (msgbuf));
kernel/sched/core.c +2 -2
@@ -2233,7 +2233,7 @@ void __kprobes add_preempt_count(int valif (DEBUG_LOCKS_WARN_ON ((preempt_count () <0 )))return ;#endif -preempt_count () += val; + *preempt_count_ptr () += val;#ifdef CONFIG_DEBUG_PREEMPT /* * Spinlock count overflowing soon?
@@ -2264,7 +2264,7 @@ void __kprobes sub_preempt_count(int valif (preempt_count () == val)trace_preempt_on (CALLER_ADDR0 ,get_parent_ip (CALLER_ADDR1 )); -preempt_count () -= val; + *preempt_count_ptr () -= val; }EXPORT_SYMBOL (sub_preempt_count);
kernel/softirq.c +2 -2
@@ -106,7 +106,7 @@ static void __local_bh_disable(unsigned* We must manually increment preempt_count here and manually * call the trace_preempt_off later. */ -preempt_count () += cnt; + *preempt_count_ptr () += cnt;/* * Were softirqs turned off above: */
@@ -256,7 +256,7 @@ asmlinkage void __do_softirq(void)" exited with %08x?\n" , vec_nr, softirq_to_name[vec_nr], h->action, prev_count, preempt_count()); -preempt_count () = prev_count; + *preempt_count_ptr () = prev_count; }rcu_bh_qs (cpu);
kernel/timer.c +4 -4
@@ -1092,7 +1092,7 @@ static int cascade(struct tvec_base *basstatic void call_timer_fn (struct timer_list *timer,void (*fn)(unsigned long ),unsigned long data) { -int preempt_count =preempt_count (); +int count =preempt_count ();#ifdef CONFIG_LOCKDEP /*
@@ -1119,16 +1119,16 @@ static void call_timer_fn(struct timer_llock_map_release (&lockdep_map); -if (preempt_count !=preempt_count ()) { +if (count !=preempt_count ()) {WARN_ONCE (1 ,"timer: %pF preempt leak: %08x -> %08x\n" , - fn, preempt_count,preempt_count ()); + fn, count,preempt_count ());/* * Restore the preempt count. That gives us a decent * chance to survive and extract information. If the * callback kept a lock held, bad luck, but not worse * than the BUG() we had. */ -preempt_count () = preempt_count; + *preempt_count_ptr () = count; } }
lib/smp_processor_id.c +1 -2
@@ -9,10 +9,9 @@ notrace unsigned int debug_smp_processor_id(void) { -unsigned long preempt_count =preempt_count ();int this_cpu =raw_smp_processor_id (); -if (likely (preempt_count)) +if (likely (preempt_count ())) goto out;if (irqs_disabled ())
Peter Zijlstra
maintainerPATCH 6/11 sched: Add NEED_RESCHED to the preempt_count
In reply to Peter ZijlstraIn order to combine the preemption and need_resched test we need to fold the need_resched information into the preempt_count value.
Since the NEED_RESCHED flag is set across CPUs this needs to be an atomic operation, however we very much want to avoid making preempt_count atomic, therefore we keep the existing TIF_NEED_RESCHED infrastructure in place but at 3 sites test it and fold its value into preempt_count; namely:
- resched_task() when setting TIF_NEED_RESCHED on the current task
- scheduler_ipi() when resched_task() sets TIF_NEED_RESCHED on a remote task it follows it up with a reschedule IPI and we can modify the cpu local preempt_count from there.
- cpu_idle_loop() for when resched_task() found tsk_is_polling().
We use an inverted bitmask to indicate need_resched so that a 0 means both need_resched and !atomic.
Also remove the barrier() in preempt_enable() between preempt_enable_no_resched() and preempt_check_resched() to avoid having to reload the preemption value and allow the compiler to use the flags of the previuos decrement. I couldn't come up with any sane reason for this barrier() to be there as preempt_enable_no_resched() already has a barrier() before doing the decrement.
--- include/linux/preempt.h | 42 +++++++++++++++++++++++++++++++++++++----- include/linux/sched.h | 2 +- kernel/context_tracking.c | 2 +- kernel/cpu/idle.c | 7 +++++++ kernel/sched/core.c | 18 ++++++++++++++---- 5 files changed, 60 insertions(+), 11 deletions(-)
include/linux/preempt.h +37 -5
@@ -10,9 +10,19 @@#include <linux/linkage.h> #include <linux/list.h> +/* +* We use the MSB mostly because its available; see <linux/hardirq.h> for +* the other bits. +*/ +#define PREEMPT_NEED_RESCHED 0x80000000 + +/* +* We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users +* that think a non-zero value indicates we cannot preempt. +*/ static __always_inline int preempt_count(void) { -return current_thread_info ()->preempt_count; +return current_thread_info ()->preempt_count & ~PREEMPT_NEED_RESCHED ; } static __always_inline int *preempt_count_ptr(void)
@@ -20,6 +30,30 @@ static __always_inline int *preempt_counreturn ¤t_thread_info ()->preempt_count; } +/* +* We fold the NEED_RESCHED bit into the preempt count such that +* preempt_enable() can decrement and test for needing to reschedule with a +* single instruction. +* +* We invert the actual bit, so that when the decrement hits 0 we know we both +* need to resched (the bit is cleared) and can resched (no preempt count). +*/ + +static __always_inline void set_preempt_need_resched(void) +{ + *preempt_count_ptr () &= ~PREEMPT_NEED_RESCHED ; +} + +static __always_inline void clear_preempt_need_resched(void) +{ + *preempt_count_ptr () |=PREEMPT_NEED_RESCHED ; +} + +static __always_inline bool test_preempt_need_resched(void) +{ +return !(*preempt_count_ptr () &PREEMPT_NEED_RESCHED ); +} +#if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void add_preempt_count (int val);extern void sub_preempt_count (int val);
@@ -37,7 +71,7 @@ asmlinkage void preempt_schedule(void);#define preempt_check_resched () \ do { \ - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + if (unlikely(!*preempt_count_ptr())) \ preempt_schedule(); \ } while (0)
@@ -47,7 +81,7 @@ void preempt_schedule_context(void);#define preempt_check_resched_context () \ do { \ - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + if (unlikely(!*preempt_count_ptr())) \ preempt_schedule_context(); \ } while (0)#else
@@ -83,7 +117,6 @@ do { \#define preempt_enable () \ do { \ preempt_enable_no_resched(); \ - barrier(); \ preempt_check_resched(); \ } while (0)
@@ -111,7 +144,6 @@ do { \#define preempt_enable_notrace () \ do { \ preempt_enable_no_resched_notrace(); \ - barrier(); \ preempt_check_resched_context(); \ } while (0)
include/linux/sched.h +1 -1
@@ -2408,7 +2408,7 @@ static inline int signal_pending_state(lstatic inline int need_resched (void ) { -return unlikely (test_thread_flag (TIF_NEED_RESCHED )); +return unlikely (test_preempt_need_resched ()); }/*
kernel/context_tracking.c +1 -1
@@ -115,7 +115,7 @@ void __sched notrace preempt_schedule_co {enum ctx_state prev_ctx; -if (likely (!preemptible ())) +if (likely (preempt_count () ||irqs_disabled ()))return ;/*
kernel/cpu/idle.c +7 -0
@@ -105,6 +105,13 @@ static void cpu_idle_loop(void)__current_set_polling (); }arch_cpu_idle_exit (); +/* +* We need to test and propagate the TIF_NEED_RESCHED +* bit here because we might not have send the +* reschedule IPI to idle tasks. +*/ +if (tif_need_resched ()) +set_preempt_need_resched (); }tick_nohz_idle_exit ();schedule_preempt_disabled ();
kernel/sched/core.c +14 -4
@@ -525,8 +525,10 @@ void resched_task(struct task_struct *p)set_tsk_need_resched (p); cpu =task_cpu (p); -if (cpu ==smp_processor_id ()) +if (cpu ==smp_processor_id ()) { +set_preempt_need_resched ();return ; + }/* NEED_RESCHED must be visible before we test polling */ smp_mb ();
@@ -1397,6 +1399,14 @@ static void sched_ttwu_pending(void)void scheduler_ipi (void ) { +/* +* Fold TIF_NEED_RESCHED into the preempt_count; anybody setting +* TIF_NEED_RESCHED remotely (for the first time) will also send +* this IPI. +*/ +if (tif_need_resched ()) +set_preempt_need_resched (); +if (llist_empty (&this_rq ()->wake_list) && !tick_nohz_full_cpu (smp_processor_id ()) && !got_nohz_idle_kick ())
@@ -2431,6 +2441,7 @@ static void __sched __schedule(void)put_prev_task (rq, prev); next =pick_next_task (rq);clear_tsk_need_resched (prev); +clear_preempt_need_resched (); rq->skip_clock_update =0 ;if (likely (prev != next)) {
@@ -2517,7 +2528,7 @@ asmlinkage void __sched notrace preempt_* If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. */ -if (likely (!preemptible ())) +if (likely (preempt_count () ||irqs_disabled ()))return ; do {
@@ -2542,11 +2553,10 @@ EXPORT_SYMBOL(preempt_schedule);*/ asmlinkage void __sched preempt_schedule_irq(void) { -struct thread_info *ti =current_thread_info ();enum ctx_state prev_state;/* Catch callers which need to be fixed */ -BUG_ON (ti->preempt_count || !irqs_disabled ()); +BUG_ON (preempt_count () || !irqs_disabled ()); prev_state =exception_enter ();
Peter Zijlstra
maintainerPATCH 7/11 sched, arch: Create asm/preempt.h
In reply to Peter ZijlstraIn order to prepare to per-arch implementations of preempt_count move the required bits into an asm-generic header and use this for all archs.
--- arch/alpha/include/asm/Kbuild | 1 arch/arc/include/asm/Kbuild | 1 arch/arm/include/asm/Kbuild | 1 arch/arm64/include/asm/Kbuild | 1 arch/avr32/include/asm/Kbuild | 1 arch/blackfin/include/asm/Kbuild | 1 arch/c6x/include/asm/Kbuild | 1 arch/cris/include/asm/Kbuild | 1 arch/frv/include/asm/Kbuild | 1 arch/h8300/include/asm/Kbuild | 1 arch/hexagon/include/asm/Kbuild | 1 arch/ia64/include/asm/Kbuild | 1 arch/m32r/include/asm/Kbuild | 1 arch/m68k/include/asm/Kbuild | 1 arch/metag/include/asm/Kbuild | 1 arch/microblaze/include/asm/Kbuild | 1 arch/mips/include/asm/Kbuild | 1 arch/mn10300/include/asm/Kbuild | 1 arch/openrisc/include/asm/Kbuild | 1 arch/parisc/include/asm/Kbuild | 1 arch/powerpc/include/asm/Kbuild | 1 arch/s390/include/asm/Kbuild | 1 arch/score/include/asm/Kbuild | 1 arch/sh/include/asm/Kbuild | 1 arch/sparc/include/asm/Kbuild | 1 arch/tile/include/asm/Kbuild | 1 arch/um/include/asm/Kbuild | 1 arch/unicore32/include/asm/Kbuild | 1 arch/x86/include/asm/Kbuild | 1 arch/xtensa/include/asm/Kbuild | 1 include/asm-generic/preempt.h | 44 +++++++++++++++++++++++++++++++++++++ include/linux/preempt.h | 39 -------------------------------- 32 files changed, 75 insertions(+), 38 deletions(-)
arch/alpha/include/asm/Kbuild +1 -0
@@ -3,3 +3,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += trace_clock.h +generic-y += preempt.h
arch/arc/include/asm/Kbuild +1 -0
@@ -46,3 +46,4 @@ generic-y += ucontext.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/arm/include/asm/Kbuild +1 -0
@@ -33,3 +33,4 @@ generic-y += timex.h generic-y += trace_clock.h generic-y += types.h generic-y += unaligned.h +generic-y += preempt.h
arch/arm64/include/asm/Kbuild +1 -0
@@ -50,3 +50,4 @@ generic-y += unaligned.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/avr32/include/asm/Kbuild +1 -0
@@ -3,3 +3,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += trace_clock.h generic-y += param.h +generic-y += preempt.h
arch/blackfin/include/asm/Kbuild +1 -0
@@ -44,3 +44,4 @@ generic-y += ucontext.h generic-y += unaligned.h generic-y += user.h generic-y += xor.h +generic-y += preempt.h
arch/c6x/include/asm/Kbuild +1 -0
@@ -56,3 +56,4 @@ generic-y += ucontext.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/cris/include/asm/Kbuild +1 -0
@@ -11,3 +11,4 @@ generic-y += module.h generic-y += trace_clock.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/frv/include/asm/Kbuild +1 -0
@@ -2,3 +2,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += trace_clock.h +generic-y += preempt.h
arch/h8300/include/asm/Kbuild +1 -0
@@ -6,3 +6,4 @@ generic-y += mmu.h generic-y += module.h generic-y += trace_clock.h generic-y += xor.h +generic-y += preempt.h
arch/hexagon/include/asm/Kbuild +1 -0
@@ -53,3 +53,4 @@ generic-y += types.h generic-y += ucontext.h generic-y += unaligned.h generic-y += xor.h +generic-y += preempt.h
arch/ia64/include/asm/Kbuild +1 -0
@@ -3,4 +3,5 @@ generic-y += clkdev.h generic-y += exec.h generic-y += kvm_para.h generic-y += trace_clock.h +generic-y += preempt.h generic-y += vtime.h
arch/m32r/include/asm/Kbuild +1 -0
@@ -3,3 +3,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += module.h generic-y += trace_clock.h +generic-y += preempt.h
arch/m68k/include/asm/Kbuild +1 -0
@@ -31,3 +31,4 @@ generic-y += trace_clock.h generic-y += types.h generic-y += word-at-a-time.h generic-y += xor.h +generic-y += preempt.h
arch/metag/include/asm/Kbuild +1 -0
@@ -52,3 +52,4 @@ generic-y += unaligned.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/microblaze/include/asm/Kbuild +1 -0
@@ -3,3 +3,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += trace_clock.h generic-y += syscalls.h +generic-y += preempt.h
arch/mips/include/asm/Kbuild +1 -0
@@ -11,5 +11,6 @@ generic-y += sections.h generic-y += segment.h generic-y += serial.h generic-y += trace_clock.h +generic-y += preempt.h generic-y += ucontext.h generic-y += xor.h
arch/mn10300/include/asm/Kbuild +1 -0
@@ -2,3 +2,4 @@ generic-y += clkdev.h generic-y += exec.h generic-y += trace_clock.h +generic-y += preempt.h
arch/openrisc/include/asm/Kbuild +1 -0
@@ -67,3 +67,4 @@ generic-y += ucontext.h generic-y += user.h generic-y += word-at-a-time.h generic-y += xor.h +generic-y += preempt.h
arch/parisc/include/asm/Kbuild +1 -0
@@ -4,3 +4,4 @@ generic-y += word-at-a-time.h auxvec.h u div64.h irq_regs.h kdebug.h kvm_para.h local64.h local.h param.h \ poll.h xor.h clkdev.h exec.h generic-y += trace_clock.h +generic-y += preempt.h
arch/powerpc/include/asm/Kbuild +1 -0
@@ -2,4 +2,5 @@ generic-y += clkdev.h generic-y += rwsem.h generic-y += trace_clock.h +generic-y += preempt.h generic-y += vtime.h
arch/s390/include/asm/Kbuild +1 -0
@@ -2,3 +2,4 @@ generic-y += clkdev.h generic-y += trace_clock.h +generic-y += preempt.h
arch/score/include/asm/Kbuild +1 -0
@@ -4,3 +4,4 @@ header-y += generic-y += clkdev.h generic-y += trace_clock.h generic-y += xor.h +generic-y += preempt.h
arch/sh/include/asm/Kbuild +1 -0
@@ -34,3 +34,4 @@ generic-y += termios.h generic-y += trace_clock.h generic-y += ucontext.h generic-y += xor.h +generic-y += preempt.h
arch/sparc/include/asm/Kbuild +1 -0
@@ -16,3 +16,4 @@ generic-y += serial.h generic-y += trace_clock.h generic-y += types.h generic-y += word-at-a-time.h +generic-y += preempt.h
arch/tile/include/asm/Kbuild +1 -0
@@ -38,3 +38,4 @@ generic-y += termios.h generic-y += trace_clock.h generic-y += types.h generic-y += xor.h +generic-y += preempt.h
arch/um/include/asm/Kbuild +1 -0
@@ -3,3 +3,4 @@ generic-y += hw_irq.h irq_regs.h kdebug. generic-y += ftrace.h pci.h io.h param.h delay.h mutex.h current.h exec.h generic-y += switch_to.h clkdev.h generic-y += trace_clock.h +generic-y += preempt.h
arch/unicore32/include/asm/Kbuild +1 -0
@@ -60,3 +60,4 @@ generic-y += unaligned.h generic-y += user.h generic-y += vga.h generic-y += xor.h +generic-y += preempt.h
arch/x86/include/asm/Kbuild +1 -0
@@ -5,3 +5,4 @@ genhdr-y += unistd_64.h genhdr-y += unistd_x32.h generic-y += clkdev.h +generic-y += preempt.h
arch/xtensa/include/asm/Kbuild +1 -0
@@ -28,3 +28,4 @@ generic-y += termios.h generic-y += topology.h generic-y += trace_clock.h generic-y += xor.h +generic-y += preempt.h
include/asm-generic/preempt.h new file +44 -0
44 lines not on this page · raw message
include/linux/preempt.h +1 -38
@@ -6,7 +6,6 @@* preempt_count (used for kernel preemption, interrupt count, etc.) */ -#include <linux/thread_info.h> #include <linux/linkage.h> #include <linux/list.h>
@@ -16,43 +15,7 @@*/ #define PREEMPT_NEED_RESCHED 0x80000000 -/* -* We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users -* that think a non-zero value indicates we cannot preempt. -*/ -static __always_inline int preempt_count(void) -{ -return current_thread_info ()->preempt_count & ~PREEMPT_NEED_RESCHED ; -} - -static __always_inline int *preempt_count_ptr(void) -{ -return ¤t_thread_info ()->preempt_count; -} - -/* -* We fold the NEED_RESCHED bit into the preempt count such that -* preempt_enable() can decrement and test for needing to reschedule with a -* single instruction. -* -* We invert the actual bit, so that when the decrement hits 0 we know we both -* need to resched (the bit is cleared) and can resched (no preempt count). -*/ - -static __always_inline void set_preempt_need_resched(void) -{ - *preempt_count_ptr () &= ~PREEMPT_NEED_RESCHED ; -} - -static __always_inline void clear_preempt_need_resched(void) -{ - *preempt_count_ptr () |=PREEMPT_NEED_RESCHED ; -} - -static __always_inline bool test_preempt_need_resched(void) -{ -return !(*preempt_count_ptr () &PREEMPT_NEED_RESCHED ); -} +#include <asm/preempt.h> #if defined(CONFIG_DEBUG_PREEMPT ) || defined(CONFIG_PREEMPT_TRACER )extern void add_preempt_count (int val);
Peter Zijlstra
maintainerPATCH 8/11 sched: Create more preempt_count accessors
In reply to Peter ZijlstraWe need a few special preempt_count accessors:
- task_preempt_count() for when we're interested in the preemption count of another (non-running) task.
- init_task_preempt_count() for properly initializing the preemption count.
- init_idle_preempt_count() a special case of the above for the idle threads.
With these no generic code ever touches thread_info::preempt_count anymore and architectures could choose to remove it.
--- include/asm-generic/preempt.h | 14 ++++++++++++++ include/trace/events/sched.h | 2 +- kernel/sched/core.c | 7 +++---- 3 files changed, 18 insertions(+), 5 deletions(-)
include/asm-generic/preempt.h +14 -0
@@ -18,6 +18,20 @@ static __always_inline int *preempt_coun }/* +* must be macros to avoid header recursion hell +*/ +#define task_preempt_count (p) \ + (task_thread_info(p)->preempt_count & ~PREEMPT_NEED_RESCHED) + +#define init_task_preempt_count (p) do { \ + task_thread_info(p)->preempt_count = 1 | PREEMPT_NEED_RESCHED; \ +} while (0) + +#define init_idle_preempt_count (p, cpu) do { \ + task_thread_info(p)->preempt_count = 0; \ +} while (0) + +/* * We fold the NEED_RESCHED bit into the preempt count such that * preempt_enable() can decrement and test for needing to reschedule with a * single instruction.
include/trace/events/sched.h +1 -1
@@ -100,7 +100,7 @@ static inline long __trace_sched_switch_/* * For all intents and purposes a preempted task is a running task. */ -if (task_thread_info (p)->preempt_count &PREEMPT_ACTIVE ) +if (task_preempt_count (p) &PREEMPT_ACTIVE ) state =TASK_RUNNING |TASK_STATE_MAX ;#endif
kernel/sched/core.c +3 -4
@@ -996,7 +996,7 @@ void set_task_cpu(struct task_struct *p,* ttwu() will sort out the placement. */ WARN_ON_ONCE (p->state !=TASK_RUNNING && p->state !=TASK_WAKING && - !(task_thread_info (p)->preempt_count &PREEMPT_ACTIVE )); + !(task_preempt_count (p) &PREEMPT_ACTIVE ));#ifdef CONFIG_LOCKDEP /*
@@ -1743,8 +1743,7 @@ void sched_fork(struct task_struct *p) p->on_cpu =0 ;#endif #ifdef CONFIG_PREEMPT_COUNT -/* Want to start with kernel preemption disabled. */ -task_thread_info (p)->preempt_count =1 ; +init_task_preempt_count (p);#endif #ifdef CONFIG_SMP plist_node_init (&p->pushable_tasks,MAX_PRIO );
@@ -4237,7 +4236,7 @@ void init_idle(struct task_struct *idle,raw_spin_unlock_irqrestore (&rq->lock, flags);/* Set the preempt count _outside_ the spinlocks! */ -task_thread_info (idle)->preempt_count =0 ; +init_idle_preempt_count (idle, cpu);/* * The idle tasks have their own, simple scheduling class:
Peter Zijlstra
maintainerPATCH 9/11 sched: Extract the basic add/sub preempt_count modifiers
In reply to Peter ZijlstraRewrite the preempt_count macros in order to extract the 3 basic preempt_count value modifiers:
__preempt_count_add()
__preempt_count_sub()
and the new:
__preempt_count_dec_and_test()
And since we're at it anyway, replace the unconventional $op_preempt_count names with the more conventional preempt_count_$op.
Since these basic operators are equivalent to the previous _notrace() variants, do away with the _notrace() versions.
--- arch/mips/mm/init.c | 5 - arch/x86/kernel/traps.c | 4 - include/asm-generic/preempt.h | 35 +++++++++++++ include/linux/hardirq.h | 8 +-- include/linux/preempt.h | 106 +++++++++++++++++++----------------------- include/linux/sched.h | 5 - include/linux/uaccess.h | 8 --- kernel/context_tracking.c | 2 kernel/sched/core.c | 29 ++++------- kernel/softirq.c | 12 ++-- 10 files changed, 112 insertions(+), 102 deletions(-)
arch/mips/mm/init.c +2 -3
@@ -124,7 +124,7 @@ void *kmap_coherent(struct page *page, uBUG_ON (Page_dcache_dirty (page)); -inc_preempt_count (); +pagefault_disable (); idx = (addr >>PAGE_SHIFT ) & (FIX_N_COLOURS -1 );#ifdef CONFIG_MIPS_MT_SMTC idx += FIX_N_COLOURS * smp_processor_id() +
@@ -193,8 +193,7 @@ void kunmap_coherent(void)write_c0_entryhi (old_ctx);EXIT_CRITICAL (flags);#endif -dec_preempt_count (); -preempt_check_resched (); +pagefault_enable (); } void copy_user_highpage(struct page *to, struct page *from,
arch/x86/kernel/traps.c +2 -2
@@ -88,7 +88,7 @@ static inline void conditional_sti(strucstatic inline void preempt_conditional_sti (struct pt_regs *regs) { -inc_preempt_count (); +preempt_count_inc ();if (regs->flags &X86_EFLAGS_IF )local_irq_enable (); }
@@ -103,7 +103,7 @@ static inline void preempt_conditional_c {if (regs->flags &X86_EFLAGS_IF )local_irq_disable (); -dec_preempt_count (); +preempt_count_dec (); } static int __kprobes
include/asm-generic/preempt.h +35 -0
@@ -55,4 +55,39 @@ static __always_inline bool test_preempt return !(*preempt_count_ptr() & PREEMPT_NEED_RESCHED); } +/* +* The various preempt_count add/sub methods +*/ + +static __always_inline void __preempt_count_add(int val) +{ + *preempt_count_ptr () += val; +} + +static __always_inline void __preempt_count_sub(int val) +{ + *preempt_count_ptr () -= val; +} + +static __always_inline bool __preempt_count_dec_and_test(void) +{ +return !--*preempt_count_ptr (); +} + +/* +* Returns true when we need to resched -- even if we can not. +*/ +static __always_inline bool need_resched(void) +{ +return unlikely (test_preempt_need_resched ()); +} + +/* +* Returns true when we need to resched and can (barring IRQ state). +*/ +static __always_inline bool should_resched(void) +{ +return unlikely (!*preempt_count_ptr ()); +} +#endif /* __ASM_PREEMPT_H */
include/linux/hardirq.h +4 -4
@@ -37,7 +37,7 @@ extern void rcu_nmi_exit(void);#define __irq_enter () \ do { \ account_irq_enter_time(current); \ - add_preempt_count(HARDIRQ_OFFSET); \ + preempt_count_add(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ } while (0)
@@ -53,7 +53,7 @@ extern void irq_enter(void);do { \trace_hardirq_exit (); \account_irq_exit_time (current); \ -sub_preempt_count (HARDIRQ_OFFSET ); \ +preempt_count_sub (HARDIRQ_OFFSET ); \ }while (0 )/*
@@ -66,7 +66,7 @@ extern void irq_exit(void);lockdep_off (); \ftrace_nmi_enter (); \BUG_ON (in_nmi ()); \ -add_preempt_count (NMI_OFFSET +HARDIRQ_OFFSET ); \ +preempt_count_add (NMI_OFFSET +HARDIRQ_OFFSET ); \rcu_nmi_enter (); \trace_hardirq_enter (); \ }while (0 )
@@ -76,7 +76,7 @@ extern void irq_exit(void);trace_hardirq_exit (); \rcu_nmi_exit (); \BUG_ON (!in_nmi ()); \ -sub_preempt_count (NMI_OFFSET +HARDIRQ_OFFSET ); \ +preempt_count_sub (NMI_OFFSET +HARDIRQ_OFFSET ); \ftrace_nmi_exit (); \lockdep_on (); \ }while (0 )
include/linux/preempt.h +48 -58
155 lines not on this page · raw message
include/linux/sched.h +0 -5
@@ -2403,11 +2403,6 @@ static inline int signal_pending_state(lreturn (state &TASK_INTERRUPTIBLE ) ||__fatal_signal_pending (p); } -static inline int need_resched (void ) -{ -return unlikely (test_preempt_need_resched ()); -} -/* * cond_resched() and cond_resched_lock(): latency reduction via * explicit rescheduling in places that are safe. The return
include/linux/uaccess.h +2 -6
@@ -15,7 +15,7 @@*/ static inline void pagefault_disable (void ) { -inc_preempt_count (); +preempt_count_inc ();/* * make sure to have issued the store before a pagefault * can hit.
@@ -30,11 +30,7 @@ static inline void pagefault_enable(void* the pagefault handler again. */ barrier (); -dec_preempt_count (); -/* -* make sure we do.. -*/ -barrier (); +preempt_count_dec ();preempt_check_resched (); }
kernel/context_tracking.c +1 -1
@@ -111,7 +111,7 @@ void context_tracking_user_enter(void)* instead of preempt_schedule() to exit user context if needed before * calling the scheduler. */ -void __sched notrace preempt_schedule_context(void) +asmlinkage void __sched notrace preempt_schedule_context(void) {enum ctx_state prev_ctx;
kernel/sched/core.c +12 -17
@@ -2239,7 +2239,7 @@ notrace unsigned long get_parent_ip(unsi#if defined(CONFIG_PREEMPT ) && (defined(CONFIG_DEBUG_PREEMPT ) || \ defined(CONFIG_PREEMPT_TRACER )) -void __kprobes add_preempt_count (int val) +void __kprobes preempt_count_add (int val) {#ifdef CONFIG_DEBUG_PREEMPT /*
@@ -2248,7 +2248,7 @@ void __kprobes add_preempt_count(int valif (DEBUG_LOCKS_WARN_ON ((preempt_count () <0 )))return ;#endif - *preempt_count_ptr () += val; +__preempt_count_add (val);#ifdef CONFIG_DEBUG_PREEMPT /* * Spinlock count overflowing soon?
@@ -2259,9 +2259,9 @@ void __kprobes add_preempt_count(int valif (preempt_count () == val)trace_preempt_off (CALLER_ADDR0 ,get_parent_ip (CALLER_ADDR1 )); } -EXPORT_SYMBOL (add_preempt_count); +EXPORT_SYMBOL (preempt_count_add); -void __kprobes sub_preempt_count (int val) +void __kprobes preempt_count_sub (int val) {#ifdef CONFIG_DEBUG_PREEMPT /*
@@ -2279,9 +2279,9 @@ void __kprobes sub_preempt_count(int valif (preempt_count () == val)trace_preempt_on (CALLER_ADDR0 ,get_parent_ip (CALLER_ADDR1 )); - *preempt_count_ptr() -= val; + __preempt_count_sub(val); } -EXPORT_SYMBOL (sub_preempt_count); +EXPORT_SYMBOL (preempt_count_sub);#endif
@@ -2545,9 +2545,9 @@ asmlinkage void __sched notrace preempt_return ; do { - add_preempt_count_notrace(PREEMPT_ACTIVE); + __preempt_count_add(PREEMPT_ACTIVE); __schedule(); - sub_preempt_count_notrace(PREEMPT_ACTIVE); + __preempt_count_sub(PREEMPT_ACTIVE); /* * Check again in case we missed a preemption opportunity
@@ -2574,11 +2574,11 @@ asmlinkage void __sched preempt_schedule prev_state =exception_enter (); do { - add_preempt_count(PREEMPT_ACTIVE); + __preempt_count_add(PREEMPT_ACTIVE); local_irq_enable(); __schedule(); local_irq_disable(); - sub_preempt_count(PREEMPT_ACTIVE); + __preempt_count_sub(PREEMPT_ACTIVE); /* * Check again in case we missed a preemption opportunity
@@ -3818,16 +3818,11 @@ SYSCALL_DEFINE0(sched_yield)return 0 ; } -static inline int should_resched (void ) -{ -return need_resched () && !(preempt_count () &PREEMPT_ACTIVE ); -} -static void __cond_resched (void ) { -add_preempt_count (PREEMPT_ACTIVE ); +preempt_count_add (PREEMPT_ACTIVE );__schedule (); -sub_preempt_count (PREEMPT_ACTIVE ); +preempt_count_sub (PREEMPT_ACTIVE ); } int __sched _cond_resched(void)
kernel/softirq.c +6 -6
@@ -100,7 +100,7 @@ static void __local_bh_disable(unsignedraw_local_irq_save (flags);/* -* The preempt tracer hooks into add_preempt_count and will break +* The preempt tracer hooks into preempt_count_add and will break * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET * is set and before current->softirq_enabled is cleared. * We must manually increment preempt_count here and manually
@@ -120,7 +120,7 @@ static void __local_bh_disable(unsigned#else /* !CONFIG_TRACE_IRQFLAGS */ static inline void __local_bh_disable (unsigned long ip,unsigned int cnt) { -add_preempt_count (cnt); +preempt_count_add (cnt);barrier (); }#endif /* CONFIG_TRACE_IRQFLAGS */
@@ -139,7 +139,7 @@ static void __local_bh_enable(unsigned iif (softirq_count () == cnt)trace_softirqs_on (_RET_IP_); -sub_preempt_count (cnt); +preempt_count_sub (cnt); }/*
@@ -169,12 +169,12 @@ static inline void _local_bh_enable_ip(u* Keep preemption disabled until we are done with * softirq processing: */ -sub_preempt_count (SOFTIRQ_DISABLE_OFFSET -1 ); +preempt_count_sub (SOFTIRQ_DISABLE_OFFSET -1 );if (unlikely (!in_interrupt () &&local_softirq_pending ()))do_softirq (); -dec_preempt_count (); +preempt_count_dec ();#ifdef CONFIG_TRACE_IRQFLAGS local_irq_enable ();#endif
@@ -360,7 +360,7 @@ void irq_exit(void)account_irq_exit_time (current);trace_hardirq_exit (); -sub_preempt_count (HARDIRQ_OFFSET ); +preempt_count_sub (HARDIRQ_OFFSET );if (!in_interrupt () &&local_softirq_pending ())invoke_softirq ();
Peter Zijlstra
maintainerPATCH 10/11 sched, x86: Provide a per-cpu preempt_count implementation
In reply to Peter ZijlstraConvert x86 to use a per-cpu preemption count. The reason for doing so is that accessing per-cpu variables is a lot cheaper than accessing thread_info variables.
We still need to save/restore the actual preemption count due to PREEMPT_ACTIVE so we place the per-cpu __preempt_count variable in the same cache-line as the other hot __switch_to() variables such as current_task.
Also rename thread_info::preempt_count to ensure nobody is 'accidentally' still poking at it.
--- arch/x86/include/asm/Kbuild | 1 arch/x86/include/asm/preempt.h | 98 +++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/thread_info.h | 5 - arch/x86/kernel/asm-offsets.c | 1 arch/x86/kernel/cpu/common.c | 5 + arch/x86/kernel/entry_32.S | 7 -- arch/x86/kernel/entry_64.S | 4 - arch/x86/kernel/process_32.c | 10 +++ arch/x86/kernel/process_64.c | 10 +++ 9 files changed, 128 insertions(+), 13 deletions(-)
arch/x86/include/asm/Kbuild +0 -1
@@ -5,4 +5,3 @@ genhdr-y += unistd_64.h genhdr-y += unistd_x32.h generic-y += clkdev.h -generic-y += preempt.h
arch/x86/include/asm/preempt.h new file +98 -0
98 lines not on this page · raw message
arch/x86/include/asm/thread_info.h +2 -3
@@ -28,8 +28,7 @@ struct thread_info {__u32 flags;/* low level flags */ __u32 status;/* thread synchronous flags */ __u32 cpu;/* current CPU */ -int preempt_count;/* 0 => preemptable, -<0 => BUG */ +int saved_preempt_count;mm_segment_t addr_limit;struct restart_block restart_block;void __user *sysenter_return;
@@ -49,7 +48,7 @@ struct thread_info { .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ - .preempt_count = INIT_PREEMPT_COUNT, \ + .saved_preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ .restart_block = { \ .fn = do_no_restart_syscall, \
arch/x86/kernel/asm-offsets.c +0 -1
@@ -32,7 +32,6 @@ void common(void) {OFFSET (TI_flags, thread_info, flags);OFFSET (TI_status, thread_info, status);OFFSET (TI_addr_limit, thread_info, addr_limit); -OFFSET (TI_preempt_count, thread_info, preempt_count);BLANK ();OFFSET (crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
arch/x86/kernel/cpu/common.c +5 -0
@@ -1095,6 +1095,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) = DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; +DEFINE_PER_CPU (int, __preempt_count) =INIT_PREEMPT_COUNT ; +EXPORT_PER_CPU_SYMBOL (__preempt_count); + DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);/*
@@ -1169,6 +1172,8 @@ void debug_stack_reset(void) DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;EXPORT_PER_CPU_SYMBOL (current_task); +DEFINE_PER_CPU (int, __preempt_count) =INIT_PREEMPT_COUNT ; +EXPORT_PER_CPU_SYMBOL (__preempt_count); DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);#ifdef CONFIG_CC_STACKPROTECTOR
arch/x86/kernel/entry_32.S +2 -5
@@ -362,12 +362,9 @@ END(ret_from_exception) #ifdef CONFIG_PREEMPT ENTRY(resume_kernel) DISABLE_INTERRUPTS(CLBR_ANY) - cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? - jnz restore_all need_resched: - movl TI_flags(%ebp), %ecx # need_resched set ? - testb $_TIF_NEED_RESCHED, %cl - jz restore_all + cmpl $0,PER_CPU_VAR(__preempt_count) + jnz restore_all testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? jz restore_all call preempt_schedule_irq
arch/x86/kernel/entry_64.S +1 -3
@@ -1103,10 +1103,8 @@ ENTRY(native_iret) /* Returning to kernel space. Check if we need preemption */ /* rcx: threadinfo. interrupts off. */ ENTRY(retint_kernel) - cmpl $0,TI_preempt_count(%rcx) + cmpl $0,PER_CPU_VAR(__preempt_count) jnz retint_restore_args - bt $TIF_NEED_RESCHED,TI_flags(%rcx) - jnc retint_restore_args bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ jnc retint_restore_args call preempt_schedule_irq
arch/x86/kernel/process_32.c +10 -0
@@ -291,6 +291,16 @@ __switch_to(struct task_struct *prev_p,if (get_kernel_rpl () &&unlikely (prev->iopl != next->iopl))set_iopl_mask (next->iopl); +#ifdef CONFIG_PREEMPT_COUNT +/* +* If it were not for PREEMPT_ACTIVE we could guarantee that the +* preempt_count of all tasks was equal here and this would not be +* needed. +*/ +task_thread_info (prev_p)->saved_preempt_count =this_cpu_read (__preempt_count); +this_cpu_write (__preempt_count,task_thread_info (next_p)->saved_preempt_count); +#endif +/* * Now maybe handle debug registers and/or IO bitmaps */
arch/x86/kernel/process_64.c +10 -0
@@ -363,6 +363,16 @@ __switch_to(struct task_struct *prev_p,this_cpu_write (old_rsp, next->usersp);this_cpu_write (current_task, next_p); +#ifdef CONFIG_PREEMPT_COUNT +/* +* If it were not for PREEMPT_ACTIVE we could guarantee that the +* preempt_count of all tasks was equal here and this would not be +* needed. +*/ +task_thread_info (prev_p)->saved_preempt_count =this_cpu_read (__preempt_count); +this_cpu_write (__preempt_count,task_thread_info (next_p)->saved_preempt_count); +#endif +this_cpu_write (kernel_stack, (unsigned long )task_stack_page (next_p) +THREAD_SIZE -KERNEL_STACK_OFFSET );
Peter Zijlstra
maintainerPATCH 11/11 sched, x86: Optimize the preempt_schedule() call
In reply to Peter ZijlstraRemove the bloat of the C calling convention out of the preempt_enable() sites by creating an ASM wrapper which allows us to do an asm("call ___preempt_schedule") instead.
calling.h bits by Andi Kleen
--- arch/x86/include/asm/calling.h | 50 +++++++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/preempt.h | 8 ++++++ arch/x86/kernel/Makefile | 2 + arch/x86/kernel/preempt.S | 25 ++++++++++++++++++++ include/asm-generic/preempt.h | 10 ++++++++ include/linux/preempt.h | 13 ++++------ 6 files changed, 100 insertions(+), 8 deletions(-)
arch/x86/include/asm/calling.h +50 -0
@@ -48,6 +48,8 @@ For 32-bit we have the following convent#include <asm/dwarf2.h> +#ifdef CONFIG_X86_64 +/* * 64-bit system call stack frame layout defines and helpers, * for assembly code:
@@ -192,3 +194,51 @@ For 32-bit we have the following convent .macro icebp .byte 0xf1 .endm + +#else /* CONFIG_X86_64 */ + +/* +* For 32bit only simplified versions of SAVE_ALL/RESTORE_ALL. These +* are different from the entry_32.S versions in not changing the segment +* registers. So only suitable for in kernel use, not when transitioning +* from or to user space. The resulting stack frame is not a standard +* pt_regs frame. The main use case is calling C code from assembler +* when all the registers need to be preserved. +*/ + + .macro SAVE_ALL + pushl_cfi %eax + CFI_REL_OFFSET eax, 0 + pushl_cfi %ebp + CFI_REL_OFFSET ebp, 0 + pushl_cfi %edi + CFI_REL_OFFSET edi, 0 + pushl_cfi %esi + CFI_REL_OFFSET esi, 0 + pushl_cfi %edx + CFI_REL_OFFSET edx, 0 + pushl_cfi %ecx + CFI_REL_OFFSET ecx, 0 + pushl_cfi %ebx + CFI_REL_OFFSET ebx, 0 + .endm + + .macro RESTORE_ALL + popl_cfi %ebx + CFI_RESTORE ebx + popl_cfi %ecx + CFI_RESTORE ecx + popl_cfi %edx + CFI_RESTORE edx + popl_cfi %esi + CFI_RESTORE esi + popl_cfi %edi + CFI_RESTORE edi + popl_cfi %ebp + CFI_RESTORE ebp + popl_cfi %eax + CFI_RESTORE eax + .endm + +#endif /* CONFIG_X86_64 */ +
arch/x86/include/asm/preempt.h +8 -0
@@ -99,4 +99,12 @@ static __always_inline bool should_resch return unlikely(!__this_cpu_read_4(__preempt_count)); } +#ifdef CONFIG_PREEMPT +#define __preempt_schedule () asm ("call ___preempt_schedule") + +#ifdef CONFIG_CONTEXT_TRACKING +#define __preempt_schedule_context () asm ("call ___preempt_schedule_context") +#endif +#endif /* CONFIG_PREEMPT */ +#endif /* __ASM_PREEMPT_H */
arch/x86/kernel/Makefile +2 -0
@@ -36,6 +36,8 @@ obj-y += tsc.o io_delay.o rtc.o obj-y += pci-iommu_table.o obj-y += resource.o +obj-$(CONFIG_PREEMPT) += preempt.o + obj-y += process.o obj-y += i387.o xsave.o obj-y += ptrace.o
arch/x86/kernel/preempt.S new file +25 -0
@@ -0,0 +1,25 @@ + +#include <linux/linkage.h> +#include <asm/dwarf2.h> +#include <asm/asm.h> +#include <asm/calling.h> + +ENTRY(___preempt_schedule) + CFI_STARTPROC + SAVE_ALL + call preempt_schedule + RESTORE_ALL + ret + CFI_ENDPROC + +#ifdef CONFIG_CONTEXT_TRACKING + +ENTRY(___preempt_schedule_context) + CFI_STARTPROC + SAVE_ALL + call preempt_schedule_context + RESTORE_ALL + ret + CFI_ENDPROC + +#endif
include/asm-generic/preempt.h +10 -0
@@ -90,4 +90,14 @@ static __always_inline bool should_reschreturn unlikely (!*preempt_count_ptr ()); } +#ifdef CONFIG_PREEMPT +extern asmlinkage void preempt_schedule(void); +#define __preempt_schedule () preempt_schedule() + +#ifdef CONFIG_CONTEXT_TRACKING +extern asmlinkage void preempt_schedule_context(void); +#define __preempt_schedule_context () preempt_schedule_context() +#endif +#endif /* CONFIG_PREEMPT */ +#endif /* __ASM_PREEMPT_H */
include/linux/preempt.h +5 -8
@@ -50,18 +50,17 @@ do { \#define preempt_enable_no_resched () sched_preempt_enable_no_resched()#ifdef CONFIG_PREEMPT -asmlinkage void preempt_schedule(void);#define preempt_enable () \ do { \ barrier(); \ if (unlikely(preempt_count_dec_and_test())) \ - preempt_schedule(); \ + __preempt_schedule(); \ } while (0)#define preempt_check_resched () \ do { \ if (should_resched()) \ - preempt_schedule(); \ + __preempt_schedule(); \ } while (0)#else
@@ -83,17 +82,15 @@ do { \#ifdef CONFIG_PREEMPT -#ifdef CONFIG_CONTEXT_TRACKING -asmlinkage void preempt_schedule_context(void); -#else -#define preempt_schedule_context () preempt_schedule() +#ifndef CONFIG_CONTEXT_TRACKING +#define __preempt_schedule_context () __preempt_schedule()#endif #define preempt_enable_notrace () \ do { \ barrier(); \ if (unlikely(__preempt_count_dec_and_test())) \ - preempt_schedule_context(); \ + __preempt_schedule_context(); \ } while (0)#else #define preempt_enable_notrace () preempt_enable_no_resched_notrace()
Peter Zijlstra
maintainerRe: [PATCH 11/11] sched, x86: Optimize the preempt_schedule() call
Another build error by Thomas Meyer.. we need to export the asm symbols.
---
Subject: sched, x86: Optimize the preempt_schedule() call
From: Peter Zijlstra <[email protected]>
Date: Wed Aug 14 14:51:00 CEST 2013
Remove the bloat of the C calling convention out of the preempt_enable() sites by creating an ASM wrapper which allows us to do an asm("call ___preempt_schedule") instead.
calling.h bits by Andi Kleen
--- arch/x86/include/asm/calling.h | 50 +++++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/preempt.h | 8 ++++++ arch/x86/kernel/Makefile | 2 + arch/x86/kernel/i386_ksyms_32.c | 7 +++++ arch/x86/kernel/preempt.S | 25 +++++++++++++++++++ arch/x86/kernel/x8664_ksyms_64.c | 7 +++++ include/asm-generic/preempt.h | 10 +++++++ include/linux/preempt.h | 13 +++------- 8 files changed, 114 insertions(+), 8 deletions(-)
arch/x86/include/asm/calling.h +50 -0
@@ -48,6 +48,8 @@ For 32-bit we have the following convent#include <asm/dwarf2.h> +#ifdef CONFIG_X86_64 +/* * 64-bit system call stack frame layout defines and helpers, * for assembly code:
@@ -192,3 +194,51 @@ For 32-bit we have the following convent .macro icebp .byte 0xf1 .endm + +#else /* CONFIG_X86_64 */ + +/* +* For 32bit only simplified versions of SAVE_ALL/RESTORE_ALL. These +* are different from the entry_32.S versions in not changing the segment +* registers. So only suitable for in kernel use, not when transitioning +* from or to user space. The resulting stack frame is not a standard +* pt_regs frame. The main use case is calling C code from assembler +* when all the registers need to be preserved. +*/ + + .macro SAVE_ALL + pushl_cfi %eax + CFI_REL_OFFSET eax, 0 + pushl_cfi %ebp + CFI_REL_OFFSET ebp, 0 + pushl_cfi %edi + CFI_REL_OFFSET edi, 0 + pushl_cfi %esi + CFI_REL_OFFSET esi, 0 + pushl_cfi %edx + CFI_REL_OFFSET edx, 0 + pushl_cfi %ecx + CFI_REL_OFFSET ecx, 0 + pushl_cfi %ebx + CFI_REL_OFFSET ebx, 0 + .endm + + .macro RESTORE_ALL + popl_cfi %ebx + CFI_RESTORE ebx + popl_cfi %ecx + CFI_RESTORE ecx + popl_cfi %edx + CFI_RESTORE edx + popl_cfi %esi + CFI_RESTORE esi + popl_cfi %edi + CFI_RESTORE edi + popl_cfi %ebp + CFI_RESTORE ebp + popl_cfi %eax + CFI_RESTORE eax + .endm + +#endif /* CONFIG_X86_64 */ +
arch/x86/include/asm/preempt.h +8 -0
@@ -95,4 +95,12 @@ static __always_inline bool should_resch return unlikely(!__this_cpu_read_4(__preempt_count)); } +#ifdef CONFIG_PREEMPT +#define __preempt_schedule () asm ("call ___preempt_schedule") + +#ifdef CONFIG_CONTEXT_TRACKING +#define __preempt_schedule_context () asm ("call ___preempt_schedule_context") +#endif +#endif /* CONFIG_PREEMPT */ +#endif /* __ASM_PREEMPT_H */
arch/x86/kernel/Makefile +2 -0
@@ -36,6 +36,8 @@ obj-y += tsc.o io_delay.o rtc.o obj-y += pci-iommu_table.o obj-y += resource.o +obj-$(CONFIG_PREEMPT) += preempt.o + obj-y += process.o obj-y += i387.o xsave.o obj-y += ptrace.o
arch/x86/kernel/i386_ksyms_32.c +7 -0
@@ -37,3 +37,10 @@ EXPORT_SYMBOL(strstr);EXPORT_SYMBOL (csum_partial);EXPORT_SYMBOL (empty_zero_page); + +#ifdef CONFIG_PREEMPT +EXPORT_SYMBOL (___preempt_schedule); +#ifdef CONFIG_CONTEXT_TRACKING +EXPORT_SYMBOL (___preempt_schedule_context); +#endif +#endif
arch/x86/kernel/preempt.S new file +25 -0
@@ -0,0 +1,25 @@ + +#include <linux/linkage.h> +#include <asm/dwarf2.h> +#include <asm/asm.h> +#include <asm/calling.h> + +ENTRY(___preempt_schedule) + CFI_STARTPROC + SAVE_ALL + call preempt_schedule + RESTORE_ALL + ret + CFI_ENDPROC + +#ifdef CONFIG_CONTEXT_TRACKING + +ENTRY(___preempt_schedule_context) + CFI_STARTPROC + SAVE_ALL + call preempt_schedule_context + RESTORE_ALL + ret + CFI_ENDPROC + +#endif
arch/x86/kernel/x8664_ksyms_64.c +7 -0
@@ -66,3 +66,10 @@ EXPORT_SYMBOL(empty_zero_page);#ifndef CONFIG_PARAVIRT EXPORT_SYMBOL (native_load_gs_index);#endif + +#ifdef CONFIG_PREEMPT +EXPORT_SYMBOL (___preempt_schedule); +#ifdef CONFIG_CONTEXT_TRACKING +EXPORT_SYMBOL (___preempt_schedule_context); +#endif +#endif
include/asm-generic/preempt.h +10 -0
@@ -90,4 +90,14 @@ static __always_inline bool should_reschreturn unlikely (!*preempt_count_ptr ()); } +#ifdef CONFIG_PREEMPT +extern asmlinkage void preempt_schedule(void); +#define __preempt_schedule () preempt_schedule() + +#ifdef CONFIG_CONTEXT_TRACKING +extern asmlinkage void preempt_schedule_context(void); +#define __preempt_schedule_context () preempt_schedule_context() +#endif +#endif /* CONFIG_PREEMPT */ +#endif /* __ASM_PREEMPT_H */
include/linux/preempt.h +5 -8
@@ -50,18 +50,17 @@ do { \#define preempt_enable_no_resched () sched_preempt_enable_no_resched()#ifdef CONFIG_PREEMPT -asmlinkage void preempt_schedule(void);#define preempt_enable () \ do { \ barrier(); \ if (unlikely(preempt_count_dec_and_test())) \ - preempt_schedule(); \ + __preempt_schedule(); \ } while (0)#define preempt_check_resched () \ do { \ if (should_resched()) \ - preempt_schedule(); \ + __preempt_schedule(); \ } while (0)#else
@@ -83,17 +82,15 @@ do { \#ifdef CONFIG_PREEMPT -#ifdef CONFIG_CONTEXT_TRACKING -asmlinkage void preempt_schedule_context(void); -#else -#define preempt_schedule_context () preempt_schedule() +#ifndef CONFIG_CONTEXT_TRACKING +#define __preempt_schedule_context () __preempt_schedule()#endif #define preempt_enable_notrace () \ do { \ barrier(); \ if (unlikely(__preempt_count_dec_and_test())) \ - preempt_schedule_context(); \ + __preempt_schedule_context(); \ } while (0)#else #define preempt_enable_notrace () preempt_enable_no_resched_notrace()
Quoted 11 lines · Peter Zijlstra
These patches optimize
preempt_enableby firstly folding the preempt andneed_reschedtests into one -- this should work for all architectures. And secondly by providing per-archpreempt_countimplementations; with x86 using per-cpupreempt_countfor fastest access.These patches have been boot tested on
CONFIG_PREEMPT=yx86_64and survive building a x86_64-defconfig kernel.text data bss filename
11387014 1454776 1187840 defconfig-build/vmlinux.before 11352294 1454776 1187840 defconfig-build/vmlinux.after
That's a 0.3% size improvement (and most of the improvement is in hotpaths), despite GCC is being somewhat stupid about not allowing us to mark asm goto targets as cold paths and thus causes some unnecessary register shuffling in some cases, right?
Not bad IMO.
Thanks,
Ingo
Peter Zijlstra
maintainerQuoted 11 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
These patches optimize
preempt_enableby firstly folding the preempt andneed_reschedtests into one -- this should work for all architectures. And secondly by providing per-archpreempt_countimplementations; with x86 using per-cpupreempt_countfor fastest access.These patches have been boot tested on
CONFIG_PREEMPT=yx86_64and survive building a x86_64-defconfig kernel.
Quoted 8 lines
Quoted 3 lines
text data bss filename
11387014 1454776 1187840 defconfig-build/vmlinux.before 11352294 1454776 1187840 defconfig-build/vmlinux.afterThat's a 0.3% size improvement (and most of the improvement is in hotpaths), despite GCC is being somewhat stupid about not allowing us to mark asm goto targets as cold paths and thus causes some unnecessary register shuffling in some cases, right?
I'm not entire sure where the bloat in 1/11 comes from; several functions look like they avoid using stack variables for using more registers which create more push/pop on entry/exit paths. Others I'm not entirely sure of what happens with.
But it does look like the unlikely() thing still works, even with the asm goto, you'll note that the call to schedule_preempt is out-of-line.
cover Make all preempt_count related constants generic
In reply to Peter ZijlstraThere is no point in having PREEMPT_ACTIVE and HARDIRQ_BITS defined by an architecture.
The PREEMPT_ACTIVE bit manipulation in the low level entry code is historical and has been replaced by preempt_schedule_irq().
The HARDIRQ_BITS constant could be 1 in theory because we run all interrupt handlers with interrupts disabled nowadays. Though we have some palaeontolical drivers (IDE) which still enable interrupts in the interrupt handler, so we need to allow at least some limited nesting.
This series converts the remaining low level PREEMPT_ACTIVE users to utilize preempt_schedule_irq() and moves the PREEMPT_ACTIVE and HARDIRQ_BITS defines to the core code.
I stumbled over this historical left overs while trying to implement a lazy interrupt disable scheme (which still has some rough edges due to completely braindamaged hardware).
Peter, it would be nice if you could pick that up into your preempt/need_resched series.
Thanks,
tglx
--- arch/alpha/include/asm/thread_info.h | 2 - arch/arc/include/asm/thread_info.h | 2 - arch/arm/include/asm/thread_info.h | 6 ---- arch/arm64/include/asm/thread_info.h | 6 ---- arch/avr32/include/asm/thread_info.h | 2 - arch/blackfin/include/asm/hardirq.h | 3 -- arch/blackfin/include/asm/thread_info.h | 2 - arch/c6x/include/asm/thread_info.h | 2 - arch/cris/include/asm/hardirq.h | 12 -------- arch/cris/include/asm/thread_info.h | 2 - arch/frv/include/asm/thread_info.h | 2 - arch/h8300/include/asm/hardirq.h | 12 -------- arch/h8300/include/asm/thread_info.h | 2 - arch/h8300/kernel/entry.S | 6 ---- arch/hexagon/include/asm/thread_info.h | 4 -- arch/ia64/include/asm/thread_info.h | 3 -- arch/ia64/kernel/entry.S | 15 ---------- arch/m32r/include/asm/hardirq.h | 16 ----------- arch/m32r/include/asm/thread_info.h | 2 - arch/m32r/kernel/entry.S | 8 ----- arch/m68k/include/asm/hardirq.h | 11 -------- arch/m68k/include/asm/thread_info.h | 2 - arch/metag/include/asm/thread_info.h | 2 - arch/microblaze/include/asm/thread_info.h | 2 - arch/mips/include/asm/thread_info.h | 2 - arch/mn10300/include/asm/thread_info.h | 2 - arch/parisc/include/asm/thread_info.h | 3 -- arch/powerpc/include/asm/thread_info.h | 2 - arch/s390/include/asm/hardirq.h | 2 - arch/s390/include/asm/thread_info.h | 2 - arch/score/include/asm/thread_info.h | 2 - arch/sh/include/asm/thread_info.h | 2 - arch/sh/kernel/entry-common.S | 6 +--- arch/sparc/include/asm/hardirq_32.h | 1 arch/sparc/include/asm/hardirq_64.h | 2 - arch/sparc/include/asm/thread_info_32.h | 2 - arch/sparc/include/asm/thread_info_64.h | 2 - arch/sparc/kernel/rtrap_64.S | 5 --- arch/tile/include/asm/hardirq.h | 2 - arch/tile/include/asm/thread_info.h | 2 - arch/um/include/asm/thread_info.h | 2 - arch/unicore32/include/asm/thread_info.h | 6 ---- arch/x86/include/asm/thread_info.h | 2 - arch/xtensa/include/asm/thread_info.h | 2 - include/linux/preempt_mask.h | 41 ++++++++---------------------- 45 files changed, 17 insertions(+), 201 deletions(-)
PATCH 1/6 hardirq: Make hardirq bits generic
There is no reason for per arch hardirq bits. Make them all generic
--- arch/blackfin/include/asm/hardirq.h | 3 --- arch/cris/include/asm/hardirq.h | 12 ------------ arch/h8300/include/asm/hardirq.h | 12 ------------ arch/m32r/include/asm/hardirq.h | 16 ---------------- arch/m68k/include/asm/hardirq.h | 11 ----------- arch/s390/include/asm/hardirq.h | 2 -- arch/sparc/include/asm/hardirq_32.h | 1 - arch/sparc/include/asm/hardirq_64.h | 2 -- arch/tile/include/asm/hardirq.h | 2 -- include/linux/preempt_mask.h | 30 ++++++++---------------------- 10 files changed, 8 insertions(+), 83 deletions(-)
arch/blackfin/include/asm/hardirq.h +0 -3
@@ -12,9 +12,6 @@extern void ack_bad_irq (unsigned int irq);#define ack_bad_irq ack_bad_irq -/* Define until common code gets sane defaults */ -#define HARDIRQ_BITS 9 -#include <asm-generic/hardirq.h> #endif
arch/cris/include/asm/hardirq.h +0 -12
@@ -2,18 +2,6 @@#define __ASM_HARDIRQ_H#include <asm/irq.h> - -#define HARDIRQ_BITS 8 - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#include <asm-generic/hardirq.h> #endif /* __ASM_HARDIRQ_H */
arch/h8300/include/asm/hardirq.h +0 -12
@@ -2,18 +2,6 @@#define __H8300_HARDIRQ_H#include <asm/irq.h> - -#define HARDIRQ_BITS 8 - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#include <asm-generic/hardirq.h> #endif
arch/m32r/include/asm/hardirq.h +0 -16
@@ -3,22 +3,6 @@#define __ASM_HARDIRQ_H#include <asm/irq.h> - -#if NR_IRQS >256 -#define HARDIRQ_BITS 9 -#else -#define HARDIRQ_BITS 8 -#endif - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#include <asm-generic/hardirq.h> #endif /* __ASM_HARDIRQ_H */
arch/m68k/include/asm/hardirq.h +0 -11
@@ -5,17 +5,6 @@#include <linux/cache.h> #include <asm/irq.h> -#define HARDIRQ_BITS 8 - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#ifdef CONFIG_MMU static inline void ack_bad_irq (unsigned int irq)
arch/s390/include/asm/hardirq.h +0 -2
@@ -18,8 +18,6 @@#define __ARCH_HAS_DO_SOFTIRQ#define __ARCH_IRQ_EXIT_IRQS_DISABLED -#define HARDIRQ_BITS 8 -static inline void ack_bad_irq (unsigned int irq) {printk (KERN_CRIT "unexpected IRQ trap at vector %02x\n" , irq);
arch/sparc/include/asm/hardirq_32.h +0 -1
@@ -7,7 +7,6 @@#ifndef __SPARC_HARDIRQ_H#define __SPARC_HARDIRQ_H -#define HARDIRQ_BITS 8#include <asm-generic/hardirq.h> #endif /* __SPARC_HARDIRQ_H */
arch/sparc/include/asm/hardirq_64.h +0 -2
@@ -14,6 +14,4 @@void ack_bad_irq (unsigned int irq); -#define HARDIRQ_BITS 8 -#endif /* !(__SPARC64_HARDIRQ_H) */
arch/tile/include/asm/hardirq.h +0 -2
@@ -42,6 +42,4 @@ DECLARE_PER_CPU(irq_cpustat_t, irq_stat)#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ -#define HARDIRQ_BITS 8 -#endif /* _ASM_TILE_HARDIRQ_H */
include/linux/preempt_mask.h +8 -22
@@ -11,36 +11,22 @@* - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) * -* The hardirq count can in theory reach the same as NR_IRQS. -* In reality, the number of nested IRQS is limited to the stack -* size as well. For archs with over 1000 IRQS it is not practical -* to expect that they will all nest. We give a max of 10 bits for -* hardirq nesting. An arch may choose to give less than 10 bits. -* m68k expects it to be 8. -* -* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024) -* - bit 26 is the NMI_MASK -* - bit 27 is the PREEMPT_ACTIVE flag +* The hardirq count could in theory be the same as the number of +* interrupts in the system, but we run all interrupt handlers with +* interrupts disabled, so we cannot have nesting interrupts. Though +* there are a few palaeontologic drivers which reenable interrupts in +* the handler, so we need more than one bit here. * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x03ff0000 -* NMI_MASK: 0x04000000 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 +#define HARDIRQ_BITS 4#define NMI_BITS 1 -#define MAX_HARDIRQ_BITS 10 - -#ifndef HARDIRQ_BITS -# define HARDIRQ_BITS MAX_HARDIRQ_BITS -#endif - -#if HARDIRQ_BITS >MAX_HARDIRQ_BITS -#error HARDIRQ_BITS too high! -#endif -#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 11 lines · Thomas Gleixner
arch/m68k/include/asm/hardirq.h +0
-1@@ -5,17 +5,6 @@#include <linux/cache.h> #include <asm/irq.h> -#define HARDIRQ_BITS 8
include/linux/preempt_mask.h line 19
Quoted 9 lines
include/linux/preempt_mask.h +0
-6@@ -11,36 +11,22 @@* - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) * -* The hardirq count can in theory reach the same as NR_IRQS. -* In reality, the number of nested IRQS is limited to the stack -* size as well. For archs with over 1000 IRQS it is not practical -* to expect that they will all nest. We give a max of 10 bits for -* hardirq nesting. An arch may choose to give less than 10 bits. -* m68k expects it to be 8.
m68k needs some changes in arch/m68k/kernel/entry.S, cfr. this check in arch/m68k/kernel/ints.c:
/* assembly irq entry code relies on this... */
if (HARDIRQ_MASK != 0x00ff0000) {
extern void hardirq_mask_is_broken(void);
hardirq_mask_is_broken();
}Haven't looked into the details yet...
Quoted 10 lines
include/linux/preempt_mask.h +5
-3-* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024) -* - bit 26 is the NMI_MASK -* - bit 27 is the PREEMPT_ACTIVE flag +* The hardirq count could in theory be the same as the number of +* interrupts in the system, but we run all interrupt handlers with +* interrupts disabled, so we cannot have nesting interrupts. Though +* there are a few palaeontologic drivers which reenable interrupts in +* the handler, so we need more than one bit here. * * PREEMPT_MASK: 0x000000ff
include/linux/preempt_mask.h line 28
Quoted 10 lines
include/linux/preempt_mask.h +3
-2* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x03ff0000 -* NMI_MASK: 0x04000000 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 +#define HARDIRQ_BITS 4#define NMI_BITS 1
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 25 lines · Geert Uytterhoeven
Quoted 11 lines · Thomas Gleixner
arch/m68k/include/asm/hardirq.h +0
-1@@ -5,17 +5,6 @@#include <linux/cache.h> #include <asm/irq.h> -#define HARDIRQ_BITS 8include/linux/preempt_mask.h line 19
Quoted 9 lines
include/linux/preempt_mask.h +0
-6@@ -11,36 +11,22 @@* - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) * -* The hardirq count can in theory reach the same as NR_IRQS. -* In reality, the number of nested IRQS is limited to the stack -* size as well. For archs with over 1000 IRQS it is not practical -* to expect that they will all nest. We give a max of 10 bits for -* hardirq nesting. An arch may choose to give less than 10 bits. -* m68k expects it to be 8. m68k needs some changes in
arch/m68k/kernel/entry.S, cfr. this check inarch/m68k/kernel/ints.c:
Quoted 7 lines
/* assembly irq entry code relies on this... */ if (HARDIRQ_MASK != 0x00ff0000) { extern void hardirq_mask_is_broken(void); hardirq_mask_is_broken(); }Haven't looked into the details yet...
Whee. Did not notice that one. Though I can't find anything interesting in the low level entry code... Looks like some more histerical left overs.
Thanks,
tglx
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 33 lines · Thomas Gleixner
Quoted 31 lines · Geert Uytterhoeven
Quoted 11 lines · Thomas Gleixner
arch/m68k/include/asm/hardirq.h +0
-1@@ -5,17 +5,6 @@#include <linux/cache.h> #include <asm/irq.h> -#define HARDIRQ_BITS 8include/linux/preempt_mask.h line 19
Quoted 9 lines
include/linux/preempt_mask.h +0
-6@@ -11,36 +11,22 @@* - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) * -* The hardirq count can in theory reach the same as NR_IRQS. -* In reality, the number of nested IRQS is limited to the stack -* size as well. For archs with over 1000 IRQS it is not practical -* to expect that they will all nest. We give a max of 10 bits for -* hardirq nesting. An arch may choose to give less than 10 bits. -* m68k expects it to be 8. m68k needs some changes in
arch/m68k/kernel/entry.S, cfr. this check inarch/m68k/kernel/ints.c:/* assembly irq entry code relies on this... */ if (HARDIRQ_MASK != 0x00ff0000) { extern void hardirq_mask_is_broken(void); hardirq_mask_is_broken(); }
Quoted 5 lines
Quoted 1 line
Haven't looked into the details yet...
Whee. Did not notice that one. Though I can't find anything interesting in the low level entry code... Looks like some more histerical left overs.
Duh. With brain awake I can see it.
The low level entry code is fiddling with preempt_count by adding HARDIRQ_OFFSET to it to keep track of nested interrupts. If the count goes to 0, it invokes do_softirq(). And you have another safety guard:
ret_from_last_interrupt:
moveq #(~ALLOWINT>>8)&0xff,%d0 andb %sp@(PT_OFF_SR),%d0 jne 2b
That's due to the irq priority level stuff, which results in nested interrupts depending on the level of the serviced interrupt, right? And that's why you fiddle yourself with the HARDIRQ bits in the preempt count to prevent the core code from calling do_softirq().
Though this scheme also prevents that other parts of irq_exit() are working correctly, because they depend on the hardirq count being zero, e.g. the nohz code.
Needs more thoughts how to fix that w/o wasting precious bits for the HARDIRQ count.
Thanks,
tglx
Re: [patch 1/6] hardirq: Make hardirq bits generic
Geert,
Quoted 14 lines · Thomas Gleixner
The low level entry code is fiddling with
preempt_countby addingHARDIRQ_OFFSETto it to keep track of nested interrupts. If the count goes to 0, it invokesdo_softirq(). And you have another safety guard:
ret_from_last_interrupt:moveq #(~ALLOWINT>>8)&0xff,%d0 andb %sp@(PT_OFF_SR),%d0 jne 2bThat's due to the irq priority level stuff, which results in nested interrupts depending on the level of the serviced interrupt, right? And that's why you fiddle yourself with the HARDIRQ bits in the preempt count to prevent the core code from calling
do_softirq().
Quoted 6 lines
Though this scheme also prevents that other parts of
irq_exit()are working correctly, because they depend on the hardirq count being zero, e.g. the nohz code.Needs more thoughts how to fix that w/o wasting precious bits for the HARDIRQ count.
So after staring a while into the m68k code I came up with the following (untested and uncompiled) solution:
Instead of doing the HARDIRQ fiddling and the softirq handling from the low level entry code, I provided an irq_exit_nested() variant which has an argument to tell the core code, that it shouldn't invoke softirq handling and the nohz exit code. That way 4 HARDIRQ bits in preempt_count() should be sufficient and we can move them around as we see fit without being bound by some magic asm code fiddling with it.
By modifying do_IRQ to return an indicator whether this is the last irq in the chain, we can also simplify the asm code significantly.
Can you have a look with m68k eyes on that please? I'm sure that my vague memory of the 68k ASM tricked me into doing something fundamentally wrong :)
Thanks,
tglx
---
Subject: m68k: Deal with interrupt nesting in the core code
From: Thomas Gleixner <[email protected]>
Date: Wed, 18 Sep 2013 11:56:58 +0200
m68k increments the HARDIRQ part of preempt_count in the low level interrupt entry code, which prevents the core code from restructuring the preempt_count layout.
This is done to prevent softirq invocation for nested interrupts. The nesting can happen due to the interrupt priority scheme of the 68k.
This patch removes the low level handling and moves it to the interrupt core code by providing an irq_exit_nested() variant. The nested argument to this function tells the core code whether to invoke softirqs or not.
Also let do_IRQ() and the other handler variants return the nest indicator so the low level entry code can utilize this to select either immediate return or the expensive work functions.
--- arch/m68k/include/asm/irq.h | 2 +- arch/m68k/kernel/entry.S | 36 +++++++++--------------------------- arch/m68k/kernel/ints.c | 6 ------ arch/m68k/kernel/irq.c | 7 ++++--- arch/m68k/platform/68000/entry.S | 19 ++++++------------- arch/m68k/platform/68000/ints.c | 7 +++---- arch/m68k/platform/68360/entry.S | 25 ++++++++----------------- arch/m68k/q40/q40ints.c | 12 ++++++------ include/linux/hardirq.h | 1 + kernel/softirq.c | 15 +++++++++++---- 10 files changed, 49 insertions(+), 81 deletions(-)
arch/m68k/include/asm/irq.h +1 -1
@@ -74,7 +74,7 @@ extern unsigned int irq_canonicalize(uns#define irq_canonicalize (irq) (irq)#endif /* !(CONFIG_M68020 || CONFIG_M68030 || CONFIG_M68040 || CONFIG_M68060) */ -asmlinkage void do_IRQ(int irq, struct pt_regs *regs); +asmlinkage int do_IRQ(int irq, struct pt_regs *regs);extern atomic_t irq_err_count;#endif /* _M68K_IRQ_H_ */
arch/m68k/kernel/entry.S +9 -27
@@ -274,9 +274,6 @@ do_delayed_trace: ENTRY(auto_inthandler) SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0
@@ -284,34 +281,22 @@ ENTRY(auto_inthandler) movel %sp,%sp@- movel %d0,%sp@- | put vector # on stack auto_irqhandler_fixup = . + 2 - jsr do_IRQ | process the IRQ + jsr do_IRQ | process the IRQ, returns nest level addql #8,%sp | pop parameters off stack ret_from_interrupt: - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt -2: RESTORE_ALL - - ALIGN -ret_from_last_interrupt: - moveq #(~ALLOWINT>>8)&0xff,%d0 - andb %sp@(PT_OFF_SR),%d0 - jne 2b - - /* check if we need to do software interrupts */ - tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING + /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0 jeq .Lret_from_exception - pea ret_from_exception - jra do_softirq + RESTORE_ALL /* Handler for user defined interrupt vectors */ ENTRY(user_inthandler) SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 user_irqvec_fixup = . + 2
@@ -319,13 +304,10 @@ user_irqvec_fixup = . + 2 movel %sp,%sp@- movel %d0,%sp@- | put vector # on stack - jsr do_IRQ | process the IRQ + jsr do_IRQ | process the IRQ, returns nest level addql #8,%sp | pop parameters off stack - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL + jra ret_from_interrupt /* Handler for uninitialized and spurious interrupts */
arch/m68k/kernel/ints.c +0 -6
@@ -58,12 +58,6 @@ void __init init_IRQ(void) {int i; -/* assembly irq entry code relies on this... */ -if (HARDIRQ_MASK !=0x00ff0000 ) { -extern void hardirq_mask_is_broken (void ); -hardirq_mask_is_broken (); - } -for (i =IRQ_AUTO_1 ; i <=IRQ_AUTO_7 ; i++)irq_set_chip_and_handler (i, &auto_irq_chip, handle_simple_irq);
arch/m68k/kernel/irq.c +4 -3
@@ -17,18 +17,19 @@#include <linux/seq_file.h> #include <asm/traps.h> -asmlinkage void do_IRQ(int irq, struct pt_regs *regs) +asmlinkage int do_IRQ(int irq, struct pt_regs *regs) {struct pt_regs *oldregs =set_irq_regs (regs); +int nested = regs->sr & ~ALLOWINT ;irq_enter ();generic_handle_irq (irq); -irq_exit (); +irq_exit_nested (nested);set_irq_regs (oldregs); +return nested; } -/* The number of spurious interrupts */ atomic_t irq_err_count;
arch/m68k/platform/68000/entry.S +6 -13
@@ -217,20 +217,13 @@ inthandler: bra ret_from_interrupt ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - - /* check if we need to do software interrupts */ + /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0 jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + RESTORE_ALL /* * Handler for uninitialized and spurious interrupts.
arch/m68k/platform/68000/ints.c +3 -4
@@ -74,11 +74,9 @@ asmlinkage irqreturn_t inthandler7(void)* into one vector and look in the blasted mask register... * This code is designed to be fast, almost constant time, not clean! */ -void process_int (int vec,struct pt_regs *fp) +int process_int (int vec,struct pt_regs *fp) { -int irq; -int mask; - +int irq, mask, nested =fp->sr & ~ALLOWINT ;unsigned long pend =ISR ;while (pend) {
@@ -128,6 +126,7 @@ void process_int(int vec, struct pt_regsdo_IRQ (irq, fp); pend &= ~mask; } +return nested; }static void intc_irq_unmask (struct irq_data *d)
arch/m68k/platform/68360/entry.S +8 -17
@@ -132,26 +132,17 @@ inthandler: movel %sp,%sp@- movel %d0,%sp@- /* put vector # on stack*/ - jbsr do_IRQ /* process the IRQ*/ -3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + jbsr do_IRQ /* process the IRQ, returns nest level */ + addql #8,%sp /* pop parameters off stack*/ ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - /* check if we need to do software interrupts */ - - movel irq_stat+CPUSTAT_SOFTIRQ_PENDING,%d0 + /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0 jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + RESTORE_ALL /* * Handler for uninitialized and spurious interrupts.
arch/m68k/q40/q40ints.c +6 -6
@@ -202,10 +202,10 @@ static int aliased_irq=0; /* how many t/* got interrupt, dispatch to ISA or keyboard/timer IRQs */ -static void q40_irq_handler (unsigned int irq,struct pt_regs *fp) +static int q40_irq_handler (unsigned int irq,struct pt_regs *fp) {unsigned mir, mer; -int i; +int i, nested = fp->sr & ~ALLOWINT ;//repeat: mir =master_inb (IIRQ_REG );
@@ -213,14 +213,14 @@ static void q40_irq_handler(unsigned intif ((mir &Q40_IRQ_EXT_MASK ) && (master_inb (EIRQ_REG ) &Q40_IRQ6_MASK )) {floppy_hardint (); -return ; +return nested; }#endif switch (irq) {case 4 :case 6 :do_IRQ (Q40_IRQ_SAMPLE , fp); -return ; +return nested; }if (mir &Q40_IRQ_FRAME_MASK ) {do_IRQ (Q40_IRQ_FRAME , fp);
@@ -277,7 +277,7 @@ static void q40_irq_handler(unsigned int#endif }// used to do 'goto repeat;' here, this delayed bh processing too long -return ; +return nested; } }if (mer && ccleirq >0 && !aliased_irq) {
@@ -291,7 +291,7 @@ static void q40_irq_handler(unsigned intif (mir &Q40_IRQ_KEYB_MASK )do_IRQ (Q40_IRQ_KEYBOARD , fp); -return ; +return nested; }void q40_irq_enable (struct irq_data *data)
include/linux/hardirq.h +1 -0
@@ -55,6 +55,7 @@ extern void irq_enter(void);/* * Exit irq context and process softirqs if needed: */ +extern void irq_exit_nested (bool nested);extern void irq_exit (void );#define nmi_enter () \
kernel/softirq.c +11 -4
@@ -350,7 +350,7 @@ static inline void tick_irq_exit(void)/* * Exit an interrupt context. Process softirqs if needed and possible: */ -void irq_exit (void ) +void irq_exit_nested (bool nested) {#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLEDlocal_irq_disable ();
@@ -361,13 +361,20 @@ void irq_exit(void)account_irq_exit_time (current);trace_hardirq_exit ();sub_preempt_count (HARDIRQ_OFFSET ); -if (!in_interrupt () &&local_softirq_pending ()) -invoke_softirq (); -tick_irq_exit (); +if (!nested) { +if (!in_interrupt () &&local_softirq_pending ()) +invoke_softirq (); +tick_irq_exit (); + }rcu_irq_exit (); } +void irq_exit (void ) +{ +irq_exit_nested (false); +} +/* * This function must run with irqs disabled! */
Re: [patch 1/6] hardirq: Make hardirq bits generic
arch/m68k/kernel/entry.S ENTRY(auto_inthandler) · line 292
Quoted 5 lines · Thomas Gleixner
arch/m68k/kernel/entry.S +5
-0+ /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0
tstl %d0
Andreas.
Signature · 3 lines
Andreas Schwab, [email protected] GPG Key fingerprint = 58CA 54C7 6D53 942B 1756 01D3 44D5 214B 8276 4ED5 "And now for something completely different."
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 7 lines · Andreas Schwab
Review
Quoted 5 lines · Thomas Gleixner
(quoted diff) +5
-0+ /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0tstl %d0
arch/m68k/kernel/built-in.o: In function `bad_inthandler':
(.text+0x2a6): undefined reference to `ret_from_last_interrupt'
I came up with the quick (whitespace-damaged-gmail) fix below. Or should we handle the nesting in handle_badint(), too?
arch/m68k/kernel/entry.S +0 -6
@@ -313,17 +313,11 @@ user_irqvec_fixup = . + 2 ENTRY(bad_inthandler) SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt RESTORE_ALL
However, the resulting kernel hangs (on ARAnyM) after starting userspace:
| INIT: version 2.86 booting
I'll have a deeper look when I have some more time...
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 7 lines · Geert Uytterhoeven
Quoted 6 lines · Andreas Schwab
Quoted 5 lines · Thomas Gleixner
(quoted diff) +5
-0+ /* + * Only the last interrupt leaving the kernel goes through the + * various exception return checks. + */ + cmpl #0, %d0
Quoted 7 lines
Quoted 1 line
tstl %d0
arch/m68k/kernel/built-in.o: In function `bad_inthandler':
(.text+0x2a6): undefined reference to `ret_from_last_interrupt'I came up with the quick (whitespace-damaged-gmail) fix below. Or should we handle the nesting in
handle_badint(), too?
Hmm, probably yes. If a badint gets interrupted by a good one, you would fail to go through ret_from_exception.
Quoted 21 lines
arch/m68k/kernel/entry.S +0
-6@@ -313,17 +313,11 @@ user_irqvec_fixup = . + 2 ENTRY(bad_inthandler) SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt RESTORE_ALLHowever, the resulting kernel hangs (on ARAnyM) after starting userspace:
Quoted 1 line
| INIT: version 2.86 booting
Hmm.
Quoted 1 line
I'll have a deeper look when I have some more time...
Thanks a lot!
tglx
Re: [patch 1/6] hardirq: Make hardirq bits generic
In reply to Geert UytterhoevenQuoted 5 lines · Geert Uytterhoeven
However, the resulting kernel hangs (on ARAnyM) after starting userspace:
| INIT: version 2.86 booting
I'll have a deeper look when I have some more time...
Any chance that you find some more time? :)
Thanks,
tglx
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
Hi Thomas,
Quoted 8 lines · Thomas Gleixner
Quoted 5 lines · Geert Uytterhoeven
However, the resulting kernel hangs (on ARAnyM) after starting userspace:
| INIT: version 2.86 booting
I'll have a deeper look when I have some more time...
Any chance that you find some more time? :)
Sure!
But only if you look at "[m68k] IRQ: add handle_polled_irq() for timer based soft interrupt" (http://www.spinics.net/lists/linux-m68k/msg05889.html)
first ;-)
Below is a patch with some fixups, on top of your two patches.
Unfortunately it still hangs somewhere after mounting the root filesystem.
Using this debug code for do_IRQ():
arch/m68k/kernel/irq.c +10 -0
@@ -22,11 +22,21 @@ asmlinkage int do_IRQ(int irq, struct pt_regs *regs)struct pt_regs *oldregs =set_irq_regs (regs);int nested = regs->sr & ~ALLOWINT ; +static int nesting; +const char prefix[] =" " ; +unsigned long flags; +local_irq_save (flags); +nesting++; +printk ("# %sirq %d nested %d\n" , &prefix[16 -2 *nesting], irq, nested); +local_irq_restore (flags);irq_enter ();generic_handle_irq (irq);irq_exit_nested (nested);set_irq_regs (oldregs); +local_irq_save (flags); +nesting--; +local_irq_restore (flags);return nested; }
I get output like
# irq 15 nested 0
# irq 15 nested 1024
irq 15 while irq 15 in progress??
# irq 15 nested 1024 # irq 15 nested 1024 # irq 15 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 4 nested 0
# irq 13 nested 1024
# irq 4 nested 0
# irq 13 nested 1024
# irq 4 nested 0
# irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 4 nested 0 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024 # irq 13 nested 1024
# irq 4 nested 0
# irq 13 nested 1024
# irq 13 nested 1024
[...]
# irq 13 nested 1024 # irq 13 nested 1024 # irq 4 nested 0
# irq 13 nested 1024
# irq 4 nested 0
irq 4 while irq 4 in progress?
# irq 13 nested 1024 # irq 4 nested 0 # irq 13 nested 0
and then it stops printing anything.
With similar debug code on the old working do_IRQ(), I get
- slightly less deep nesting,
- do_IRQ() is never re-entered with the same irq number.
Also note that the value of "nested" doesn't match the indentation level, which depends on my own bookkeeping using "nesting".
Anyone with an idea where it's going wrong?
Thanks!
From 209b6ac37811297cd305821c5689dff75226af48 Mon Sep 17 00:00:00 2001
From: Geert Uytterhoeven <[email protected]>
Date: Sun, 22 Sep 2013 11:31:25 +0200
Subject: [PATCH] m68k/hardirq: Make hardirq bits generic fixups
- tstl instead of cmpl #0 (from Andreas)
- arch/m68k/kernel/built-in.o: In function bad_inthandler': (.text+0x2a6): undefined reference to ret_from_last_interrupt'
- Handle nesting in bad_inthandler() and handle_badint(),
- As do_IRQ() now returns int, m68k_setup_auto_interrupt() should take a
function that returns int, too,
- Forgot to update forward declaration of q40_irq_handler(),
- Whitespace fixes
--- arch/m68k/include/asm/irq.h | 4 ++-- arch/m68k/kernel/entry.S | 10 ++-------- arch/m68k/kernel/ints.c | 9 +++++++-- arch/m68k/platform/68000/entry.S | 2 +- arch/m68k/platform/68000/ints.c | 2 +- arch/m68k/platform/68360/entry.S | 4 ++-- arch/m68k/q40/q40ints.c | 2 +- 7 files changed, 16 insertions(+), 17 deletions(-)
arch/m68k/include/asm/irq.h +2 -2
@@ -60,8 +60,8 @@ struct irq_desc;extern unsigned int m68k_irq_startup (struct irq_data *data);extern unsigned int m68k_irq_startup_irq (unsigned int irq);extern void m68k_irq_shutdown (struct irq_data *data); -extern void m68k_setup_auto_interrupt (void (*handler)(unsigned int , -struct pt_regs *)); +extern void m68k_setup_auto_interrupt (int (*handler)(unsigned int , +struct pt_regs *));extern void m68k_setup_user_interrupt (unsigned int vec,unsigned int cnt); extern void m68k_setup_irq_controller(struct irq_chip *, void (*handle)(unsigned int irq,
arch/m68k/kernel/entry.S +2 -8
@@ -289,7 +289,7 @@ ret_from_interrupt: * Only the last interrupt leaving the kernel goes through the * various exception return checks. */ - cmpl #0, %d0 + tstl %d0 jeq .Lret_from_exception RESTORE_ALL
@@ -313,18 +313,12 @@ user_irqvec_fixup = . + 2 ENTRY(bad_inthandler) SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL + jra ret_from_interrupt resume:
arch/m68k/kernel/ints.c +7 -2
@@ -72,7 +72,8 @@ void __init init_IRQ(void)* standard do_IRQ(), it will be called with irq numbers in the range * from IRQ_AUTO_1 - IRQ_AUTO_7. */ -void __init m68k_setup_auto_interrupt (void (*handler)(unsigned int ,struct pt_regs *)) +void __init m68k_setup_auto_interrupt (int (*handler)(unsigned int , +struct pt_regs *)) {if (handler) *auto_irqhandler_fixup = (u32 )handler;
@@ -162,8 +163,12 @@ unsigned int irq_canonicalize(unsigned int irq)EXPORT_SYMBOL (irq_canonicalize); -asmlinkage void handle_badint(struct pt_regs *regs) +asmlinkage int handle_badint(struct pt_regs *regs) { +int nested = regs->sr & ~ALLOWINT ; +atomic_inc (&irq_err_count);pr_warn ("unexpected interrupt from %u\n" , regs->vector); + +return nested; }
arch/m68k/platform/68000/entry.S +1 -1
@@ -221,7 +221,7 @@ ret_from_interrupt: * Only the last interrupt leaving the kernel goes through the * various exception return checks. */ - cmpl #0, %d0 + tstl %d0 jeq ret_from_exception RESTORE_ALL
arch/m68k/platform/68000/ints.c +1 -1
@@ -76,7 +76,7 @@ asmlinkage irqreturn_t inthandler7(void);*/ int process_int (int vec,struct pt_regs *fp) { -int irq, mask, nested =fp->sr & ~ALLOWINT ; +int irq, mask, nested = fp->sr & ~ALLOWINT ;unsigned long pend =ISR ;while (pend) {
arch/m68k/platform/68360/entry.S +2 -2
@@ -133,14 +133,14 @@ inthandler: movel %sp,%sp@- movel %d0,%sp@- /* put vector # on stack*/ jbsr do_IRQ /* process the IRQ, returns nest level */ - addql #8,%sp /* pop parameters off stack*/ + addql #8,%sp /* pop parameters off stack*/ ret_from_interrupt: /* * Only the last interrupt leaving the kernel goes through the * various exception return checks. */ - cmpl #0, %d0 + tstl %d0 jeq ret_from_exception RESTORE_ALL
arch/m68k/q40/q40ints.c +1 -1
@@ -33,7 +33,7 @@* */ -static void q40_irq_handler (unsigned int ,struct pt_regs *fp); +static int q40_irq_handler (unsigned int ,struct pt_regs *fp);static void q40_irq_enable (struct irq_data *data);static void q40_irq_disable (struct irq_data *data);
Signature · 12 lines
1.7.9.5
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
-- Linus TorvaldsRe: [patch 1/6] hardirq: Make hardirq bits generic
Geert,
Quoted 3 lines · Geert Uytterhoeven
But only if you look at "[m68k] IRQ: add
handle_polled_irq()for timer based soft interrupt" (http://www.spinics.net/lists/linux-m68k/msg05889.html)
first ;-)
Done. Thanks for the reminder!
Quoted 34 lines
Below is a patch with some fixups, on top of your two patches.
Unfortunately it still hangs somewhere after mounting the root filesystem.
Using this debug code for
do_IRQ():arch/m68k/kernel/irq.c +10
-0@@ -22,11 +22,21 @@ asmlinkage int do_IRQ(int irq, struct pt_regs *regs)struct pt_regs *oldregs =set_irq_regs (regs);int nested = regs->sr & ~ALLOWINT ; +static int nesting; +const char prefix[] =" " ; +unsigned long flags; +local_irq_save (flags); +nesting++; +printk ("# %sirq %d nested %d\n" , &prefix[16 -2 *nesting], irq, nested); +local_irq_restore (flags);irq_enter ();generic_handle_irq (irq);irq_exit_nested (nested);set_irq_regs (oldregs); +local_irq_save (flags); +nesting--; +local_irq_restore (flags);return nested; }I get output like
Quoted 4 lines
# irq 15 nested 0
# irq 15 nested 1024irq 15 while irq 15 in progress??
Huch, that's odd.
Quoted 6 lines
With similar debug code on the old working
do_IRQ(), I get
- slightly less deep nesting,
-do_IRQ()is never re-entered with the same irq number.Also note that the value of "nested" doesn't match the indentation level, which depends on my own bookkeeping using "nesting".
Well, nested is just an indicator. It's not the nest level.
nested = pt->sr & ~ALLOWINT;
i.e.:
nested = pt->sr & 0x0700;So in the case above nested is 0x400
Quoted 1 line
Anyone with an idea where it's going wrong?
The original code does:
add_preempt_count(HARDIRQ_OFFSET);
do_IRQ()
irq_enter();
add_preempt_count(HARDIRQ_OFFSET);
handle_irq();
irq_exit();
local_irq_disable();
sub_preempt_count(HARDIRQ_OFFSET);
sub_preempt_count(HARDIRQ_OFFSET);
/* Check for nested irq */
if (in_hardirq())
reti();
/* Check for nested irq again */
if (pt->sr & ~ALLOWINT != 0)
reti();
do_softirq();
....
ret_from_exception();With the patches in place it looks like this:
do_IRQ()
nested = pt->sr & ~ALLOWINT;
irq_enter();
add_preempt_count(HARDIRQ_OFFSET);
handle_irq();
irq_exit_nested(nested);
local_irq_disable();
sub_preempt_count(HARDIRQ_OFFSET);
if (!nested && !in_hardirq())
do_softirq()
return nested;
if (nested)
reti();
ret_from_exception();So all it does essentially is to move the softirq invocation in the non nested case a tad earlier. I'm really puzzled as I can't spot the point where this change makes a real difference.
Thanks,
tglx
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
Hi Thomas,
Quoted 4 lines · Thomas Gleixner
Quoted 2 lines
Also note that the value of "nested" doesn't match the indentation level, which depends on my own bookkeeping using "nesting".
Well, nested is just an indicator. It's not the nest level.
I know, the only thing that matters is whether it's zero or not. But it should always be zero if there's no nesting, and non-zero if there is, right?
So:
# irq 13 nested 1024
nested should be 0 here.
# irq 4 nested 0
ok
# irq 13 nested 1024
ok (two extra spaces in front of "irq").
# irq 4 nested 0
nested should be non-zero here.
Quoted 59 lines
nested = pt->sr & ~ALLOWINT; i.e.: nested = pt->sr & 0x0700;So in the case above nested is
0x400Quoted 1 line
Anyone with an idea where it's going wrong?
The original code does:
add_preempt_count(HARDIRQ_OFFSET); do_IRQ() irq_enter(); add_preempt_count(HARDIRQ_OFFSET); handle_irq(); irq_exit(); local_irq_disable(); sub_preempt_count(HARDIRQ_OFFSET); sub_preempt_count(HARDIRQ_OFFSET); /* Check for nested irq */ if (in_hardirq()) reti(); /* Check for nested irq again */ if (pt->sr & ~ALLOWINT != 0) reti(); do_softirq(); .... ret_from_exception();With the patches in place it looks like this:
do_IRQ() nested = pt->sr & ~ALLOWINT; irq_enter(); add_preempt_count(HARDIRQ_OFFSET); handle_irq(); irq_exit_nested(nested); local_irq_disable(); sub_preempt_count(HARDIRQ_OFFSET); if (!nested && !in_hardirq()) do_softirq() return nested; if (nested) reti(); ret_from_exception();
Quoted 3 lines
So all it does essentially is to move the softirq invocation in the non nested case a tad earlier. I'm really puzzled as I can't spot the point where this change makes a real difference.
Yes, that's also my understanding.
But I can't spot it neither :-(
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 22 lines · Geert Uytterhoeven
Hi Thomas,
Quoted 4 lines · Thomas Gleixner
Quoted 2 lines
Also note that the value of "nested" doesn't match the indentation level, which depends on my own bookkeeping using "nesting".
Well, nested is just an indicator. It's not the nest level.
I know, the only thing that matters is whether it's zero or not. But it should always be zero if there's no nesting, and non-zero if there is, right?
So:
# irq 13 nested 1024
nested should be 0 here.
# irq 4 nested 0
ok
Quoted 7 lines
# irq 13 nested 1024
ok (two extra spaces in front of "irq").
# irq 4 nested 0
nested should be non-zero here.
Hmm. The softirq code reenables interrupts unconditionally. So when an interrupt hits there SR on stack has the bits cleared. You could verify that by checking in_serving_softirq() at the entry to do_IRQ(). That could also explain the irq 4 nests in irq 4 issue. You can't observe that on the original code as the softirq invocation and therefor the interrupt enable happens outside of do_IRQ().
Though that does not explain the non nested case where nested is !=
0. But it looks like that irq 13 has a higher level than 4:
Quoted 3 lines
# irq 13 nested 1024
ok (two extra spaces in front of "irq").
So it could actually be the following:
irq X arrives, SR I2/1/0 is set to 4
Now before we reach do_IRQ()
irq 13 arrives and interrupts irq X as it has a higher level
Your nest accounting shows 0, but the SR says nested, which is actually the correct state.
Is there an easy to setup/use emulator around on which I could try to dig into that myself?
Thanks,
tglx
Re: [patch 1/6] hardirq: Make hardirq bits generic
Thomas,
Quoted 2 lines
Is there an easy to setup/use emulator around on which I could try to dig into that myself?
I believe Geert uses ARAnyM for his tests of m68k kernels on emulators - it is reasonably easy to set up and use. I've used it to debug problems we had with the SLUB allocator two years ago.
Emulation is for Atari Falcon 040 hardware, things like interrupt priorities ought to be reproduced faithfully.
Not that it would hurt to try Geert's last non-booting kernel on true hardware - either send the kernel image or a patch to apply to your current tree please, Geert.
Regards,
Michael
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 6 lines · Michael Schmitz
Quoted 2 lines
Is there an easy to setup/use emulator around on which I could try to dig into that myself?
I believe Geert uses ARAnyM for his tests of m68k kernels on emulators - it is reasonably easy to set up and use. I've used it to debug problems we had with the SLUB allocator two years ago.
Indeed. ARAnyM is the way to go.
Quoted 3 lines
Not that it would hurt to try Geert's last non-booting kernel on true hardware - either send the kernel image or a patch to apply to your current tree please, Geert.
Patches sent by private email.
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 10 lines · Geert Uytterhoeven
Quoted 6 lines · Michael Schmitz
Quoted 2 lines
Is there an easy to setup/use emulator around on which I could try to dig into that myself?
I believe Geert uses ARAnyM for his tests of m68k kernels on emulators - it is reasonably easy to set up and use. I've used it to debug problems we had with the SLUB allocator two years ago.
Indeed. ARAnyM is the way to go.
Ok. Got it running and looked a bit deeper. I haven't yet found the root cause, but there are quite some fishy things going on. Adding enough debug printks makes the thing boot. Aside of that there seems to be a violation of the 68k interrupt model.
The 68k interrupt handling allows only interrupts which have an higher level than the value of the interrupt priority mask in SR. Further the cpu sets the SR priority mask on interrupt entry to the level of the interrupt which is serviced.
So now with aranym I can see a different behaviour. I just added the debug patch below to a vanilla 3.12.
And I can see ever repeating
IRQ 13 flags 0x400 regs->sr 0x400
with a few
IRQ 15 flags 0x400 regs->sr 0x400
sprinkeled in.
Not what you would expect, right?
Thanks,
tglx
arch/m68k/kernel/irq.c +6 -0
@@ -20,6 +20,12 @@ asmlinkage void do_IRQ(int irq, struct pt_regs *regs) {struct pt_regs *oldregs =set_irq_regs (regs); +unsigned long nested = regs->sr & ~ALLOWINT ; +unsigned long flags =arch_local_save_flags () & ~ALLOWINT ; + +if (nested >= flags) +printk (KERN_ERR "IRQ %d flags 0x%lx regs->sr 0x%lx\n" , + irq, flags, nested);irq_enter ();generic_handle_irq (irq);
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 1 line · Thomas Gleixner
Not what you would expect, right?
Finally found the issue. The patch below fixes the problem here. The little missing detail is, that I zapped GET_CURRENT() assuming blindly that this is only needed for the preempt_count hackery. But in fact the world and some more depends on it which leads to interesting explosions.
Thanks,
tglx
----
arch/m68k/kernel/entry.S +3 -0
@@ -274,6 +274,7 @@ do_delayed_trace: ENTRY(auto_inthandler) SAVE_ALL_INT + GET_CURRENT(%d0) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0
@@ -297,6 +298,7 @@ ret_from_interrupt: ENTRY(user_inthandler) SAVE_ALL_INT + GET_CURRENT(%d0) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 user_irqvec_fixup = . + 2
@@ -313,6 +315,7 @@ user_irqvec_fixup = . + 2 ENTRY(bad_inthandler) SAVE_ALL_INT + GET_CURRENT(%d0) movel %sp,%sp@- jsr handle_badint
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 8 lines · Thomas Gleixner
Quoted 1 line · Thomas Gleixner
Not what you would expect, right?
Finally found the issue. The patch below fixes the problem here. The little missing detail is, that I zapped
GET_CURRENT()assuming blindly that this is only needed for thepreempt_counthackery. But in fact the world and some more depends on it which leads to interesting explosions.
Some more thoughts on this.
The whole nesting check in the exisiting low level entry code and what I tried to resemble with the irq_exit_nested() is pretty pointless.
Let's look at auto_inthandler and ret_from_exception
ENTRY(auto_inthandler)
SAVE_ALL_INT
GET_CURRENT(%d0)
movel %d0,%a1
addqb #1,%a1@(TINFO_PREEMPT+1)
| put exception # in d0
bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
subw #VEC_SPUR,%d0
movel %sp,%sp@-
movel %d0,%sp@- | put vector # on stackauto_irqhandler_fixup = . + 2
jsr do_IRQ | process the IRQ
addql #8,%sp | pop parameters off stack
ret_from_interrupt:
movel %curptr@(TASK_STACK),%a1 subqb #1,%a1@(TINFO_PREEMPT+1) jeq ret_from_last_interrupt
2: RESTORE_ALL
ALIGNret_from_last_interrupt:
moveq #(~ALLOWINT>>8)&0xff,%d0 andb %sp@(PT_OFF_SR),%d0 jne 2b /* check if we need to do software interrupts */ tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING jeq .Lret_from_exception pea ret_from_exception jra do_softirq
ENTRY(ret_from_exception)
.Lret_from_exception:
btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
bnes 1f | if so, skip resched, signals
....
1: RESTORE_ALL
So in every interrupt exit path we check:
1) Whether the hardirq part of preempt_count is zero
2) Whether the interrupt prio mask of SR on stack is zero
and if we finally reach ret_from_exception we have the final check:
3) whether we return to kernel or user space.
And this final check is the only one which matters, really.
If you look at the probability of the first two checks catching anything, then it's pretty low. Most interrupts returns go through ret_from_exception. Yes, I added counters which prove that at least on the aranym, but I doubt that it will make a real difference if you run this on real hardware.
So what's the point of having these checks in the hotpath? The patch below against 3.12 vanilla works nicely on the aranym and I don't see a reason why this extra hackery is necessary at all. It's just code bloat in a hotpath.
Now the only valid concern might be do_softirq itself, but that's pointless as well. If the softirq is interrupted, then we do not invoke it again. If the nested interrupt happens before irq_exit() actually disables interrupts, then we won't invoke it either as the hardirq part of preempt_count is still not zero.
As a side note: The do_softirq calls in the platform/68xxx entry pathes are just copied leftovers as well. Both entry code pathes are not fiddling with the preempt count and both call do_IRQ() which will call irq_exit() at the end which will invoke do_softirq(), so the check for more softirqs in the irq return path is just pointless.
Thanks,
tglx
--- kernel/entry.S | 40 ++++------------------------------------ kernel/ints.c | 6 ------ platform/68000/entry.S | 33 ++++++++------------------------- platform/68360/entry.S | 24 +++--------------------- 4 files changed, 15 insertions(+), 88 deletions(-)
arch/m68k/kernel/entry.S +4 -36
@@ -45,7 +45,7 @@ .globl system_call, buserr, trap, resume .globl sys_call_table .globl __sys_fork, __sys_clone, __sys_vfork -.globl ret_from_interrupt, bad_interrupt +.globl bad_interrupt .globl auto_irqhandler_fixup .globl user_irqvec_fixup
@@ -275,8 +275,6 @@ do_delayed_trace: ENTRY(auto_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0
@@ -286,32 +284,13 @@ ENTRY(auto_inthandler) auto_irqhandler_fixup = . + 2 jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - -ret_from_interrupt: - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt -2: RESTORE_ALL - - ALIGN -ret_from_last_interrupt: - moveq #(~ALLOWINT>>8)&0xff,%d0 - andb %sp@(PT_OFF_SR),%d0 - jne 2b - - /* check if we need to do software interrupts */ - tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING - jeq .Lret_from_exception - pea ret_from_exception - jra do_softirq + jra ret_from_exception /* Handler for user defined interrupt vectors */ ENTRY(user_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 user_irqvec_fixup = . + 2
@@ -321,29 +300,18 @@ user_irqvec_fixup = . + 2 movel %d0,%sp@- | put vector # on stack jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL + jra ret_from_exception /* Handler for uninitialized and spurious interrupts */ ENTRY(bad_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL - + jra ret_from_exception resume: /*
arch/m68k/kernel/ints.c +0 -6
@@ -58,12 +58,6 @@ void __init init_IRQ(void) {int i; -/* assembly irq entry code relies on this... */ -if (HARDIRQ_MASK !=0x00ff0000 ) { -extern void hardirq_mask_is_broken (void ); -hardirq_mask_is_broken (); - } -for (i =IRQ_AUTO_1 ; i <=IRQ_AUTO_7 ; i++)irq_set_chip_and_handler (i, &auto_irq_chip, handle_simple_irq);
arch/m68k/platform/68000/entry.S +8 -25
@@ -27,7 +27,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler1 .globl inthandler2
@@ -137,7 +136,7 @@ inthandler1: movel #65,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler2: SAVE_ALL_INT
@@ -148,7 +147,7 @@ inthandler2: movel #66,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler3: SAVE_ALL_INT
@@ -159,7 +158,7 @@ inthandler3: movel #67,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler4: SAVE_ALL_INT
@@ -170,7 +169,7 @@ inthandler4: movel #68,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler5: SAVE_ALL_INT
@@ -181,7 +180,7 @@ inthandler5: movel #69,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler6: SAVE_ALL_INT
@@ -192,7 +191,7 @@ inthandler6: movel #70,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler7: SAVE_ALL_INT
@@ -203,7 +202,7 @@ inthandler7: movel #71,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler: SAVE_ALL_INT
@@ -214,23 +213,7 @@ inthandler: movel %d0,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - - /* check if we need to do software interrupts */ - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + bra ret_from_exception /* * Handler for uninitialized and spurious interrupts.
arch/m68k/platform/68360/entry.S +3 -21
@@ -29,7 +29,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler
@@ -132,26 +131,9 @@ inthandler: movel %sp,%sp@- movel %d0,%sp@- /* put vector # on stack*/ - jbsr do_IRQ /* process the IRQ*/ -3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - /* check if we need to do software interrupts */ - - movel irq_stat+CPUSTAT_SOFTIRQ_PENDING,%d0 - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + jbsr do_IRQ /* process the IRQ */ + addql #8,%sp /* pop parameters off stack*/ + jra ret_from_exception /* * Handler for uninitialized and spurious interrupts.
Re: [patch 1/6] hardirq: Make hardirq bits generic
Thomas,
Quoted 5 lines
Quoted 5 lines
Finally found the issue. The patch below fixes the problem here. The little missing detail is, that I zapped
GET_CURRENT()assuming blindly that this is only needed for thepreempt_counthackery. But in fact the world and some more depends on it which leads to interesting explosions.
Thanks for debugging this - I won't speculate on why we attempt to handle softirqs explicitly, as I have only a passing acquaintance with this code, many years back.
Quoted 59 lines
Some more thoughts on this.
The whole nesting check in the exisiting low level entry code and what I tried to resemble with the
irq_exit_nested()is pretty pointless.Let's look at
auto_inthandlerandret_from_exception
ENTRY(auto_inthandler)SAVE_ALL_INT GET_CURRENT(%d0) movel %d0,%a1 addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0 movel %sp,%sp@- movel %d0,%sp@- | put vector # on stack
auto_irqhandler_fixup= . + 2
jsrdo_IRQ| process the IRQ
addql #8,%sp | pop parameters off stack
ret_from_interrupt:movel %curptr@(TASK_STACK),%a1 subqb #1,%a1@(TINFO_PREEMPT+1) jeq ret_from_last_interrupt2:
RESTORE_ALLALIGN
ret_from_last_interrupt:moveq #(~ALLOWINT>>8)&0xff,%d0 andb %sp@(PT_OFF_SR),%d0 jne 2b /* check if we need to do software interrupts */ tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING jeq .Lret_from_exception pea ret_from_exception jra do_softirq
ENTRY(ret_from_exception)
.Lret_from_exception:
btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
bnes 1f | if so, skip resched, signals
....
1:RESTORE_ALLSo in every interrupt exit path we check:
1) Whether the hardirq part of
preempt_countis zero2) Whether the interrupt prio mask of SR on stack is zero
and if we finally reach
ret_from_exceptionwe have the final check:3) whether we return to kernel or user space.
Quoted 7 lines
And this final check is the only one which matters, really.
If you look at the probability of the first two checks catching anything, then it's pretty low. Most interrupts returns go through
ret_from_exception. Yes, I added counters which prove that at least on the aranym, but I doubt that it will make a real difference if you run this on real hardware.
I'm happy to try that (indeed, verify your patch works on Atari hardware first) - I trust the patch is relative to m68k git, not your previous patches?
Cheers,
Michael
Quoted 284 lines
So what's the point of having these checks in the hotpath? The patch below against 3.12 vanilla works nicely on the aranym and I don't see a reason why this extra hackery is necessary at all. It's just code bloat in a hotpath.
Now the only valid concern might be
do_softirqitself, but that's pointless as well. If the softirq is interrupted, then we do not invoke it again. If the nested interrupt happens beforeirq_exit()actually disables interrupts, then we won't invoke it either as the hardirq part ofpreempt_countis still not zero.As a side note: The
do_softirqcalls in the platform/68xxx entry pathes are just copied leftovers as well. Both entry code pathes are not fiddling with the preempt count and both calldo_IRQ()which will callirq_exit()at the end which will invokedo_softirq(), so the check for more softirqs in the irq return path is just pointless.Thanks,
tglx
--- kernel/entry.S | 40 ++++------------------------------------ kernel/ints.c | 6 ------ platform/68000/entry.S | 33 ++++++++------------------------- platform/68360/entry.S | 24 +++--------------------- 4 files changed, 15 insertions(+), 88 deletions(-)Index:
linux-2.6/arch/m68k/kernel/entry.S
===================================================================arch/m68k/kernel/entry.S +4
-36@@ -45,7 +45,7 @@ .globl system_call, buserr, trap, resume .globl sys_call_table .globl __sys_fork, __sys_clone, __sys_vfork -.globl ret_from_interrupt, bad_interrupt +.globl bad_interrupt .globl auto_irqhandler_fixup .globl user_irqvec_fixup@@ -275,8 +275,6 @@ do_delayed_trace: ENTRY(auto_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0@@ -286,32 +284,13 @@ ENTRY(auto_inthandler) auto_irqhandler_fixup = . + 2 jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - -ret_from_interrupt: - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt -2: RESTORE_ALL - - ALIGN -ret_from_last_interrupt: - moveq #(~ALLOWINT>>8)&0xff,%d0 - andb %sp@(PT_OFF_SR),%d0 - jne 2b - - /* check if we need to do software interrupts */ - tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING - jeq .Lret_from_exception - pea ret_from_exception - jra do_softirq + jra ret_from_exception /* Handler for user defined interrupt vectors */ ENTRY(user_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 user_irqvec_fixup = . + 2@@ -321,29 +300,18 @@ user_irqvec_fixup = . + 2 movel %d0,%sp@- | put vector # on stack jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL + jra ret_from_exception /* Handler for uninitialized and spurious interrupts */ ENTRY(bad_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL - + jra ret_from_exception resume: /*Index:
linux-2.6/arch/m68k/kernel/ints.c
===================================================================arch/m68k/kernel/ints.c +0
-6@@ -58,12 +58,6 @@ void __init init_IRQ(void) {int i; -/* assembly irq entry code relies on this... */ -if (HARDIRQ_MASK !=0x00ff0000 ) { -extern void hardirq_mask_is_broken (void ); -hardirq_mask_is_broken (); - } -for (i =IRQ_AUTO_1 ; i <=IRQ_AUTO_7 ; i++)irq_set_chip_and_handler (i, &auto_irq_chip, handle_simple_irq);Index:
linux-2.6/arch/m68k/platform/68000/entry.S
===================================================================arch/m68k/platform/68000/entry.S +8
-25@@ -27,7 +27,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler1 .globl inthandler2@@ -137,7 +136,7 @@ inthandler1: movel #65,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler2: SAVE_ALL_INT@@ -148,7 +147,7 @@ inthandler2: movel #66,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler3: SAVE_ALL_INT@@ -159,7 +158,7 @@ inthandler3: movel #67,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler4: SAVE_ALL_INT@@ -170,7 +169,7 @@ inthandler4: movel #68,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler5: SAVE_ALL_INT@@ -181,7 +180,7 @@ inthandler5: movel #69,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler6: SAVE_ALL_INT@@ -192,7 +191,7 @@ inthandler6: movel #70,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler7: SAVE_ALL_INT@@ -203,7 +202,7 @@ inthandler7: movel #71,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler: SAVE_ALL_INT@@ -214,23 +213,7 @@ inthandler: movel %d0,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - - /* check if we need to do software interrupts */ - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + bra ret_from_exception /* * Handler for uninitialized and spurious interrupts.Index:
linux-2.6/arch/m68k/platform/68360/entry.S
===================================================================arch/m68k/platform/68360/entry.S +3
-22@@ -29,7 +29,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler@@ -132,26 +131,9 @@ inthandler: movel %sp,%sp@- movel %d0,%sp@- /* put vector # on stack*/ - jbsr do_IRQ /* process the IRQ*/ -3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - /* check if we need to do software interrupts */ - - movel irq_stat+CPUSTAT_SOFTIRQ_PENDING,%d0 - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + jbsr do_IRQ /* process the IRQ */ + addql #8,%sp /* pop parameters off stack*/ + jra ret_from_exception /* * Handler for uninitialized and spurious interrupts. --To unsubscribe from this list: send the line "unsubscribe linux-m68k" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [patch 1/6] hardirq: Make hardirq bits generic
Thomas,
Quoted 9 lines
Quoted 7 lines
If you look at the probability of the first two checks catching anything, then it's pretty low. Most interrupts returns go through
ret_from_exception. Yes, I added counters which prove that at least on the aranym, but I doubt that it will make a real difference if you run this on real hardware.I'm happy to try that (indeed, verify your patch works on Atari hardware first) - I trust the patch is relative to m68k git, not your previous patches?
Pleased to report the patch works, as expected, on the actual hardware. I doubt we still need to profile the interrupt return path, Geert?
Cheers,
Michael
Re: [patch 1/6] hardirq: Make hardirq bits generic
In reply to Thomas GleixnerHi Thomas,
Quoted 60 lines · Thomas Gleixner
Some more thoughts on this.
The whole nesting check in the exisiting low level entry code and what I tried to resemble with the
irq_exit_nested()is pretty pointless.Let's look at
auto_inthandlerandret_from_exception
ENTRY(auto_inthandler)SAVE_ALL_INT GET_CURRENT(%d0) movel %d0,%a1 addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0 movel %sp,%sp@- movel %d0,%sp@- | put vector # on stack auto_irqhandler_fixup = . + 2 jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack
ret_from_interrupt:movel %curptr@(TASK_STACK),%a1 subqb #1,%a1@(TINFO_PREEMPT+1) jeq ret_from_last_interrupt2:
RESTORE_ALLALIGN
ret_from_last_interrupt:moveq #(~ALLOWINT>>8)&0xff,%d0 andb %sp@(PT_OFF_SR),%d0 jne 2b/* check if we need to do software interrupts */ tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING jeq .Lret_from_exception pea ret_from_exception jra do_softirq
ENTRY(ret_from_exception)
.Lret_from_exception:btst #5,%sp@(PT_OFF_SR) | check if returning to kernel bnes 1f | if so, skip resched, signals ....1:
RESTORE_ALLSo in every interrupt exit path we check:
1) Whether the hardirq part of
preempt_countis zero2) Whether the interrupt prio mask of SR on stack is zero
and if we finally reach
ret_from_exceptionwe have the final check:3) whether we return to kernel or user space.
And this final check is the only one which matters, really.
Quoted 7 lines
If you look at the probability of the first two checks catching anything, then it's pretty low. Most interrupts returns go through
ret_from_exception. Yes, I added counters which prove that at least on the aranym, but I doubt that it will make a real difference if you run this on real hardware.So what's the point of having these checks in the hotpath? The patch
Most of this seems to be as old as stone-age. It was rewritten for v2.5.29, but the initial bookkeeping was there in v2.1, and even in some form in v1.3.94.
Quoted 10 lines
below against 3.12 vanilla works nicely on the aranym and I don't see a reason why this extra hackery is necessary at all. It's just code bloat in a hotpath.
Now the only valid concern might be
do_softirqitself, but that's pointless as well. If the softirq is interrupted, then we do not invoke it again. If the nested interrupt happens beforeirq_exit()actually disables interrupts, then we won't invoke it either as the hardirq part ofpreempt_countis still not zero.
Quoted 5 lines
As a side note: The
do_softirqcalls in the platform/68xxx entry pathes are just copied leftovers as well. Both entry code pathes are not fiddling with the preempt count and both calldo_IRQ()which will callirq_exit()at the end which will invokedo_softirq(), so the check for more softirqs in the irq return path is just pointless.
Your reasoning sounds OK, and it works on ARAnyM, so, thanks again and
BTW, do you plan to get the "make hardirq bits generic" series in 3.13? Or do you want me to take this patch through the m68k tree? So far I don't have any plans to send another pull request for 3.13.
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] m68k: Simplify low level interrupt handling code
In reply to Thomas Gleixnerm68k: Simplify low level interrupt handling code
Commit-ID: 09f90f6685cd88b6b904c141035d096169958cc4 Gitweb: http://git.kernel.org/tip/09f90f6685cd88b6b904c141035d096169958cc4 Author: Thomas Gleixner <[email protected]> AuthorDate: Mon, 11 Nov 2013 21:01:03 +0100 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:46 +0100
m68k: Simplify low level interrupt handling code
The low level interrupt entry code of m68k contains the following:
add_preempt_count(HARDIRQ_OFFSET);
do_IRQ();
irq_enter();
add_preempt_count(HARDIRQ_OFFSET);
handle_interrupt();
irq_exit();
sub_preempt_count(HARDIRQ_OFFSET);
if (in_interrupt())
return; <---- On m68k always taken!
if (local_softirq_pending())
do_softirq();
sub_preempt_count(HARDIRQ_OFFSET);
if (in_hardirq())
return;
if (status_on_stack_has_interrupt_priority_mask > 0)
return;
if (local_softirq_pending())
do_softirq();
ret_from_exception:
if (interrupted_context_is_kernel)
return:
....I tried to find a proper explanation for this, but the changelog is sparse and there are no mails explaining it further. But obviously this relates to the interrupt priority levels of the m68k and tries to be extra clever with nested interrupts. Though this cleverness just adds code bloat to the interrupt hotpath.
For the common case of non nested interrupts the code runs through two extra conditionals to the only important one, which checks whether the return is to kernel or user space.
For the nested case the checks for in_hardirq() and the priority mask value on stack catch only the case where the nested interrupt happens inside the hard irq context of the first interrupt. If the nested interrupt happens while the first interrupt handles soft interrupts, then these extra checks buy nothing. The nested interrupt will fall through to the final kernel/user space return check at ret_from_exception.
Changing the code flow in the following way:
do_IRQ();
irq_enter();
add_preempt_count(HARDIRQ_OFFSET);
handle_interrupt();
irq_exit();
sub_preempt_count(HARDIRQ_OFFSET);
if (in_interrupt())
return;
if (local_softirq_pending())
do_softirq();
ret_from_exception:
if (interrupted_context_is_kernel)
return:makes the region protected by the hardirq count slightly smaller and the softirq handling is invoked with a minimal deeper stack. But otherwise it's completely functional equivalent and saves 104 bytes of text in arch/m68k/kernel/entry.o.
This modification allows us further to get rid of the limitations which m68k puts on the preempt_count layout, so we can make the preempt count bits completely generic.
--- arch/m68k/kernel/entry.S | 40 ++++------------------------------------ arch/m68k/kernel/ints.c | 6 ------ arch/m68k/platform/68000/entry.S | 33 ++++++++------------------------- arch/m68k/platform/68360/entry.S | 24 +++--------------------- 4 files changed, 15 insertions(+), 88 deletions(-)
arch/m68k/kernel/entry.S +4 -36
@@ -45,7 +45,7 @@ .globl system_call, buserr, trap, resume .globl sys_call_table .globl __sys_fork, __sys_clone, __sys_vfork -.globl ret_from_interrupt, bad_interrupt +.globl bad_interrupt .globl auto_irqhandler_fixup .globl user_irqvec_fixup
@@ -275,8 +275,6 @@ do_delayed_trace: ENTRY(auto_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 subw #VEC_SPUR,%d0
@@ -286,32 +284,13 @@ ENTRY(auto_inthandler) auto_irqhandler_fixup = . + 2 jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - -ret_from_interrupt: - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt -2: RESTORE_ALL - - ALIGN -ret_from_last_interrupt: - moveq #(~ALLOWINT>>8)&0xff,%d0 - andb %sp@(PT_OFF_SR),%d0 - jne 2b - - /* check if we need to do software interrupts */ - tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING - jeq .Lret_from_exception - pea ret_from_exception - jra do_softirq + jra ret_from_exception /* Handler for user defined interrupt vectors */ ENTRY(user_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) | put exception # in d0 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 user_irqvec_fixup = . + 2
@@ -321,29 +300,18 @@ user_irqvec_fixup = . + 2 movel %d0,%sp@- | put vector # on stack jsr do_IRQ | process the IRQ addql #8,%sp | pop parameters off stack - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL + jra ret_from_exception /* Handler for uninitialized and spurious interrupts */ ENTRY(bad_inthandler) SAVE_ALL_INT GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) movel %sp,%sp@- jsr handle_badint addql #4,%sp - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL - + jra ret_from_exception resume: /*
arch/m68k/kernel/ints.c +0 -6
@@ -58,12 +58,6 @@ void __init init_IRQ(void) {int i; -/* assembly irq entry code relies on this... */ -if (HARDIRQ_MASK !=0x00ff0000 ) { -extern void hardirq_mask_is_broken (void ); -hardirq_mask_is_broken (); - } -for (i =IRQ_AUTO_1 ; i <=IRQ_AUTO_7 ; i++)irq_set_chip_and_handler (i, &auto_irq_chip, handle_simple_irq);
arch/m68k/platform/68000/entry.S +8 -25
@@ -27,7 +27,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler1 .globl inthandler2
@@ -137,7 +136,7 @@ inthandler1: movel #65,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler2: SAVE_ALL_INT
@@ -148,7 +147,7 @@ inthandler2: movel #66,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler3: SAVE_ALL_INT
@@ -159,7 +158,7 @@ inthandler3: movel #67,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler4: SAVE_ALL_INT
@@ -170,7 +169,7 @@ inthandler4: movel #68,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler5: SAVE_ALL_INT
@@ -181,7 +180,7 @@ inthandler5: movel #69,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler6: SAVE_ALL_INT
@@ -192,7 +191,7 @@ inthandler6: movel #70,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler7: SAVE_ALL_INT
@@ -203,7 +202,7 @@ inthandler7: movel #71,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt + bra ret_from_exception inthandler: SAVE_ALL_INT
@@ -214,23 +213,7 @@ inthandler: movel %d0,%sp@- /* put vector # on stack*/ jbsr process_int /* process the IRQ*/ 3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - - /* check if we need to do software interrupts */ - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + bra ret_from_exception /* * Handler for uninitialized and spurious interrupts.
arch/m68k/platform/68360/entry.S +3 -21
@@ -29,7 +29,6 @@ .globl ret_from_exception .globl ret_from_signal .globl sys_call_table -.globl ret_from_interrupt .globl bad_interrupt .globl inthandler
@@ -132,26 +131,9 @@ inthandler: movel %sp,%sp@- movel %d0,%sp@- /* put vector # on stack*/ - jbsr do_IRQ /* process the IRQ*/ -3: addql #8,%sp /* pop parameters off stack*/ - bra ret_from_interrupt - -ret_from_interrupt: - jeq 1f -2: - RESTORE_ALL -1: - moveb %sp@(PT_OFF_SR), %d0 - and #7, %d0 - jhi 2b - /* check if we need to do software interrupts */ - - movel irq_stat+CPUSTAT_SOFTIRQ_PENDING,%d0 - jeq ret_from_exception - - pea ret_from_exception - jra do_softirq - + jbsr do_IRQ /* process the IRQ */ + addql #8,%sp /* pop parameters off stack*/ + jra ret_from_exception /* * Handler for uninitialized and spurious interrupts.
Geert Uytterhoeven
maintainerRe: [patch 1/6] hardirq: Make hardirq bits generic
In reply to Thomas GleixnerHi Thomas,
Quoted 5 lines · Thomas Gleixner
Finally found the issue. The patch below fixes the problem here. The little missing detail is, that I zapped
GET_CURRENT()assuming blindly that this is only needed for thepreempt_counthackery. But in fact the world and some more depends on it which leads to interesting explosions.
Yes, GET_CURRENT() sets up the current task in %a2.
Many thanks for tracking this down!
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Re: [patch 1/6] hardirq: Make hardirq bits generic
In reply to Thomas GleixnerQuoted 11 lines · Thomas Gleixner
And I can see ever repeating
IRQ 13 flags
0x400regs->sr0x400with a few
IRQ 15 flags
0x400regs->sr0x400sprinkeled in.
Not what you would expect, right?
If you configured for ATARI only, then ALLOWINT filters out the I1 bit, which makes irq level 4 and level 6 look the same (all MFP interrupts are at level 6).
Andreas.
Signature · 3 lines
Andreas Schwab, [email protected] GPG Key fingerprint = 58CA 54C7 6D53 942B 1756 01D3 44D5 214B 8276 4ED5 "And now for something completely different."
Re: [patch 1/6] hardirq: Make hardirq bits generic
Quoted 10 lines · Andreas Schwab
Quoted 8 lines · Thomas Gleixner
And I can see ever repeating
IRQ 13 flags
0x400regs->sr0x400with a few
IRQ 15 flags
0x400regs->sr0x400
Quoted 7 lines
Quoted 3 lines
sprinkeled in.
Not what you would expect, right?
If you configured for ATARI only, then ALLOWINT filters out the I1 bit, which makes irq level 4 and level 6 look the same (all MFP interrupts are at level 6).
Fair enough.
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] hardirq: Make hardirq bits generic
In reply to Thomas Gleixnerhardirq: Make hardirq bits generic
Commit-ID: 54197e43a4a9a0f3fc406d72d9815754e84fab1e Gitweb: http://git.kernel.org/tip/54197e43a4a9a0f3fc406d72d9815754e84fab1e Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Sep 2013 18:53:05 +0000 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:46 +0100
hardirq: Make hardirq bits generic
There is no reason for per arch hardirq bits. Make them all generic
--- arch/blackfin/include/asm/hardirq.h | 3 --- arch/cris/include/asm/hardirq.h | 12 ------------ arch/m32r/include/asm/hardirq.h | 16 ---------------- arch/m68k/include/asm/hardirq.h | 11 ----------- arch/s390/include/asm/hardirq.h | 2 -- arch/sparc/include/asm/hardirq_32.h | 1 - arch/sparc/include/asm/hardirq_64.h | 2 -- arch/tile/include/asm/hardirq.h | 2 -- include/linux/preempt_mask.h | 30 ++++++++---------------------- 9 files changed, 8 insertions(+), 71 deletions(-)
arch/blackfin/include/asm/hardirq.h +0 -3
@@ -12,9 +12,6 @@extern void ack_bad_irq (unsigned int irq);#define ack_bad_irq ack_bad_irq -/* Define until common code gets sane defaults */ -#define HARDIRQ_BITS 9 -#include <asm-generic/hardirq.h> #endif
arch/cris/include/asm/hardirq.h +0 -12
@@ -2,18 +2,6 @@#define __ASM_HARDIRQ_H#include <asm/irq.h> - -#define HARDIRQ_BITS 8 - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#include <asm-generic/hardirq.h> #endif /* __ASM_HARDIRQ_H */
arch/m32r/include/asm/hardirq.h +0 -16
@@ -3,22 +3,6 @@#define __ASM_HARDIRQ_H#include <asm/irq.h> - -#if NR_IRQS >256 -#define HARDIRQ_BITS 9 -#else -#define HARDIRQ_BITS 8 -#endif - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#include <asm-generic/hardirq.h> #endif /* __ASM_HARDIRQ_H */
arch/m68k/include/asm/hardirq.h +0 -11
@@ -5,17 +5,6 @@#include <linux/cache.h> #include <asm/irq.h> -#define HARDIRQ_BITS 8 - -/* -* The hardirq mask has to be large enough to have -* space for potentially all IRQ sources in the system -* nesting on a single CPU: -*/ -#if (1 <<HARDIRQ_BITS ) <NR_IRQS -# error HARDIRQ_BITS is too low! -#endif -#ifdef CONFIG_MMU static inline void ack_bad_irq (unsigned int irq)
arch/s390/include/asm/hardirq.h +0 -2
@@ -18,8 +18,6 @@#define __ARCH_HAS_DO_SOFTIRQ#define __ARCH_IRQ_EXIT_IRQS_DISABLED -#define HARDIRQ_BITS 8 -static inline void ack_bad_irq (unsigned int irq) {printk (KERN_CRIT "unexpected IRQ trap at vector %02x\n" , irq);
arch/sparc/include/asm/hardirq_32.h +0 -1
@@ -7,7 +7,6 @@#ifndef __SPARC_HARDIRQ_H#define __SPARC_HARDIRQ_H -#define HARDIRQ_BITS 8#include <asm-generic/hardirq.h> #endif /* __SPARC_HARDIRQ_H */
arch/sparc/include/asm/hardirq_64.h +0 -2
@@ -14,6 +14,4 @@void ack_bad_irq (unsigned int irq); -#define HARDIRQ_BITS 8 -#endif /* !(__SPARC64_HARDIRQ_H) */
arch/tile/include/asm/hardirq.h +0 -2
@@ -42,6 +42,4 @@ DECLARE_PER_CPU(irq_cpustat_t, irq_stat);#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ -#define HARDIRQ_BITS 8 -#endif /* _ASM_TILE_HARDIRQ_H */
include/linux/preempt_mask.h +8 -22
@@ -11,36 +11,22 @@* - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) * -* The hardirq count can in theory reach the same as NR_IRQS. -* In reality, the number of nested IRQS is limited to the stack -* size as well. For archs with over 1000 IRQS it is not practical -* to expect that they will all nest. We give a max of 10 bits for -* hardirq nesting. An arch may choose to give less than 10 bits. -* m68k expects it to be 8. -* -* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024) -* - bit 26 is the NMI_MASK -* - bit 27 is the PREEMPT_ACTIVE flag +* The hardirq count could in theory be the same as the number of +* interrupts in the system, but we run all interrupt handlers with +* interrupts disabled, so we cannot have nesting interrupts. Though +* there are a few palaeontologic drivers which reenable interrupts in +* the handler, so we need more than one bit here. * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x03ff0000 -* NMI_MASK: 0x04000000 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 +#define HARDIRQ_BITS 4#define NMI_BITS 1 -#define MAX_HARDIRQ_BITS 10 - -#ifndef HARDIRQ_BITS -# define HARDIRQ_BITS MAX_HARDIRQ_BITS -#endif - -#if HARDIRQ_BITS >MAX_HARDIRQ_BITS -#error HARDIRQ_BITS too high! -#endif -#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
PATCH 2/6 h8300: Use schedule_preempt_irq
In reply to Thomas GleixnerUse the proper function instead of fiddling with PREEMPT_ACTIVE and interrupt enable/disable in the low level code.
--- arch/h8300/kernel/entry.S | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
arch/h8300/kernel/entry.S +1 -5
@@ -316,13 +316,9 @@ need_resched: beq restore_all:8 mov.b @(LCCR+1:16,sp),r0l /* Interrupt Enabled? */ bmi restore_all:8 - mov.l #PREEMPT_ACTIVE,er0 - mov.l er0,@(TI_PRE_COUNT:16,er4) - andc #0x7f,ccr mov.l sp,er0 jsr @_set_esp0 - jsr @_schedule - orc #0x80,ccr + jsr @_preempt_schedule_irq bra need_resched:8 #endif
Re: [patch 2/6] h8300: Use schedule_preempt_irq
Quoted 7 lines · Thomas Gleixner
Use the proper function instead of fiddling with
PREEMPT_ACTIVEand interrupt enable/disable in the low level code.Signed-off-by Thomas Gleixner <[email protected]>Cc Yoshinori Sato <[email protected]>Cc Geert Uytterhoeven <[email protected]>
This will result in a conflict in -next as h8300 support has been removed there, and I'll send a pull request to Linus in the next commit window.
Guenter
Re: [patch 2/6] h8300: Use schedule_preempt_irq
Quoted 10 lines · Guenter Roeck
Quoted 7 lines · Thomas Gleixner
Use the proper function instead of fiddling with
PREEMPT_ACTIVEand interrupt enable/disable in the low level code.Signed-off-by Thomas Gleixner <[email protected]>Cc Yoshinori Sato <[email protected]>Cc Geert Uytterhoeven <[email protected]>This will result in a conflict in -next as h8300 support has been removed there, and I'll send a pull request to Linus in the next commit window.
An easy to resolve one methinks :)
PATCH 3/6 m32r: Use preempt_schedule_irq
In reply to Thomas GleixnerUse the proper core function instead of fiddling with preempt_active and interrupt enable in the low level code.
--- arch/m32r/kernel/entry.S | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
arch/m32r/kernel/entry.S +1 -7
@@ -182,13 +182,7 @@ need_resched: ld r4, PSW(sp) ; interrupts off (exception path) ? and3 r4, r4, #0x4000 beqz r4, restore_all - LDIMM (r4, PREEMPT_ACTIVE) - st r4, @(TI_PRE_COUNT, r8) - ENABLE_INTERRUPTS(r4) - bl schedule - ldi r4, #0 - st r4, @(TI_PRE_COUNT, r8) - DISABLE_INTERRUPTS(r4) + bl preempt_schedule_irq bra need_resched #endif
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] m32r: Use preempt_schedule_irq
m32r: Use preempt_schedule_irq
Commit-ID: 650e4dc2a75959fdca1eecc0147bbb21dbfadf0f Gitweb: http://git.kernel.org/tip/650e4dc2a75959fdca1eecc0147bbb21dbfadf0f Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Sep 2013 18:53:07 +0000 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:46 +0100
m32r: Use preempt_schedule_irq
Use the proper core function instead of fiddling with preempt_active and interrupt enable in the low level code.
--- arch/m32r/kernel/entry.S | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
arch/m32r/kernel/entry.S +1 -7
@@ -182,13 +182,7 @@ need_resched: ld r4, PSW(sp) ; interrupts off (exception path) ? and3 r4, r4, #0x4000 beqz r4, restore_all - LDIMM (r4, PREEMPT_ACTIVE) - st r4, @(TI_PRE_COUNT, r8) - ENABLE_INTERRUPTS(r4) - bl schedule - ldi r4, #0 - st r4, @(TI_PRE_COUNT, r8) - DISABLE_INTERRUPTS(r4) + bl preempt_schedule_irq bra need_resched #endif
PATCH 4/6 ia64: Use preempt_schedule_irq
In reply to Thomas GleixnerUse the proper core function instead of fiddling with PREEMPT_ACTIVE and enable/disable interrupts in the low level code.
--- arch/ia64/kernel/entry.S | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-)
arch/ia64/kernel/entry.S +1 -14
@@ -1169,21 +1169,8 @@ skip_rbs_switch: .work_pending: tbit.z p6,p0=r31,TIF_NEED_RESCHED // is resched not needed? (p6) br.cond.sptk.few .notify -#ifdef CONFIG_PREEMPT -(pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1 - ;; -(pKStk) st4 [r20]=r21 -#endif - SSM_PSR_I(p0, p6, r2) // enable interrupts - br.call.spnt.many rp=schedule + br.call.spnt.many rp=preempt_schedule_irq .ret9: cmp.eq p6,p0=r0,r0 // p6 <- 1 (re-check) - RSM_PSR_I(p0, r2, r20) // disable interrupts - ;; -#ifdef CONFIG_PREEMPT -(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 - ;; -(pKStk) st4 [r20]=r0 // preempt_count() <- 0 -#endif (pLvSys)br.cond.sptk.few __paravirt_pending_syscall_end br.cond.sptk.many .work_processed_kernel
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] ia64: Use preempt_schedule_irq
ia64: Use preempt_schedule_irq
Commit-ID: aa0d53260596c1bef23547537724d4bad78e6f52 Gitweb: http://git.kernel.org/tip/aa0d53260596c1bef23547537724d4bad78e6f52 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Sep 2013 18:53:08 +0000 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:47 +0100
ia64: Use preempt_schedule_irq
Use the proper core function instead of fiddling with PREEMPT_ACTIVE and enable/disable interrupts in the low level code.
--- arch/ia64/kernel/entry.S | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-)
arch/ia64/kernel/entry.S +1 -14
@@ -1169,21 +1169,8 @@ skip_rbs_switch: .work_pending: tbit.z p6,p0=r31,TIF_NEED_RESCHED // is resched not needed? (p6) br.cond.sptk.few .notify -#ifdef CONFIG_PREEMPT -(pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1 - ;; -(pKStk) st4 [r20]=r21 -#endif - SSM_PSR_I(p0, p6, r2) // enable interrupts - br.call.spnt.many rp=schedule + br.call.spnt.many rp=preempt_schedule_irq .ret9: cmp.eq p6,p0=r0,r0 // p6 <- 1 (re-check) - RSM_PSR_I(p0, r2, r20) // disable interrupts - ;; -#ifdef CONFIG_PREEMPT -(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 - ;; -(pKStk) st4 [r20]=r0 // preempt_count() <- 0 -#endif (pLvSys)br.cond.sptk.few __paravirt_pending_syscall_end br.cond.sptk.many .work_processed_kernel
Re: [patch 4/6] ia64: Use preempt_schedule_irq
In reply to Thomas GleixnerQuoted 7 lines · Thomas Gleixner
Use the proper core function instead of fiddling with
PREEMPT_ACTIVEand enable/disable interrupts in the low level code.Index:
linux-2.6/arch/ia64/kernel/entry.S
===================================================================arch/ia64/kernel/entry.S +0
-0
...
arch/ia64/kernel/entry.S skip_rbs_switch: · line 1172
Quoted 2 lines
arch/ia64/kernel/entry.S +1
-1- br.call.spnt.many rp=schedule + br.call.spnt.many rp=preempt_schedule_irq
This just hit mainline ... and I'm getting
arch/ia64/kernel/built-in.o: In function `skip_rbs_switch':
(.text+0xef2): undefined reference to `preempt_schedule_irq'
make: *** [vmlinux] Error 1
Probably because I have CONFIG_PREEMPT=n, and there seems only to be a definition for this function in kernel/sched/core.c when it is =y.
Do we need a stub for the =n case:
asmlinkage void __sched preempt_schedule_irq(void)
{
schedule();
}Or is life more complicated than that?
-Tony
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 9 lines · Tony Luck
Quoted 7 lines · Thomas Gleixner
Use the proper core function instead of fiddling with
PREEMPT_ACTIVEand enable/disable interrupts in the low level code.Index:
linux-2.6/arch/ia64/kernel/entry.S
===================================================================arch/ia64/kernel/entry.S +0
-0...
Quoted 8 lines
arch/ia64/kernel/entry.S skip_rbs_switch: · line 1172
Quoted 2 lines
arch/ia64/kernel/entry.S +1
-1- br.call.spnt.many rp=schedule + br.call.spnt.many rp=preempt_schedule_irqThis just hit mainline ... and I'm getting
arch/ia64/kernel/built-in.o: In function `skip_rbs_switch':
(.text+0xef2): undefined reference to `preempt_schedule_irq'
make: *** [vmlinux] Error 1
Hrmpf.
Quoted 2 lines
Probably because I have
CONFIG_PREEMPT=n, and there seems only to be a definition for this function inkernel/sched/core.cwhen it is =y.
Bah. I probably compiled that with preempt=y ...
Quoted 8 lines
Do we need a stub for the =n case:
asmlinkage void
__schedpreempt_schedule_irq(void){ schedule(); }Or is life more complicated than that?
Hmm, I think I fubared that and you decided to ignore my patch :)
Let me look at it tomorrow morning with full awake brain cells.
Thanks,
tglx
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 11 lines · Thomas Gleixner
Quoted 6 lines · Tony Luck
asmlinkage void
__schedpreempt_schedule_irq(void){ schedule(); }Or is life more complicated than that?
Hmm, I think I fubared that and you decided to ignore my patch :)
Let me look at it tomorrow morning with full awake brain cells.
Ok, wrapped my brain around it. I tripped over the magic asm foo which has a single need_resched check and schedule point for both sys call return and interrupt return.
So you need the schedule_preempt_irq() for kernel preemption from interrupt return while on a normal syscall preemption a schedule would be sufficient. But using schedule_preempt_irq() is not harmful here in any way. It just sets the preempt_active bit also in cases where it would not be required.
Even on preempt=n kernels adding the preempt_active bit is completely harmless. So instead of having an extra function, moving the existing one out of the ifdef PREEMPT looks like the sanest thing to do.
Peter, Ingo ?
Thanks,
tglx
------
kernel/sched/core.c +1 -2
@@ -2660,6 +2660,7 @@ asmlinkage void __sched notrace preempt_schedule(void) } while (need_resched()); }EXPORT_SYMBOL (preempt_schedule); +#endif /* CONFIG_PREEMPT */ /* * this is the entry point to schedule() from kernel preemption
@@ -2693,8 +2694,6 @@ asmlinkage void __sched preempt_schedule_irq(void)exception_exit (prev_state); } -#endif /* CONFIG_PREEMPT */ -int default_wake_function (wait_queue_t *curr,unsigned mode,int wake_flags,void *key) {
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 23 lines · Thomas Gleixner
Quoted 11 lines · Thomas Gleixner
Quoted 6 lines · Tony Luck
asmlinkage void
__schedpreempt_schedule_irq(void){ schedule(); }Or is life more complicated than that?
Hmm, I think I fubared that and you decided to ignore my patch :)
Let me look at it tomorrow morning with full awake brain cells.
Ok, wrapped my brain around it. I tripped over the magic asm foo which has a single
need_reschedcheck and schedule point for both sys call return and interrupt return.So you need the
schedule_preempt_irq()for kernel preemption from interrupt return while on a normal syscall preemption a schedule would be sufficient. But usingschedule_preempt_irq()is not harmful here in any way. It just sets thepreempt_activebit also in cases where it would not be required.
Quoted 5 lines
Even on preempt=n kernels adding the
preempt_activebit is completely harmless. So instead of having an extra function, moving the existing one out of the ifdef PREEMPT looks like the sanest thing to do.Peter, Ingo ?
So that's because the schedule point is on a common user and kernel return path?
If necessary, why not having a SCHEDULE_IRQ macro in ia64 that maps to either schedule or preempt_schedule_irq() instead?
Unless that problem happens elsewhere as well?
Peter Zijlstra
maintainerRe: [patch 4/6] ia64: Use preempt_schedule_irq
In reply to Thomas GleixnerQuoted 23 lines · Thomas Gleixner
Quoted 11 lines · Thomas Gleixner
Quoted 6 lines · Tony Luck
asmlinkage void
__schedpreempt_schedule_irq(void){ schedule(); }Or is life more complicated than that?
Hmm, I think I fubared that and you decided to ignore my patch :)
Let me look at it tomorrow morning with full awake brain cells.
Ok, wrapped my brain around it. I tripped over the magic asm foo which has a single
need_reschedcheck and schedule point for both sys call return and interrupt return.So you need the
schedule_preempt_irq()for kernel preemption from interrupt return while on a normal syscall preemption a schedule would be sufficient. But usingschedule_preempt_irq()is not harmful here in any way. It just sets thepreempt_activebit also in cases where it would not be required.
Quoted 5 lines
Even on preempt=n kernels adding the
preempt_activebit is completely harmless. So instead of having an extra function, moving the existing one out of the ifdef PREEMPT looks like the sanest thing to do.Peter, Ingo ?
Uhm, preempt_schedule_irq() assumes interrupts are disabled and explicitly returns with interrupts disabled again. Does the ia64 callsite conform?
If so, schedule() would actually be actively wrong, because that will whinge when called with interrupts disabled, and will return with interrupts enabled.
Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 28 lines · Peter Zijlstra
Quoted 27 lines · Thomas Gleixner
Quoted 11 lines · Thomas Gleixner
Quoted 6 lines · Tony Luck
asmlinkage void
__schedpreempt_schedule_irq(void){ schedule(); }Or is life more complicated than that?
Hmm, I think I fubared that and you decided to ignore my patch :)
Let me look at it tomorrow morning with full awake brain cells.
Ok, wrapped my brain around it. I tripped over the magic asm foo which has a single
need_reschedcheck and schedule point for both sys call return and interrupt return.So you need the
schedule_preempt_irq()for kernel preemption from interrupt return while on a normal syscall preemption a schedule would be sufficient. But usingschedule_preempt_irq()is not harmful here in any way. It just sets thepreempt_activebit also in cases where it would not be required.Even on preempt=n kernels adding the
preempt_activebit is completely harmless. So instead of having an extra function, moving the existing one out of the ifdef PREEMPT looks like the sanest thing to do.
Quoted 5 lines
Quoted 1 line
Peter, Ingo ?
Uhm,
preempt_schedule_irq()assumes interrupts are disabled and explicitly returns with interrupts disabled again. Does the ia64 callsite conform?
Yep.
Quoted 6 lines
If so,
schedule()would actually be actively wrong, because that will whinge when called with interrupts disabled, and will return with interrupts enabled.Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
Well, we could get rid of quite some other sti/schedule/cli asm magic all over the archs.
Thanks,
tglx
Tony Luck
Tested-byRe: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 3 lines · Thomas Gleixner
Quoted 2 lines · Peter Zijlstra
Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
I tried it out ... if does fix both build & boot for ia64 with PREMPT=n and PREMPT=y
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 9 lines · Tony Luck
Quoted 3 lines · Thomas Gleixner
Quoted 2 lines · Peter Zijlstra
Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
I tried it out ... if does fix both build & boot for ia64 with PREMPT=n and PREMPT=y
Tested-by Tony Luck <[email protected]>
So - are we going with this patch? Or still thinking of something better?
-Tony
Re: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 6 lines · Tony Luck
Quoted 5 lines · Tony Luck
Quoted 3 lines · Thomas Gleixner
Quoted 2 lines · Peter Zijlstra
Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
Quoted 6 lines
Quoted 4 lines
I tried it out ... if does fix both build & boot for ia64 with PREMPT=n and PREMPT=y
Tested-by Tony Luck <[email protected]>So - are we going with this patch? Or still thinking of something better?
I queued it; let me prod Ingo (again) into actually merging it.
Ingo Molnar
AppliedRe: [patch 4/6] ia64: Use preempt_schedule_irq
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Tony Luck
Quoted 5 lines · Tony Luck
Quoted 3 lines · Thomas Gleixner
Quoted 2 lines · Peter Zijlstra
Anyway, I don't object to the patch per se, but it might bloat a few !ia64 kernels for having to carry the extra text.
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
I tried it out ... if does fix both build & boot for ia64 with PREMPT=n and PREMPT=y
Tested-by Tony Luck <[email protected]>So - are we going with this patch? Or still thinking of something better?
I queued it; let me prod Ingo (again) into actually merging it.
I merged it from you today, will try to get it to Linus later today (hopefully).
Thanks,
Ingo
tip-bot for Thomas Gleixner
merge bot[tip:sched/urgent] sched: Expose preempt_schedule_irq()
In reply to Thomas Gleixnersched: Expose preempt_schedule_irq()
Commit-ID: 32e475d76a3e40879cd9ee4f69b19615062280d7 Gitweb: http://git.kernel.org/tip/32e475d76a3e40879cd9ee4f69b19615062280d7 Author: Thomas Gleixner <[email protected]> AuthorDate: Thu, 21 Nov 2013 12:41:44 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 27 Nov 2013 11:04:53 +0100
sched: Expose preempt_schedule_irq()
Tony reported that aa0d53260596 ("ia64: Use preempt_schedule_irq") broke PREEMPT=n builds on ia64.
Ok, wrapped my brain around it. I tripped over the magic asm foo which has a single need_resched check and schedule point for both sys call return and interrupt return.
So you need the schedule_preempt_irq() for kernel preemption from interrupt return while on a normal syscall preemption a schedule would be sufficient. But using schedule_preempt_irq() is not harmful here in any way. It just sets the preempt_active bit also in cases where it would not be required.
Even on preempt=n kernels adding the preempt_active bit is completely harmless. So instead of having an extra function, moving the existing one out of the ifdef PREEMPT looks like the sanest thing to do.
It would also allow getting rid of various other sti/schedule/cli asm magic in other archs.
--- kernel/sched/core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/sched/core.c +1 -2
@@ -2660,6 +2660,7 @@ asmlinkage void __sched notrace preempt_schedule(void) } while (need_resched()); }EXPORT_SYMBOL (preempt_schedule); +#endif /* CONFIG_PREEMPT */ /* * this is the entry point to schedule() from kernel preemption
@@ -2693,8 +2694,6 @@ asmlinkage void __sched preempt_schedule_irq(void)exception_exit (prev_state); } -#endif /* CONFIG_PREEMPT */ -int default_wake_function (wait_queue_t *curr,unsigned mode,int wake_flags,void *key) {
PATCH 5/6 sparc: Use preempt_schedule_irq
In reply to Thomas GleixnerThe low level preemption code fiddles with the PREEMPT_ACTIVE bit for no reason and calls schedule() with interrupts disabled, which is wrong to begin with. Remove the PREEMPT_ACTIVE fiddling and call the proper schedule_preempt_irq() function.
--- arch/sparc/kernel/rtrap_64.S | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-)
arch/sparc/kernel/rtrap_64.S +1 -4
@@ -306,12 +306,9 @@ to_kernel: nop cmp %l4, 0 bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + call preempt_schedule_irq nop ba,pt %xcc, rtrap - stw %g0, [%g6 + TI_PRE_COUNT] #endif kern_fpucheck: ldub [%g6 + TI_FPDEPTH], %l5 brz,pt %l5, rt_continue
David Miller
maintainerRe: [patch 5/6] sparc: Use preempt_schedule_irq
From: Thomas Gleixner <[email protected]>
Date: Tue, 17 Sep 2013 18:53:08 -0000
arch/sparc/kernel/rtrap_64.S to_kernel: · line 310
Quoted 6 lines
arch/sparc/kernel/rtrap_64.S +1
-3bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + call preempt_schedule_irq nop
You've put the function call into the delay slot of the branch, which you don't want to do.
That's, btw, why we indent instructions with an extra space like that, to emphasize that it's a delay slot of the preceeding branch.
Re: [patch 5/6] sparc: Use preempt_schedule_irq
Quoted 13 lines · David Miller
From: Thomas Gleixner <[email protected]>
Date: Tue, 17 Sep 2013 18:53:08 -0000arch/sparc/kernel/rtrap_64.S to_kernel: · line 310
Quoted 6 lines
arch/sparc/kernel/rtrap_64.S +1
-3bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + call preempt_schedule_irq nopYou've put the function call into the delay slot of the branch, which you don't want to do.
Quoted 3 lines
That's, btw, why we indent instructions with an extra space like that, to emphasize that it's a delay slot of the preceeding branch.
I knew that I'd get it wrong :) So is adding another nop the right thing to do ?
bne,pn %xcc, kern_fpucheck
- sethi %hi(PREEMPT_ACTIVE), %l6
- stw %l6, [%g6 + TI_PRE_COUNT]
- call schedule
+ nop
+ call preempt_schedule_irq
nop
Thanks,
tglx
Re: [patch 5/6] sparc: Use preempt_schedule_irq
From: Thomas Gleixner <[email protected]>
Date: Wed, 18 Sep 2013 01:23:58 +0200 (CEST)
arch/sparc/kernel/rtrap_64.S to_kernel: · line 310
Quoted 10 lines
I knew that I'd get it wrong :) So is adding another nop the right thing to do ?
arch/sparc/kernel/rtrap_64.S +2
-3bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + nop + call preempt_schedule_irq nop
Yep, that's perfect:
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] sparc: Use preempt_schedule_irq
In reply to Thomas Gleixnersparc: Use preempt_schedule_irq
Commit-ID: 9385d949d5bd0eb642ed05ea263c3638c9f4e372 Gitweb: http://git.kernel.org/tip/9385d949d5bd0eb642ed05ea263c3638c9f4e372 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Sep 2013 18:53:08 +0000 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:47 +0100
sparc: Use preempt_schedule_irq
The low level preemption code fiddles with the PREEMPT_ACTIVE bit for no reason and calls schedule() with interrupts disabled, which is wrong to begin with. Remove the PREEMPT_ACTIVE fiddling and call the proper schedule_preempt_irq() function.
--- arch/sparc/kernel/rtrap_64.S | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
arch/sparc/kernel/rtrap_64.S +2 -4
@@ -306,12 +306,10 @@ to_kernel: nop cmp %l4, 0 bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + nop + call preempt_schedule_irq nop ba,pt %xcc, rtrap - stw %g0, [%g6 + TI_PRE_COUNT] #endif kern_fpucheck: ldub [%g6 + TI_FPDEPTH], %l5 brz,pt %l5, rt_continue
PATCH 6/6 preempt: Make PREEMPT_ACTIVE generic
In reply to Thomas GleixnerNo point in having this bit defined by architecture.
--- arch/alpha/include/asm/thread_info.h | 2 -- arch/arc/include/asm/thread_info.h | 2 -- arch/arm/include/asm/thread_info.h | 6 ------ arch/arm64/include/asm/thread_info.h | 6 ------ arch/avr32/include/asm/thread_info.h | 2 -- arch/blackfin/include/asm/thread_info.h | 2 -- arch/c6x/include/asm/thread_info.h | 2 -- arch/cris/include/asm/thread_info.h | 2 -- arch/frv/include/asm/thread_info.h | 2 -- arch/h8300/include/asm/thread_info.h | 2 -- arch/hexagon/include/asm/thread_info.h | 4 ---- arch/ia64/include/asm/thread_info.h | 3 --- arch/m32r/include/asm/thread_info.h | 2 -- arch/m68k/include/asm/thread_info.h | 2 -- arch/metag/include/asm/thread_info.h | 2 -- arch/microblaze/include/asm/thread_info.h | 2 -- arch/mips/include/asm/thread_info.h | 2 -- arch/mn10300/include/asm/thread_info.h | 2 -- arch/parisc/include/asm/thread_info.h | 3 --- arch/powerpc/include/asm/thread_info.h | 2 -- arch/s390/include/asm/thread_info.h | 2 -- arch/score/include/asm/thread_info.h | 2 -- arch/sh/include/asm/thread_info.h | 2 -- arch/sh/kernel/entry-common.S | 6 ++---- arch/sparc/include/asm/thread_info_32.h | 2 -- arch/sparc/include/asm/thread_info_64.h | 2 -- arch/tile/include/asm/thread_info.h | 2 -- arch/um/include/asm/thread_info.h | 2 -- arch/unicore32/include/asm/thread_info.h | 6 ------ arch/x86/include/asm/thread_info.h | 2 -- arch/xtensa/include/asm/thread_info.h | 2 -- include/linux/preempt_mask.h | 15 +++++---------- 32 files changed, 7 insertions(+), 90 deletions(-)
arch/alpha/include/asm/thread_info.h +0 -2
@@ -58,8 +58,6 @@ register struct thread_info *__current_t#define THREAD_SIZE_ORDER 1#define THREAD_SIZE (2*PAGE_SIZE) -#define PREEMPT_ACTIVE 0x40000000 -/* * Thread information flags: * - these are process state flags and used from assembly
arch/arc/include/asm/thread_info.h +0 -2
@@ -80,8 +80,6 @@ static inline __attribute_const__ struct#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/arm/include/asm/thread_info.h +0 -6
@@ -141,12 +141,6 @@ extern int vfp_restore_user_hwstate(stru#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SYSCAL_AUDIT - syscall auditing active
arch/arm64/include/asm/thread_info.h +0 -6
@@ -89,12 +89,6 @@ static inline struct thread_info *curren#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SIGPENDING - signal pending
arch/avr32/include/asm/thread_info.h +0 -2
@@ -66,8 +66,6 @@ static inline struct thread_info *curren#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x40000000 -/* * Thread information flags * - these are process state flags that various assembly files may need to access
arch/blackfin/include/asm/thread_info.h +0 -2
@@ -88,8 +88,6 @@ static inline struct thread_info *curren#define TI_CPU 12#define TI_PREEMPT 16 -#define PREEMPT_ACTIVE 0x4000000 -/* * thread information flag bit numbers */
arch/c6x/include/asm/thread_info.h +0 -2
@@ -84,8 +84,6 @@ struct thread_info *current_thread_info(#define put_thread_info (ti) put_task_struct((ti)->task)#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flag bit numbers * - pending work-to-be-done flags are in LSW
arch/cris/include/asm/thread_info.h +0 -2
@@ -44,8 +44,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/frv/include/asm/thread_info.h +0 -2
@@ -52,8 +52,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/h8300/include/asm/thread_info.h +0 -2
@@ -77,8 +77,6 @@ static inline struct thread_info *curren#define TI_CPU 12#define TI_PRE_COUNT 16 -#define PREEMPT_ACTIVE 0x4000000 -/* * thread information flag bit numbers */
arch/hexagon/include/asm/thread_info.h +0 -4
@@ -73,10 +73,6 @@ struct thread_info {#endif /* __ASSEMBLY__ */ -/* looks like "linux/hardirq.h" uses this. */ - -#define PREEMPT_ACTIVE 0x10000000 -#ifndef __ASSEMBLY__#define INIT_THREAD_INFO (tsk) \
arch/ia64/include/asm/thread_info.h +0 -3
@@ -11,9 +11,6 @@#include <asm/processor.h> #include <asm/ptrace.h> -#define PREEMPT_ACTIVE_BIT 30 -#define PREEMPT_ACTIVE (1 << PREEMPT_ACTIVE_BIT) -#ifndef __ASSEMBLY__/*
arch/m32r/include/asm/thread_info.h +0 -2
@@ -53,8 +53,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#define THREAD_SIZE (PAGE_SIZE << 1)#define THREAD_SIZE_ORDER 1/*
arch/m68k/include/asm/thread_info.h +0 -2
@@ -35,8 +35,6 @@ struct thread_info { };#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x4000000 -#define INIT_THREAD_INFO (tsk) \ { \ .task = &tsk, \
arch/metag/include/asm/thread_info.h +0 -2
@@ -46,8 +46,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_4KSTACKS #define THREAD_SHIFT 12#else
arch/microblaze/include/asm/thread_info.h +0 -2
@@ -106,8 +106,6 @@ static inline struct thread_info *curren/* thread information allocation */ #endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may
arch/mips/include/asm/thread_info.h +0 -2
@@ -92,8 +92,6 @@ static inline struct thread_info *curren#define STACK_WARN (THREAD_SIZE / 8) -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/mn10300/include/asm/thread_info.h +0 -2
@@ -16,8 +16,6 @@#include <asm/page.h> -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_4KSTACKS #define THREAD_SIZE (4096)#define THREAD_SIZE_ORDER (0)
arch/parisc/include/asm/thread_info.h +0 -3
@@ -46,9 +46,6 @@ struct thread_info {#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)#define THREAD_SHIFT (PAGE_SHIFT + THREAD_SIZE_ORDER) -#define PREEMPT_ACTIVE_BIT 28 -#define PREEMPT_ACTIVE (1 << PREEMPT_ACTIVE_BIT) -/* * thread information flags */
arch/powerpc/include/asm/thread_info.h +0 -2
@@ -82,8 +82,6 @@ static inline struct thread_info *curren#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flag bit numbers */
arch/s390/include/asm/thread_info.h +0 -2
@@ -111,6 +111,4 @@ static inline struct thread_info *curren#define is_32bit_task () (1)#endif -#define PREEMPT_ACTIVE 0x4000000 -#endif /* _ASM_THREAD_INFO_H */
arch/score/include/asm/thread_info.h +0 -2
@@ -72,8 +72,6 @@ register struct thread_info *__current_t#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/sh/include/asm/thread_info.h +0 -2
@@ -41,8 +41,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#if defined(CONFIG_4KSTACKS )#define THREAD_SHIFT 12#else
arch/sh/kernel/entry-common.S +2 -4
@@ -108,7 +108,7 @@ need_resched: and #(0xf0>>1), r0 ! interrupts off (exception path)? cmp/eq #(0xf0>>1), r0 bt noresched - mov.l 3f, r0 + mov.l 1f, r0 jsr @r0 ! call preempt_schedule_irq nop bra need_resched
@@ -119,9 +119,7 @@ noresched: nop .align 2 -1: .long PREEMPT_ACTIVE -2: .long schedule -3: .long preempt_schedule_irq +1: .long preempt_schedule_irq #endif ENTRY(resume_userspace)
arch/sparc/include/asm/thread_info_32.h +0 -2
@@ -105,8 +105,6 @@ register struct thread_info *current_thr#define TI_W_SAVED 0x250/* #define TI_RESTART_BLOCK 0x25n */ /* Nobody cares */ -#define PREEMPT_ACTIVE 0x4000000 -/* * thread information flag bit numbers */
arch/sparc/include/asm/thread_info_64.h +0 -2
@@ -111,8 +111,6 @@ struct thread_info {#define THREAD_SHIFT PAGE_SHIFT#endif /* PAGE_SHIFT == 13 */ -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/tile/include/asm/thread_info.h +0 -2
@@ -113,8 +113,6 @@ extern void _cpu_idle(void);#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * Thread information flags that various assembly files may need to access. * Keep flags accessed frequently in low bits, particular since it makes
arch/um/include/asm/thread_info.h +0 -2
@@ -60,8 +60,6 @@ static inline struct thread_info *curren#endif -#define PREEMPT_ACTIVE 0x10000000 -#define TIF_SYSCALL_TRACE 0/* syscall trace active */ #define TIF_SIGPENDING 1/* signal pending */ #define TIF_NEED_RESCHED 2/* rescheduling necessary */
arch/unicore32/include/asm/thread_info.h +0 -6
@@ -118,12 +118,6 @@ static inline struct thread_info *curren#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SIGPENDING - signal pending
arch/x86/include/asm/thread_info.h +0 -2
@@ -154,8 +154,6 @@ struct thread_info {#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW) -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_X86_32 #define STACK_WARN (THREAD_SIZE/8)
arch/xtensa/include/asm/thread_info.h +0 -2
@@ -76,8 +76,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
include/linux/preempt_mask.h +5 -10
@@ -17,10 +17,11 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8
@@ -46,15 +47,9 @@#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -#ifndef PREEMPT_ACTIVE #define PREEMPT_ACTIVE_BITS 1#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) -#endif - -#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT +NMI_BITS )) -#error PREEMPT_ACTIVE is too low! -#endif #define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)
Peter Zijlstra
maintainerRe: [patch 6/6] preempt: Make PREEMPT_ACTIVE generic
Quoted 28 lines · Thomas Gleixner
No point in having this bit defined by architecture.
Index:
linux-2.6/include/linux/preempt_mask.h
===================================================================include/linux/preempt_mask.h +5
-5@@ -17,10 +17,11 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8@@ -46,15 +47,9 @@#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -#ifndef PREEMPT_ACTIVE #define PREEMPT_ACTIVE_BITS 1
include/linux/preempt_mask.h line 55
Quoted 10 lines
include/linux/preempt_mask.h +0
-5#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) -#endif - -#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT +NMI_BITS )) -#error PREEMPT_ACTIVE is too low! -#endif #define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)
I get:
init/init_task.c:26:4: error: ‘PREEMPT_ACTIVE’ undeclared here (not in a function)
Because for some obscure reason sched.h doesn't actually include enough.
The below cures things:
include/linux/sched.h +1 -0
@@ -22,6 +22,7 @@ struct sched_param {#include <linux/errno.h> #include <linux/nodemask.h> #include <linux/mm_types.h> +#include <linux/preempt_mask.h> #include <asm/page.h> #include <asm/ptrace.h>
tip-bot for Thomas Gleixner
merge bot[tip:irq/urgent] preempt: Make PREEMPT_ACTIVE generic
In reply to Thomas Gleixnerpreempt: Make PREEMPT_ACTIVE generic
Commit-ID: 00d1a39e69d5afa7523dad515a05b21abd17c389 Gitweb: http://git.kernel.org/tip/00d1a39e69d5afa7523dad515a05b21abd17c389 Author: Thomas Gleixner <[email protected]> AuthorDate: Tue, 17 Sep 2013 18:53:09 +0000 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 13 Nov 2013 20:21:47 +0100
preempt: Make PREEMPT_ACTIVE generic
No point in having this bit defined by architecture.
--- arch/alpha/include/asm/thread_info.h | 2 -- arch/arc/include/asm/thread_info.h | 2 -- arch/arm/include/asm/thread_info.h | 6 ------ arch/arm64/include/asm/thread_info.h | 6 ------ arch/avr32/include/asm/thread_info.h | 2 -- arch/blackfin/include/asm/thread_info.h | 2 -- arch/c6x/include/asm/thread_info.h | 2 -- arch/cris/include/asm/thread_info.h | 2 -- arch/frv/include/asm/thread_info.h | 2 -- arch/hexagon/include/asm/thread_info.h | 4 ---- arch/ia64/include/asm/thread_info.h | 3 --- arch/m32r/include/asm/thread_info.h | 2 -- arch/m68k/include/asm/thread_info.h | 2 -- arch/metag/include/asm/thread_info.h | 2 -- arch/microblaze/include/asm/thread_info.h | 2 -- arch/mips/include/asm/thread_info.h | 2 -- arch/mn10300/include/asm/thread_info.h | 2 -- arch/parisc/include/asm/thread_info.h | 3 --- arch/powerpc/include/asm/thread_info.h | 2 -- arch/s390/include/asm/thread_info.h | 2 -- arch/score/include/asm/thread_info.h | 2 -- arch/sh/include/asm/thread_info.h | 2 -- arch/sh/kernel/entry-common.S | 6 ++---- arch/sparc/include/asm/thread_info_32.h | 2 -- arch/sparc/include/asm/thread_info_64.h | 2 -- arch/tile/include/asm/thread_info.h | 2 -- arch/um/include/asm/thread_info.h | 2 -- arch/unicore32/include/asm/thread_info.h | 6 ------ arch/x86/include/asm/thread_info.h | 2 -- arch/xtensa/include/asm/thread_info.h | 2 -- include/linux/preempt_mask.h | 15 +++++---------- include/linux/sched.h | 2 +- 32 files changed, 8 insertions(+), 89 deletions(-)
arch/alpha/include/asm/thread_info.h +0 -2
@@ -58,8 +58,6 @@ register struct thread_info *__current_thread_info __asm__("$8");#define THREAD_SIZE_ORDER 1#define THREAD_SIZE (2*PAGE_SIZE) -#define PREEMPT_ACTIVE 0x40000000 -/* * Thread information flags: * - these are process state flags and used from assembly
arch/arc/include/asm/thread_info.h +0 -2
@@ -80,8 +80,6 @@ static inline __attribute_const__ struct thread_info *current_thread_info(void)#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/arm/include/asm/thread_info.h +0 -6
@@ -141,12 +141,6 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *,#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SYSCAL_AUDIT - syscall auditing active
arch/arm64/include/asm/thread_info.h +0 -6
@@ -89,12 +89,6 @@ static inline struct thread_info *current_thread_info(void)#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SIGPENDING - signal pending
arch/avr32/include/asm/thread_info.h +0 -2
@@ -66,8 +66,6 @@ static inline struct thread_info *current_thread_info(void)#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x40000000 -/* * Thread information flags * - these are process state flags that various assembly files may need to access
arch/blackfin/include/asm/thread_info.h +0 -2
@@ -88,8 +88,6 @@ static inline struct thread_info *current_thread_info(void)#define TI_CPU 12#define TI_PREEMPT 16 -#define PREEMPT_ACTIVE 0x4000000 -/* * thread information flag bit numbers */
arch/c6x/include/asm/thread_info.h +0 -2
@@ -84,8 +84,6 @@ struct thread_info *current_thread_info(void)#define put_thread_info (ti) put_task_struct((ti)->task)#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flag bit numbers * - pending work-to-be-done flags are in LSW
arch/cris/include/asm/thread_info.h +0 -2
@@ -44,8 +44,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/frv/include/asm/thread_info.h +0 -2
@@ -52,8 +52,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/hexagon/include/asm/thread_info.h +0 -4
@@ -73,10 +73,6 @@ struct thread_info {#endif /* __ASSEMBLY__ */ -/* looks like "linux/hardirq.h" uses this. */ - -#define PREEMPT_ACTIVE 0x10000000 -#ifndef __ASSEMBLY__#define INIT_THREAD_INFO (tsk) \
arch/ia64/include/asm/thread_info.h +0 -3
@@ -11,9 +11,6 @@#include <asm/processor.h> #include <asm/ptrace.h> -#define PREEMPT_ACTIVE_BIT 30 -#define PREEMPT_ACTIVE (1 << PREEMPT_ACTIVE_BIT) -#ifndef __ASSEMBLY__/*
arch/m32r/include/asm/thread_info.h +0 -2
@@ -53,8 +53,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#define THREAD_SIZE (PAGE_SIZE << 1)#define THREAD_SIZE_ORDER 1/*
arch/m68k/include/asm/thread_info.h +0 -2
@@ -35,8 +35,6 @@ struct thread_info { };#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x4000000 -#define INIT_THREAD_INFO (tsk) \ { \ .task = &tsk, \
arch/metag/include/asm/thread_info.h +0 -2
@@ -46,8 +46,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_4KSTACKS #define THREAD_SHIFT 12#else
arch/microblaze/include/asm/thread_info.h +0 -2
@@ -106,8 +106,6 @@ static inline struct thread_info *current_thread_info(void)/* thread information allocation */ #endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may
arch/mips/include/asm/thread_info.h +0 -2
@@ -92,8 +92,6 @@ static inline struct thread_info *current_thread_info(void)#define STACK_WARN (THREAD_SIZE / 8) -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/mn10300/include/asm/thread_info.h +0 -2
@@ -16,8 +16,6 @@#include <asm/page.h> -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_4KSTACKS #define THREAD_SIZE (4096)#define THREAD_SIZE_ORDER (0)
arch/parisc/include/asm/thread_info.h +0 -3
@@ -46,9 +46,6 @@ struct thread_info {#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)#define THREAD_SHIFT (PAGE_SHIFT + THREAD_SIZE_ORDER) -#define PREEMPT_ACTIVE_BIT 28 -#define PREEMPT_ACTIVE (1 << PREEMPT_ACTIVE_BIT) -/* * thread information flags */
arch/powerpc/include/asm/thread_info.h +0 -2
@@ -82,8 +82,6 @@ static inline struct thread_info *current_thread_info(void)#endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flag bit numbers */
arch/s390/include/asm/thread_info.h +0 -2
@@ -111,6 +111,4 @@ static inline struct thread_info *current_thread_info(void)#define is_32bit_task () (1)#endif -#define PREEMPT_ACTIVE 0x4000000 -#endif /* _ASM_THREAD_INFO_H */
arch/score/include/asm/thread_info.h +0 -2
@@ -72,8 +72,6 @@ register struct thread_info *__current_thread_info __asm__("r28");#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * thread information flags * - these are process state flags that various assembly files may need to
arch/sh/include/asm/thread_info.h +0 -2
@@ -41,8 +41,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -#if defined(CONFIG_4KSTACKS )#define THREAD_SHIFT 12#else
arch/sh/kernel/entry-common.S +2 -4
@@ -108,7 +108,7 @@ need_resched: and #(0xf0>>1), r0 ! interrupts off (exception path)? cmp/eq #(0xf0>>1), r0 bt noresched - mov.l 3f, r0 + mov.l 1f, r0 jsr @r0 ! call preempt_schedule_irq nop bra need_resched
@@ -119,9 +119,7 @@ noresched: nop .align 2 -1: .long PREEMPT_ACTIVE -2: .long schedule -3: .long preempt_schedule_irq +1: .long preempt_schedule_irq #endif ENTRY(resume_userspace)
arch/sparc/include/asm/thread_info_32.h +0 -2
@@ -105,8 +105,6 @@ register struct thread_info *current_thread_info_reg asm("g6");#define TI_W_SAVED 0x250/* #define TI_RESTART_BLOCK 0x25n */ /* Nobody cares */ -#define PREEMPT_ACTIVE 0x4000000 -/* * thread information flag bit numbers */
arch/sparc/include/asm/thread_info_64.h +0 -2
@@ -111,8 +111,6 @@ struct thread_info {#define THREAD_SHIFT PAGE_SHIFT#endif /* PAGE_SHIFT == 13 */ -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
arch/tile/include/asm/thread_info.h +0 -2
@@ -113,8 +113,6 @@ extern void _cpu_idle(void);#endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x10000000 -/* * Thread information flags that various assembly files may need to access. * Keep flags accessed frequently in low bits, particular since it makes
arch/um/include/asm/thread_info.h +0 -2
@@ -60,8 +60,6 @@ static inline struct thread_info *current_thread_info(void)#endif -#define PREEMPT_ACTIVE 0x10000000 -#define TIF_SYSCALL_TRACE 0/* syscall trace active */ #define TIF_SIGPENDING 1/* signal pending */ #define TIF_NEED_RESCHED 2/* rescheduling necessary */
arch/unicore32/include/asm/thread_info.h +0 -6
@@ -118,12 +118,6 @@ static inline struct thread_info *current_thread_info(void)#endif /* -* We use bit 30 of the preempt_count to indicate that kernel -* preemption is occurring. See <asm/hardirq.h>. -*/ -#define PREEMPT_ACTIVE 0x40000000 - -/* * thread information flags: * TIF_SYSCALL_TRACE - syscall trace active * TIF_SIGPENDING - signal pending
arch/x86/include/asm/thread_info.h +0 -2
@@ -153,8 +153,6 @@ struct thread_info {#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW) -#define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_X86_32 #define STACK_WARN (THREAD_SIZE/8)
arch/xtensa/include/asm/thread_info.h +0 -2
@@ -76,8 +76,6 @@ struct thread_info {#endif -#define PREEMPT_ACTIVE 0x10000000 -/* * macros/functions for gaining access to the thread information structure */
include/linux/preempt_mask.h +5 -10
@@ -17,10 +17,11 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * -* PREEMPT_MASK: 0x000000ff -* SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 +* PREEMPT_MASK: 0x000000ff +* SOFTIRQ_MASK: 0x0000ff00 +* HARDIRQ_MASK: 0x000f0000 +* NMI_MASK: 0x00100000 +* PREEMPT_ACTIVE: 0x00200000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8
@@ -46,15 +47,9 @@#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -#ifndef PREEMPT_ACTIVE #define PREEMPT_ACTIVE_BITS 1#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) -#endif - -#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT +NMI_BITS )) -#error PREEMPT_ACTIVE is too low! -#endif #define hardirq_count () (preempt_count() & HARDIRQ_MASK)#define softirq_count () (preempt_count() & SOFTIRQ_MASK)
include/linux/sched.h +1 -1
@@ -22,7 +22,7 @@ struct sched_param {#include <linux/errno.h> #include <linux/nodemask.h> #include <linux/mm_types.h> -#include <linux/preempt.h> +#include <linux/preempt_mask.h> #include <asm/page.h> #include <asm/ptrace.h>