Preemption patch against ~2.5.60
OK, here's the latest preemption patch.
arch/ia64/Kconfig +18 -0
@@ -424,6 +424,18 @@ If you don't know what to do here, say N. +config PREEMPT + bool "Preemptible Kernel" + help + This option reduces the latency of the kernel when reacting to + real-time or interactive events by allowing a low priority process to + be preempted even if it is in kernel mode executing a system call. + This allows applications to run more reliably even when the system is + under load. + + Say Y here if you are building a kernel for a desktop, embedded + or real-time system. Say N if you are unsure. + config IA32_SUPPORT bool "Support running of Linux/x86 binaries" help
@@ -874,6 +886,12 @@ and certain other kinds of spinlock errors commonly made. This is best used in conjunction with the NMI watchdog so that spinlock deadlocks are also debuggable. + +config DEBUG_SPINLOCK_SLEEP + bool "Sleep-inside-spinlock checking" + help + If you say Y here, various routines which may sleep will become very + noisy if they are called with a spinlock held. config IA64_DEBUG_CMPXCHG bool "Turn on compare-and-exchange bug checking (slow!)"
arch/ia64/hp/sim/simserial.c +21 -21
149 lines not on this page · raw message
arch/ia64/ia32/ia32_support.c +2 -1
@@ -93,7 +93,7 @@ {unsigned long eflag, fsr, fcr, fir, fdr, csd, ssd, tssd;struct pt_regs *regs =ia64_task_regs (t); -int nr =smp_processor_id ();/* LDT and TSS depend on CPU number: */ +int nr =get_cpu ();/* LDT and TSS depend on CPU number: */ eflag = t->thread.eflag; fsr = t->thread.fsr;
@@ -119,6 +119,7 @@ regs->r17 = (_TSS (nr) <<48 ) | (_LDT (nr) <<32 ) | (__u32 ) regs->r17; regs->r30 =load_desc (_LDT (nr));/* LDTD */ +put_cpu (); }/*
arch/ia64/kernel/entry.S +24 -1
@@ -586,10 +586,21 @@ // work.need_resched etc. mustn't get changed by this CPU before it returns to // user- or fsys-mode: (pUStk) cmp.eq.unc p6,p0=r0,r0 // p6 <- pUStk +#ifdef CONFIG_PREEMPT + rsm psr.i // disable interrupts + adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 +(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 + ;; +(pKStk) ld4 r21=[r20] // preempt_count ->r21 + ;; +(pKStk) cmp4.eq p6,p0=r21,r0 // p6 <- preempt_count == 0 + ;; +#else // CONFIG_PREEMPT (pUStk) rsm psr.i ;; (pUStk) adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 ;; +#endif // CONFIG_PREEMPT .work_processed: (p6) ld4 r18=[r17] // load current_thread_info()->flags adds r2=PT(R8)+16,r12
@@ -810,15 +821,27 @@ .work_pending: tbit.z p6,p0=r18,TIF_NEED_RESCHED // current_thread_info()->need_resched==0? (p6) br.cond.sptk.few .notify +#ifdef CONFIG_PREEMPT +(pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1 + ;; +(pKStk) st4 [r20]=r21 + ssm psr.i // enable interrupts +#endif + #if __GNUC__ < 3 br.call.spnt.many rp=invoke_schedule #else br.call.spnt.many rp=schedule #endif .ret9: cmp.eq p6,p0=r0,r0 // p6 <- 1 - rsm psr.i + rsm psr.i // disable interrupts ;; adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 +#if CONFIG_PREEMPT +(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 + ;; +(pKStk) st4 [r20]=r0 // preempt_count() <- 0 +#endif br.cond.sptk.many .work_processed // re-check .notify:
arch/ia64/kernel/irq.c +3 -1
@@ -340,12 +340,14 @@* 0 return value means that this irq is already being * handled by some other CPU. (or is disabled) */ -int cpu =smp_processor_id (); +int cpu;irq_desc_t *desc =irq_desc (irq);struct irqaction * action;unsigned int status;irq_enter (); + cpu =smp_processor_id (); +kstat_cpu (cpu).irqs[irq]++;if (desc->status &IRQ_PER_CPU ) {
arch/ia64/kernel/palinfo.c +3 -1
@@ -894,10 +894,12 @@* in SMP mode, we may need to call another CPU to get correct * information. PAL, by definition, is processor specific */ -if (f->req_cpu ==smp_processor_id ()) +if (f->req_cpu ==get_cpu ()) len = (*palinfo_entries[f->func_id].proc_read)(page);else len =palinfo_handle_smp (f, page); + +put_cpu ();if (len <= off+count) *eof =1 ;
arch/ia64/kernel/perfmon.c +92 -15
466 lines not on this page · raw message
arch/ia64/kernel/smp.c +25 -3
@@ -90,7 +90,7 @@void handle_IPI (int irq,void *dev_id,struct pt_regs *regs) { -int this_cpu =smp_processor_id (); +int this_cpu =get_cpu ();unsigned long *pending_ipis = &__get_cpu_var (ipi_operation);unsigned long ops;
@@ -146,8 +146,12 @@ }while (ops);mb ();/* Order data access and bit testing. */ } +put_cpu (); } +/* +* Called with preeemption disabled +*/ static inline void send_IPI_single (int dest_cpu,int op) {
@@ -155,6 +159,9 @@platform_send_ipi (dest_cpu,IA64_IPI_VECTOR ,IA64_IPI_DM_INT ,0 ); } +/* +* Called with preeemption disabled +*/ static inline void send_IPI_allbutself (int op) {
@@ -166,6 +173,9 @@ } } +/* +* Called with preeemption disabled +*/ static inline void send_IPI_all (int op) {
@@ -176,12 +186,18 @@send_IPI_single (i, op); } +/* +* Called with preeemption disabled +*/ static inline void send_IPI_self (int op) {send_IPI_single (smp_processor_id (), op); } +/* +* Called with preeemption disabled +*/ void smp_send_reschedule (int cpu) {
@@ -197,12 +213,15 @@smp_send_reschedule_all (void) {int i; +int cpu =get_cpu ();/* disable preemption */ for (i =0 ; i <NR_CPUS ; i++) -if (cpu_online (i) && i !=smp_processor_id ()) +if (cpu_online (i) && i != cpu)smp_send_reschedule (i); +put_cpu (); } +void smp_flush_tlb_all (void ) {
@@ -247,9 +266,11 @@ {struct call_data_struct data;int cpus =1 ; +int me =get_cpu ();/* prevent preemption and reschedule on another processor */ -if (cpuid ==smp_processor_id ()) { +if (cpuid == me) {printk ("%s: trying to call self\n" , __FUNCTION__); +put_cpu ();return -EBUSY ; }
@@ -276,6 +297,7 @@ call_data =NULL ;spin_unlock_bh (&call_lock); +put_cpu ();return 0 ; }
arch/ia64/mm/fault.c +1 -1
@@ -55,7 +55,7 @@/* * If we're in an interrupt or have no user context, we must not take the fault.. */ -if (in_interrupt () || !mm) +if (in_atomic () || !mm) goto no_context;#ifdef CONFIG_VIRTUAL_MEM_MAP
arch/ia64/mm/tlb.c +7 -3
@@ -81,9 +81,13 @@ }read_unlock (&tasklist_lock);/* can't call flush_tlb_all() here because of race condition with O(1) scheduler [EF] */ -for (i =0 ; i <NR_CPUS ; ++i) -if (i !=smp_processor_id ()) -per_cpu (ia64_need_tlb_flush, i) =1 ; + { +int cpu =get_cpu ();/* prevent preemption/migration */ +for (i =0 ; i <NR_CPUS ; ++i) +if (i != cpu) +per_cpu (ia64_need_tlb_flush, i) =1 ; +put_cpu (); + }local_flush_tlb_all (); }
include/asm-ia64/hardirq.h +5 -5
@@ -32,18 +32,18 @@* * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) -* - bits 16-31 are the hardirq count (max # of hardirqs: 65536) +* - bits 16-29 are the hardirq count (max # of hardirqs: 16384) * * - (bit 63 is the PREEMPT_ACTIVE flag---not currently implemented.) * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0xffff0000 +* HARDIRQ_MASK: 0x3fff0000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 -#define HARDIRQ_BITS 16 +#define HARDIRQ_BITS 14#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
@@ -83,13 +83,13 @@#define hardirq_trylock () (!in_interrupt())#define hardirq_endlock () do { } while (0) -#define in_atomic () (preempt_count() != 0)#define irq_enter () (preempt_count() += HARDIRQ_OFFSET)#if CONFIG_PREEMPT -# error CONFIG_PREEMT currently not supported. +# define in_atomic () ((preempt_count() & ~PREEMPT_ACTIVE) != kernel_locked())# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)#else +# define in_atomic () (preempt_count() != 0)# define IRQ_EXIT_OFFSET HARDIRQ_OFFSET#endif
include/asm-ia64/spinlock.h +15 -0
@@ -189,4 +189,19 @@clear_bit (31 , (x)); \ }) +#define _raw_write_trylock (rw) \ +({ \ + register long result; \ + \ + __asm__ __volatile__ ( \ + "mov ar.ccv = r0\n" \ + "dep r29 = -1, r0, 31, 1\n" \ + ";;\n" \ + "1:\n" \ + "cmpxchg4.acq %0 = [%1], r29, ar.ccv\n" \ + :"=r"(result) :"r"(rw) : "ar.ccv", "r2", "r29", "memory"); \ + (result == 0); \ +}) + +#endif /* _ASM_IA64_SPINLOCK_H */
include/asm-ia64/system.h +1 -1
@@ -206,7 +206,7 @@#ifdef CONFIG_PERFMON DECLARE_PER_CPU(unsigned long, pfm_syst_info); -# define PERFMON_IS_SYSWIDE () (get_cpu_var(pfm_syst_info) & 0x1) +# define PERFMON_IS_SYSWIDE () (__get_cpu_var(pfm_syst_info) & 0x1)#else # define PERFMON_IS_SYSWIDE () (0)#endif
include/asm-ia64/thread_info.h +2 -1
@@ -15,7 +15,8 @@#define TI_ADDR_LIMIT 0x10#define TI_PRE_COUNT 0x18 -#define PREEMPT_ACTIVE 0x4000000 +#define PREEMPT_ACTIVE_BIT 30 +#define PREEMPT_ACTIVE (1<<PREEMPT_ACTIVE_BIT)#ifndef __ASSEMBLY__