O(1) MQ scheduler J9 patch for IA64
Erich> I had to add a function smp_call_function_nowait which sends Erich> an IPI and returns immediately in order to get rid of a Erich> IA64-specific race condition when the mmu_context wraps Erich> around. Now it makes a very stable and fast impression, Erich> interactive work is even possible while doing a "hackbench Erich> 55" and ping flooding the machine... Tested on 2 CPU BigSur, Erich> 4 CPU LION, 16 CPU AzusA.
Hmmh, I'm a bit worried that this is a fragile solution. It would break if there ever was a scenario where a task migrated from one CPU to another while the target CPU has interrupts disabled. I do not think this can happen with Ingo's current scheduler but I'd prefer a more robust solution.
A somewhat cheesy solution might be to have a per-CPU flag that indicates whether a CPU should flush its TLB before switching to the next task. We could check for this in activate_mm(). Do you want to try this?
--david
Quoted 4 lines · David Mosberger
A somewhat cheesy solution might be to have a per-CPU flag that indicates whether a CPU should flush its TLB before switching to the next task. We could check for this in
activate_mm(). Do you want to try this?
Would this be a good thing to put in the mmu_context? It seems like that could isolate it within arch-specific code.
Cheers,
Bill
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · William Lee Irwin III
Bill> On Thu, Feb 07, 2002 at 10:11:28PM -0800, David Mosberger
Quoted 4 lines · Bill> wrote:
Quoted 4 lines
A somewhat cheesy solution might be to have a per-CPU flag that indicates whether a CPU should flush its TLB before switching to the next task. We could check for this in
activate_mm(). Do you want to try this?
Bill> Would this be a good thing to put in the mmu_context? It seems Bill> like that could isolate it within arch-specific code.
It's arch-specific either way. The flag needs to be per CPU, not per task.
--david
[Linux-ia64] Re: O(1) MQ scheduler J9 patch for IA64 -> K3 patch
In reply to David MosbergerQuoted 14 lines · David Mosberger
Erich> I had to add a function
smp_call_function_nowaitwhich sends Erich> an IPI and returns immediately in order to get rid of a Erich> IA64-specific race condition when themmu_contextwraps Erich> around. Now it makes a very stable and fast impression, Erich> interactive work is even possible while doing a "hackbench Erich> 55" and ping flooding the machine... Tested on 2 CPU BigSur, Erich> 4 CPU LION, 16 CPU AzusA.Hmmh, I'm a bit worried that this is a fragile solution. It would break if there ever was a scenario where a task migrated from one CPU to another while the target CPU has interrupts disabled. I do not think this can happen with Ingo's current scheduler but I'd prefer a more robust solution.
Quoted 4 lines
A somewhat cheesy solution might be to have a per-CPU flag that indicates whether a CPU should flush its TLB before switching to the next task. We could check for this in
activate_mm(). Do you want to try this?
OK, I also feel better with the new solution. Didn't want to put it into the runqueues because it really is an IA-64 problem. It's architecture-specific and the CPU flags are in ia64_ctx. The context wrapping is so seldom that the single lock protecting the CPU mask shouldn't matter. But if somebody decides to use this mechanism generally for flush_tlb_all() we'd have to change it...
Besides this: the attached patch is for the latest version of the O(1) scheduler (K3). It has a different sys_sched_yield() from Ingo's because that one has a problem with more than 8 CPUs. Also the set_cpus_allowed() function has the limitation that it just works if the current process tries to change its mask.
Best regards,
Erich
arch/ia64/kdb/kdba_bt.c +1 -1
@@ -197,7 +197,7 @@ }#ifdef CONFIG_SMP else if (task_has_cpu(p)) { - sw = kdb_sw[p->processor]; + sw = kdb_sw[p->cpu]; }#endif else {
arch/ia64/kernel/entry.S +4 -2
@@ -161,7 +161,8 @@ mov r8=r13 // return pointer to previously running task mov r13=in0 // set "current" pointer ;; -(p6) ssm psr.i // renable psr.i AFTER the ic bit is serialized +//(p6) ssm psr.i // interrupt delivery should not be enabled + // with the new O(1) MQ scheduler DO_LOAD_SWITCH_STACK #ifdef CONFIG_SMP
@@ -170,7 +171,8 @@ br.ret.sptk.many rp // boogie on out in new context .map: - rsm psr.i | psr.ic + //rsm psr.i | psr.ic + rsm psr.ic movl r25=PAGE_KERNEL ;; srlz.d
arch/ia64/kernel/irq_ia64.c +9 -0
@@ -148,6 +148,14 @@ flags: SA_INTERRUPT, name: "IPI" }; + +extern void smp_task_migration_interrupt (int irq,void *dev_id,struct pt_regs *regs); + +static struct irqaction task_migration_irqaction = { + handler: smp_task_migration_interrupt, + flags:SA_INTERRUPT , + name:"Task migration" +};#endif void
@@ -172,6 +180,7 @@register_percpu_irq (IA64_SPURIOUS_INT_VECTOR ,NULL );#ifdef CONFIG_SMP register_percpu_irq (IA64_IPI_VECTOR , &ipi_irqaction); +register_percpu_irq (IA64_TASK_MIGRATION , &task_migration_irqaction);#endif platform_irq_init (); }
arch/ia64/kernel/process.c +1 -5
@@ -125,9 +125,6 @@ cpu_idle (void *unused) {/* endless idle loop with no priority at all */ -init_idle (); - current->nice =20 ; - current->counter =-100 ;while (1 ) {
@@ -136,11 +133,10 @@min_xtp ();#endif -while (!current->need_resched) { +if (!current->need_resched) {#ifdef CONFIG_IA64_SGI_SN snidle ();#endif -continue ; }#ifdef CONFIG_IA64_SGI_SN
arch/ia64/kernel/setup.c +3 -3
@@ -375,10 +375,10 @@ {#ifdef CONFIG_SMP # define lpj c->loops_per_jiffy -# define cpu c->processor +# define cpum c->processor#else # define lpj loops_per_jiffy -# define cpu 0 +# define cpum 0#endif char family[32 ], features[128 ], *cp;struct cpuinfo_ia64 *c = v;
@@ -417,7 +417,7 @@"cpu MHz : %lu.%06lu\n" "itc MHz : %lu.%06lu\n" "BogoMIPS : %lu.%02lu\n\n" , - cpu, c->vendor, family, c->model, c->revision, c->archrev, + cpum, c->vendor, family, c->model, c->revision, c->archrev, features, c->ppn, c->number, c->proc_freq /1000000 , c->proc_freq %1000000 , c->itc_freq / 1000000, c->itc_freq % 1000000,
arch/ia64/kernel/smp.c +36 -0
@@ -200,6 +200,42 @@ }void +smp_send_reschedule_all (void ) +{ +send_IPI_all (IA64_IPI_RESCHEDULE ); +} + +static spinlock_t migration_lock =SPIN_LOCK_UNLOCKED ; +static task_t *new_task; + + +/* +* This function sends a 'task migration' IPI to another CPU. +* Must be called from syscall contexts, with interrupts *enabled*. +*/ +void smp_migrate_task (int cpu,task_t *p) +{ +/* +* The target CPU will unlock the migration spinlock: +*/ +spin_lock (&migration_lock); + new_task = p; +platform_send_ipi (cpu,IA64_TASK_MIGRATION ,IA64_IPI_DM_INT ,0 ); +} + +/* +* Task migration callback. +*/ +asmlinkage void smp_task_migration_interrupt(int irq, void *dev_id, struct pt_regs *regs) +{ +task_t *p; + + p = new_task; +spin_unlock (&migration_lock); +sched_task_migrated (p); +} + +void smp_flush_tlb_all (void ) {smp_call_function ((void (*)(void *))__flush_tlb_all,0 ,1 ,1 );
arch/ia64/kernel/smpboot.c +11 -5
@@ -356,6 +356,7 @@local_irq_enable ();calibrate_delay (); local_cpu_data->loops_per_jiffy = loops_per_jiffy; +ia64_disable_timer ();/* * Allow the master to continue. */
@@ -379,7 +380,8 @@Dprintk ("CPU %d is set to go.\n" ,smp_processor_id ());while (!atomic_read (&smp_commenced)) ; - +/* reenable timer interrupts */ +ia64_cpu_local_tick ();Dprintk ("CPU %d is starting idle.\n" ,smp_processor_id ());return cpu_idle (); }
@@ -416,11 +418,10 @@if (!idle)panic ("No idle process for CPU %d" , cpu); -task_set_cpu (idle, cpu);/* we schedule the first task manually */ +init_idle (idle, cpu); ia64_cpu_to_sapicid[cpu] = sapicid; -del_from_runqueue (idle);unhash_process (idle); init_tasks[cpu] = idle;
@@ -481,8 +482,7 @@printk ("Boot processor id 0x%x/0x%x\n" ,0 , boot_cpu_id); global_irq_holder =0 ; - current->processor =0 ; -init_idle (); + current->cpu =0 ;/* * If SMP should be disabled, then really disable it!
@@ -569,3 +569,9 @@ smp_num_cpus =1 ; } } + +/* Number of ticks we consider an idle tasks still cache-hot. +* For Itanium: with 1GB/s bandwidth we need 4ms to fill up 4MB L3 cache... +* So let's try 10 ticks. +*/ +unsigned long cache_decay_ticks=10 ;
arch/ia64/kernel/time.c +7 -1
@@ -209,7 +209,7 @@/* * Encapsulate access to the itm structure for SMP. */ -void __init +void ia64_cpu_local_tick (void ) {int cpu =smp_processor_id ();
@@ -298,3 +298,9 @@efi_gettimeofday ((struct timeval *) &xtime);ia64_init_itm (); } + +void __init ia64_disable_timer (void ) +{ +ia64_set_itv (IA64_TIMER_VECTOR |IA64_TIMER_MASK ); +} +
arch/ia64/mm/fault.c +1 -2
@@ -194,8 +194,7 @@ out_of_memory:up_read (&mm->mmap_sem);if (current->pid ==1 ) { - current->policy |=SCHED_YIELD ; -schedule (); +yield ();down_read (&mm->mmap_sem); goto survive; }
arch/ia64/mm/tlb.c +5 -2
@@ -37,7 +37,8 @@ lock: SPIN_LOCK_UNLOCKED, next: 1, limit: (1 << 15) - 1, /* start out with the safe (architected) limit */ - max_ctx: ~0U + max_ctx: ~0U, + flush: 0UL };/*
@@ -76,7 +77,9 @@ ia64_ctx.limit = tsk_context; }read_unlock (&tasklist_lock); -flush_tlb_all (); +//flush_tlb_all(); /* potential race condition with O(1) scheduler [EF] */ + ia64_ctx.flush = ((1 <<smp_num_cpus) -1 ) & ~(1UL <<smp_processor_id ()); +__flush_tlb_all (); } static inline void
arch/ia64/tools/print_offsets.c +1 -1
@@ -54,7 +54,7 @@ { "IA64_TASK_PTRACE_OFFSET", offsetof (struct task_struct, ptrace) }, { "IA64_TASK_SIGPENDING_OFFSET", offsetof (struct task_struct, sigpending) }, { "IA64_TASK_NEED_RESCHED_OFFSET", offsetof (struct task_struct, need_resched) }, - { "IA64_TASK_PROCESSOR_OFFSET", offsetof (struct task_struct, processor) }, + { "IA64_TASK_PROCESSOR_OFFSET", offsetof (struct task_struct, cpu) }, { "IA64_TASK_THREAD_OFFSET", offsetof (struct task_struct, thread) }, { "IA64_TASK_THREAD_KSP_OFFSET", offsetof (struct task_struct, thread.ksp) }, #ifdef CONFIG_PERFMON
include/asm-ia64/bitops.h +60 -0
80 lines not on this page · raw message
include/asm-ia64/hw_irq.h +4 -0
@@ -49,6 +49,7 @@#define IA64_PERFMON_VECTOR 0xee/* performanc monitor interrupt vector */ #define IA64_TIMER_VECTOR 0xef/* use highest-prio group 15 interrupt for timer */ #define IA64_MCA_WAKEUP_VECTOR 0xf0/* MCA wakeup (must be >MCA_RENDEZ_VECTOR) */ +#define IA64_TASK_MIGRATION 0xfb/* task migration interrupt vector */ #define IA64_IPI_RESCHEDULE 0xfd/* SMP reschedule */ #define IA64_IPI_VECTOR 0xfe/* inter-processor interrupt vector */
@@ -65,6 +66,9 @@ IA64_IPI_DM_EXTINT = 0x7, /* pend an 8259-compatible interrupt. */ }; +/* bit for masking and discarding timer interrupts on IA64 */ +#define IA64_TIMER_MASK (1<<16) +extern __u8 isa_irq_to_vector_map[16 ];#define isa_irq_to_vector (x) isa_irq_to_vector_map[(x)]
include/asm-ia64/mmu_context.h +39 -0
@@ -33,6 +33,9 @@unsigned int next;/* next context number to use */ unsigned int limit;/* next >= limit => must call wrap_mmu_context() */ unsigned int max_ctx;/* max. context value supported by all CPUs */ +unsigned long flush;/* CPU mask for those which have to perform a +* local TLB flush before switching the context. +*/ };extern struct ia64_ctx ia64_ctx;
@@ -44,6 +47,25 @@ { } +/* +* When the context counter wraps around all TLBs need to be flushed because +* an old context number might have been reused. This is signalled by a bit +* set in ia64_ctx.flush, which is checked in the routine below. Called by +* activate_mm(). <[email protected]> +*/ +static inline void +delayed_tlb_flush (void ) +{ +extern void __flush_tlb_all (void ); + +if (unlikely (ia64_ctx.flush & (1 <<smp_processor_id ()))) { +spin_lock (&ia64_ctx.lock); +__flush_tlb_all (); + ia64_ctx.flush &= ~(1 <<smp_processor_id ()); +spin_unlock (&ia64_ctx.lock); + } +} +static inline void get_new_mmu_context (struct mm_struct *mm) {
@@ -113,11 +135,28 @@* We may get interrupts here, but that's OK because interrupt * handlers cannot touch user-space. */ +delayed_tlb_flush ();ia64_set_kr (IA64_KR_PT_BASE ,__pa (next->pgd));get_mmu_context (next); reload_context(next); } +/* +* Needed for the O(1) MQ scheduler. +*/ +#if MAX_PRIO >=192 +# error update this function. */ +#endif + +static inline int sched_find_first_bit (unsigned long *b) +{ +if (unlikely (b[0 ])) +return __ffs (b[0 ]); +if (b[1 ]) +return 64 +__ffs (b[1 ]); +return __ffs (b[2 ]) +128 ; +} +#define switch_mm (prev_mm,next_mm,next_task,cpu) activate_mm(prev_mm, next_mm)# endif /* ! __ASSEMBLY__ */
include/asm-ia64/pgalloc.h +3 -0
@@ -160,9 +160,12 @@#ifdef CONFIG_SMP extern void smp_flush_tlb_all (void ); +extern void smp_flush_tlb_all_nowait (void );# define flush_tlb_all () smp_flush_tlb_all() +# define flush_tlb_all_nowait () smp_flush_tlb_all_nowait()#else # define flush_tlb_all () __flush_tlb_all() +# define flush_tlb_all_nowait () __flush_tlb_all()#endif /*
include/asm-ia64/smp.h +4 -7
@@ -27,7 +27,7 @@#define SMP_IRQ_REDIRECTION (1 << 0)#define SMP_IPI_REDIRECTION (1 << 1) -#define smp_processor_id () (current->processor) +#define smp_processor_id () (current->cpu) extern struct smp_boot_data { int cpu_count;
@@ -48,6 +48,9 @@extern unsigned long ap_wakeup_vector; +extern void smp_send_reschedule (int cpu); +extern void smp_send_reschedule_all (void ); +/* * Function to map hard smp processor id to logical id. Slow, so * don't use this in performance-critical code.
@@ -110,12 +113,6 @@#define NO_PROC_ID 0xffffffff/* no processor magic marker */ -/* -* Extra overhead to move a task from one cpu to another (due to TLB and cache misses). -* Expressed in "negative nice value" units (larger number means higher priority/penalty). -*/ -#define PROC_CHANGE_PENALTY 20 -extern void __init init_smp_config (void );extern void smp_do_timer (struct pt_regs *regs);
include/asm-ia64/spinlock.h +1 -1
@@ -84,7 +84,7 @@"mov r29 = 1\n" \";;\n" \"1:\n" \ -"ld4.bias r2 = [%0]\n" \ +"ld4 r2 = [%0]\n" \";;\n" \"cmp4.eq p0,p7 = r0,r2\n" \"(p7) br.cond.spnt.few 1b \n" \
include/linux/smp.h +0 -6
@@ -24,12 +24,6 @@extern void smp_send_stop (void );/* -* sends a 'reschedule' event to another CPU: -*/ -extern void FASTCALL (smp_send_reschedule (int cpu)); - - -/* * Boot processor call to load the other CPU's */ extern void smp_boot_cpus (void );
kdb/kdbmain.c +1 -1
@@ -2344,7 +2344,7 @@for_each_task (p) { kdb_printf("0x%p %08d %08d %1.1d %3.3d %s 0x%p%c%s\n", (void *)p, p->pid, p->p_pptr->pid, - task_has_cpu(p), p->processor, + task_has_cpu(p), p->cpu, (p->state == 0)?"run ":(p->state>0)?"stop":"unrn", (void *)(&p->thread), (p == current) ? '*': ' ',
kernel/printk.c +2 -0
@@ -25,6 +25,8 @@#include <linux/init.h> #include <linux/module.h> #include <linux/interrupt.h> /* For in_interrupt() */ +#include <linux/config.h> +#include <linux/delay.h> #include <asm/uaccess.h>
kernel/sched.c +31 -60
124 lines not on this page · raw message
kernel/timer.c +1 -12
@@ -584,18 +584,7 @@int cpu =smp_processor_id (), system = user_tick ^1 ;update_one_process (p, user_tick, system, cpu); -if (p->pid) { -if (--p->counter <=0 ) { - p->counter =0 ; - p->need_resched =1 ; - } -if (p->nice >0 ) - kstat.per_cpu_nice[cpu] += user_tick; -else - kstat.per_cpu_user[cpu] += user_tick; - kstat.per_cpu_system[cpu] += system; - }else if (really_local_bh_count () ||really_local_irq_count () >1 ) - kstat.per_cpu_system[cpu] += system; +scheduler_tick (user_tick, system); }/*