O(1) MQ scheduler J9 patch for IA64
Hi,
please find attached a patch which aplies to
linux-2.4.17 + kdb_common + ia64_patch + kdb_ia64 + sched-O1-2.4.17-J9.patch
I had to add a function smp_call_function_nowait which sends an IPI and returns immediately in order to get rid of a IA64-specific race condition when the mmu_context wraps around. Now it makes a very stable and fast impression, interactive work is even possible while doing a "hackbench 55" and ping flooding the machine... Tested on 2 CPU BigSur, 4 CPU LION, 16 CPU AzusA.
The patch should aply to the K0 version of the scheduler, too. As this isn't out yet for 2.4.17 I couldn't test it.
Regards,
Erich
---
Erich Focht <[email protected]>
NEC European Supercomputer Systems, European HPC Technology Center
arch/ia64/kdb/kdba_bt.c +1 -1
@@ -197,7 +197,7 @@ }#ifdef CONFIG_SMP else if (task_has_cpu(p)) { - sw = kdb_sw[p->processor]; + sw = kdb_sw[p->cpu]; }#endif else {
arch/ia64/kernel/entry.S +4 -2
@@ -161,7 +161,8 @@ mov r8=r13 // return pointer to previously running task mov r13=in0 // set "current" pointer ;; -(p6) ssm psr.i // renable psr.i AFTER the ic bit is serialized +//(p6) ssm psr.i // interrupt delivery should not be enabled + // with the new O(1) MQ scheduler DO_LOAD_SWITCH_STACK #ifdef CONFIG_SMP
@@ -170,7 +171,8 @@ br.ret.sptk.many rp // boogie on out in new context .map: - rsm psr.i | psr.ic + //rsm psr.i | psr.ic + rsm psr.ic movl r25=PAGE_KERNEL ;; srlz.d
arch/ia64/kernel/irq_ia64.c +9 -0
@@ -148,6 +148,14 @@ flags: SA_INTERRUPT, name: "IPI" }; + +extern void smp_task_migration_interrupt (int irq,void *dev_id,struct pt_regs *regs); + +static struct irqaction task_migration_irqaction = { + handler: smp_task_migration_interrupt, + flags:SA_INTERRUPT , + name:"Task migration" +};#endif void
@@ -172,6 +180,7 @@register_percpu_irq (IA64_SPURIOUS_INT_VECTOR ,NULL );#ifdef CONFIG_SMP register_percpu_irq (IA64_IPI_VECTOR , &ipi_irqaction); +register_percpu_irq (IA64_TASK_MIGRATION , &task_migration_irqaction);#endif platform_irq_init (); }
arch/ia64/kernel/process.c +1 -5
@@ -125,9 +125,6 @@ cpu_idle (void *unused) {/* endless idle loop with no priority at all */ -init_idle (); - current->nice =20 ; - current->counter =-100 ;while (1 ) {
@@ -136,11 +133,10 @@min_xtp ();#endif -while (!current->need_resched) { +if (!current->need_resched) {#ifdef CONFIG_IA64_SGI_SN snidle ();#endif -continue ; }#ifdef CONFIG_IA64_SGI_SN
arch/ia64/kernel/setup.c +3 -3
@@ -375,10 +375,10 @@ {#ifdef CONFIG_SMP # define lpj c->loops_per_jiffy -# define cpu c->processor +# define cpum c->processor#else # define lpj loops_per_jiffy -# define cpu 0 +# define cpum 0#endif char family[32 ], features[128 ], *cp;struct cpuinfo_ia64 *c = v;
@@ -417,7 +417,7 @@"cpu MHz : %lu.%06lu\n" "itc MHz : %lu.%06lu\n" "BogoMIPS : %lu.%02lu\n\n" , - cpu, c->vendor, family, c->model, c->revision, c->archrev, + cpum, c->vendor, family, c->model, c->revision, c->archrev, features, c->ppn, c->number, c->proc_freq /1000000 , c->proc_freq %1000000 , c->itc_freq / 1000000, c->itc_freq % 1000000,
arch/ia64/kernel/smp.c +107 -0
147 lines not on this page · raw message
arch/ia64/kernel/smpboot.c +11 -5
@@ -356,6 +356,7 @@local_irq_enable ();calibrate_delay (); local_cpu_data->loops_per_jiffy = loops_per_jiffy; +ia64_disable_timer ();/* * Allow the master to continue. */
@@ -379,7 +380,8 @@Dprintk ("CPU %d is set to go.\n" ,smp_processor_id ());while (!atomic_read (&smp_commenced)) ; - +/* reenable timer interrupts */ +ia64_cpu_local_tick ();Dprintk ("CPU %d is starting idle.\n" ,smp_processor_id ());return cpu_idle (); }
@@ -416,11 +418,10 @@if (!idle)panic ("No idle process for CPU %d" , cpu); -task_set_cpu (idle, cpu);/* we schedule the first task manually */ +init_idle (idle, cpu); ia64_cpu_to_sapicid[cpu] = sapicid; -del_from_runqueue (idle);unhash_process (idle); init_tasks[cpu] = idle;
@@ -481,8 +482,7 @@printk ("Boot processor id 0x%x/0x%x\n" ,0 , boot_cpu_id); global_irq_holder =0 ; - current->processor =0 ; -init_idle (); + current->cpu =0 ;/* * If SMP should be disabled, then really disable it!
@@ -569,3 +569,9 @@ smp_num_cpus =1 ; } } + +/* Number of ticks we consider an idle tasks still cache-hot. +* For Itanium: with 1GB/s bandwidth we need 4ms to fill up 4MB L3 cache... +* So let's try 10 ticks. +*/ +unsigned long cache_decay_ticks=10 ;
arch/ia64/kernel/time.c +7 -1
@@ -209,7 +209,7 @@/* * Encapsulate access to the itm structure for SMP. */ -void __init +void ia64_cpu_local_tick (void ) {int cpu =smp_processor_id ();
@@ -298,3 +298,9 @@efi_gettimeofday ((struct timeval *) &xtime);ia64_init_itm (); } + +void __init ia64_disable_timer (void ) +{ +ia64_set_itv (IA64_TIMER_VECTOR |IA64_TIMER_MASK ); +} +
arch/ia64/mm/fault.c +1 -2
@@ -194,8 +194,7 @@ out_of_memory:up_read (&mm->mmap_sem);if (current->pid ==1 ) { - current->policy |=SCHED_YIELD ; -schedule (); +yield ();down_read (&mm->mmap_sem); goto survive; }
arch/ia64/mm/tlb.c +2 -1
@@ -76,7 +76,8 @@ ia64_ctx.limit = tsk_context; }read_unlock (&tasklist_lock); -flush_tlb_all (); +//flush_tlb_all(); /* potential race condition with O(1) scheduler [EF] */ +smp_flush_tlb_all_nowait (); } static inline void
arch/ia64/tools/print_offsets.c +1 -1
@@ -54,7 +54,7 @@ { "IA64_TASK_PTRACE_OFFSET", offsetof (struct task_struct, ptrace) }, { "IA64_TASK_SIGPENDING_OFFSET", offsetof (struct task_struct, sigpending) }, { "IA64_TASK_NEED_RESCHED_OFFSET", offsetof (struct task_struct, need_resched) }, - { "IA64_TASK_PROCESSOR_OFFSET", offsetof (struct task_struct, processor) }, + { "IA64_TASK_PROCESSOR_OFFSET", offsetof (struct task_struct, cpu) }, { "IA64_TASK_THREAD_OFFSET", offsetof (struct task_struct, thread) }, { "IA64_TASK_THREAD_KSP_OFFSET", offsetof (struct task_struct, thread.ksp) }, #ifdef CONFIG_PERFMON
include/asm-ia64/bitops.h +60 -0
80 lines not on this page · raw message
include/asm-ia64/hw_irq.h +4 -0
@@ -49,6 +49,7 @@#define IA64_PERFMON_VECTOR 0xee/* performanc monitor interrupt vector */ #define IA64_TIMER_VECTOR 0xef/* use highest-prio group 15 interrupt for timer */ #define IA64_MCA_WAKEUP_VECTOR 0xf0/* MCA wakeup (must be >MCA_RENDEZ_VECTOR) */ +#define IA64_TASK_MIGRATION 0xfb/* task migration interrupt vector */ #define IA64_IPI_RESCHEDULE 0xfd/* SMP reschedule */ #define IA64_IPI_VECTOR 0xfe/* inter-processor interrupt vector */
@@ -65,6 +66,9 @@ IA64_IPI_DM_EXTINT = 0x7, /* pend an 8259-compatible interrupt. */ }; +/* bit for masking and discarding timer interrupts on IA64 */ +#define IA64_TIMER_MASK (1<<16) +extern __u8 isa_irq_to_vector_map[16 ];#define isa_irq_to_vector (x) isa_irq_to_vector_map[(x)]
include/asm-ia64/mmu_context.h +16 -0
@@ -118,6 +118,22 @@reload_context(next); } +/* +* Needed for the O(1) MQ scheduler. +*/ +#if MAX_RT_PRIO !=128 ||MAX_PRIO >=192 +# error update this function. */ +#endif + +static inline int sched_find_first_bit (unsigned long *b) +{ +if (unlikely (b[0 ])) +return __ffs (b[0 ]); +if (unlikely (b[1 ])) +return 64 +__ffs (b[1 ]); +return __ffs (b[2 ]) +128 ; +} +#define switch_mm (prev_mm,next_mm,next_task,cpu) activate_mm(prev_mm, next_mm)# endif /* ! __ASSEMBLY__ */
include/asm-ia64/smp.h +4 -7
@@ -27,7 +27,7 @@#define SMP_IRQ_REDIRECTION (1 << 0)#define SMP_IPI_REDIRECTION (1 << 1) -#define smp_processor_id () (current->processor) +#define smp_processor_id () (current->cpu) extern struct smp_boot_data { int cpu_count;
@@ -48,6 +48,9 @@extern unsigned long ap_wakeup_vector; +extern void smp_send_reschedule (int cpu); +extern void smp_send_reschedule_all (void ); +/* * Function to map hard smp processor id to logical id. Slow, so * don't use this in performance-critical code.
@@ -110,12 +113,6 @@#define NO_PROC_ID 0xffffffff/* no processor magic marker */ -/* -* Extra overhead to move a task from one cpu to another (due to TLB and cache misses). -* Expressed in "negative nice value" units (larger number means higher priority/penalty). -*/ -#define PROC_CHANGE_PENALTY 20 -extern void __init init_smp_config (void );extern void smp_do_timer (struct pt_regs *regs);
include/asm-ia64/spinlock.h +1 -1
@@ -84,7 +84,7 @@"mov r29 = 1\n" \";;\n" \"1:\n" \ -"ld4.bias r2 = [%0]\n" \ +"ld4 r2 = [%0]\n" \";;\n" \"cmp4.eq p0,p7 = r0,r2\n" \"(p7) br.cond.spnt.few 1b \n" \
include/linux/smp.h +0 -6
@@ -24,12 +24,6 @@extern void smp_send_stop (void );/* -* sends a 'reschedule' event to another CPU: -*/ -extern void FASTCALL (smp_send_reschedule (int cpu)); - - -/* * Boot processor call to load the other CPU's */ extern void smp_boot_cpus (void );
kdb/kdbmain.c +1 -1
@@ -2360,7 +2360,7 @@for_each_task (p) { kdb_printf("0x%p %08d %08d %1.1d %3.3d %s 0x%p%c%s\n", (void *)p, p->pid, p->p_pptr->pid, - task_has_cpu(p), p->processor, + task_has_cpu(p), p->cpu, (p->state == 0)?"run ":(p->state>0)?"stop":"unrn", (void *)(&p->thread), (p == current) ? '*': ' ',
kernel/printk.c +2 -0
@@ -25,6 +25,8 @@#include <linux/init.h> #include <linux/module.h> #include <linux/interrupt.h> /* For in_interrupt() */ +#include <linux/config.h> +#include <linux/delay.h> #include <asm/uaccess.h>
kernel/sched.c +15 -6
@@ -93,6 +93,15 @@ p->array = array; } +/* needed on IA64, arch/ia64/kernel/head.S relies on it (EF) */ +struct task_struct * init_tasks[NR_CPUS ]__initdata = {&init_task, }; + +/* needed in kdb (EF) */ +int task_has_cpu (task_t *p) +{ +return (p ==task_rq (p)->curr); +} +/* * A task is 'heavily interactive' if it either has reached the * bottom 25% of the SCHED_OTHER priority range, or if it is below
@@ -859,16 +868,16 @@ p->cpus_allowed = new_mask;/* -* Can the task run on the current CPU? If not then +* Can the task run on its current CPU? If not then * migrate the process off to a proper CPU. */ -if (new_mask & (1UL <<smp_processor_id ())) +if (new_mask & (1UL << p->cpu))return ;#if CONFIG_SMP - current->state =TASK_UNINTERRUPTIBLE ; -smp_migrate_task (__ffs (new_mask), current); - -schedule (); + p->state =TASK_UNINTERRUPTIBLE ; +smp_migrate_task (__ffs (new_mask), p); +if (p == current) +schedule ();#endif }
kernel/timer.c +6 -7
@@ -585,17 +585,16 @@update_one_process (p, user_tick, system, cpu);if (p->pid) { -if (--p->counter <=0 ) { - p->counter =0 ; - p->need_resched =1 ; - } -if (p->nice >0 ) +if (p->__nice >0 ) kstat.per_cpu_nice[cpu] += user_tick;else kstat.per_cpu_user[cpu] += user_tick; kstat.per_cpu_system[cpu] += system; - }else if (really_local_bh_count () ||really_local_irq_count () >1 ) - kstat.per_cpu_system[cpu] += system; + }else { +if (really_local_bh_count () ||really_local_irq_count () >1 ) + kstat.per_cpu_system[cpu] += system; + } +scheduler_tick (p); }/*
Quoted 10 lines · Erich Focht
please find attached a patch which aplies to
linux-2.4.17 +
kdb_common+ia64_patch+kdb_ia64+ sched-O1-2.4.17-J9.patchI had to add a function
smp_call_function_nowaitwhich sends an IPI and returns immediately in order to get rid of a IA64-specific race condition when themmu_contextwraps around. Now it makes a very stable and fast impression, interactive work is even possible while doing a "hackbench 55" and ping flooding the machine... Tested on 2 CPU BigSur, 4 CPU LION, 16 CPU AzusA.
Why is there a global change of current->processor to current->cpu? That has knock on effects because #define cpu and references to that macro also have to be changed. I don't see any benefit.
Quoted 3 lines · Keith Owens
Why is there a global change of
current->processortocurrent->cpu? That has knock on effects because #define cpu and references to that macro also have to be changed. I don't see any benefit.
Keith,
I can only guess that Ingo wanted to see the problematic spots when he compiles. I think there was a discussion on linux-kernel ML but I didn't follow it, so maybe Ingo can tell us...
Regards,
Erich
Quoted 8 lines · Erich Focht
Quoted 3 lines
Why is there a global change of
current->processortocurrent->cpu? That has knock on effects because #define cpu and references to that macro also have to be changed. I don't see any benefit.Keith,
I can only guess that Ingo wanted to see the problematic spots when he compiles. I think there was a discussion on linux-kernel ML but I didn't follow it, so maybe Ingo can tell us...
the current->processor => current->cpu indeed was such a helper, to find places that rely on p->cpu. (it's mostly code that has scheduling relevance.)
renaming ->nice to ->__nice had another purpose, eg. in the -K2 patch there is no 'nice' field anymore, ->static_prio is used.
Ingo