[2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()
Subsystems (9)
PATCH 1/1 [2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()
This patch replaces the flush_map() function in the arch/i386/mm/pageattr.c file with flush_tlb_all() calls, as the flush_map() function wants to do the same, but just forgot the preempt_disable() and preempt_enable() calls.
To minimize future inconsistency I think this patch should be applied...
Best regards
Thomas Schlichter
arch/i386/mm/pageattr.c renamed from arch/i386/mm/pageattr.c.orig +2 -21
@@ -45,17 +45,6 @@return base; } -static void flush_kernel_map (void *dummy) -{ -/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */ -if (boot_cpu_data.x86_model >=4 ) - asmvolatile ("wbinvd" :::"memory" ); -/* Flush all to work around Errata in early athlons regarding -* large page flushing. -*/ -__flush_tlb_all (); -} -static void set_pmd_pte (pte_t *kpte,unsigned long address,pte_t pte) {set_pte_atomic (kpte, pte);/* change init_mm */
@@ -129,14 +118,6 @@return 0 ; } -static inline void flush_map (void ) -{ -#ifdef CONFIG_SMP -smp_call_function (flush_kernel_map,NULL ,1 ,1 ); -#endif -flush_kernel_map (NULL ); -} -struct deferred_page {struct deferred_page *next;struct page *fpage;
@@ -172,7 +153,7 @@struct deferred_page *df; df =kmalloc (sizeof (struct deferred_page ),GFP_KERNEL );if (!df) { -flush_map (); +flush_tlb_all ();__free_page (fpage); }else { df->next = df_list;
@@ -192,7 +173,7 @@down_read (&init_mm.mmap_sem); df =xchg (&df_list,NULL );up_read (&init_mm.mmap_sem); -flush_map (); +flush_tlb_all ();for (; df; df = next_df) { next_df = df->next;if (df->fpage)
Quoted 5 lines · Thomas Schlichter
This patch replaces the
flush_map()function in thearch/i386/mm/pageattr.cfile withflush_tlb_all()calls, as theflush_map()function wants to do the same, but just forgot thepreempt_disable()andpreempt_enable()calls.
flush_map() is doing a wbinvd:
arch/i386/mm/pageattr.c line 52
Quoted 5 lines
arch/i386/mm/pageattr.c renamed from arch/i386/mm/pageattr.c.orig +0
-5-static void flush_kernel_map (void *dummy) -{ -/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */ -if (boot_cpu_data.x86_model >=4 ) - asmvolatile ("wbinvd" :::"memory" );
It appears that by replacing flush_map() with flush_tlb_all(), you are no longer doing this?
Quoted 3 lines · Thomas Schlichter
This patch replaces the
flush_map()function in thearch/i386/mm/pageattr.cfile withflush_tlb_all()calls, as theflush_map()function wants to do the same, but just forgot thepreempt_disable()andpreempt_enable()calls.To minimize future inconsistency I think this patch should be applied...
This looks bogus. You're killing the wbinvd() in flush_kernel_map() which is needed.
Dave
Signature · 2 lines
| Dave Jones. http://www.codemonkey.org.uk | SuSE Labs
Quoted 2 lines · Dave Jones
This looks bogus. You're killing the
wbinvd()inflush_kernel_map()which is needed.
I must admit I don't exactly know the wbinvd() command, but as the comment says:
/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */
I thought it is not NEEDED, just a COULD...
Thomas
Quoted 7 lines · Thomas Schlichter
Quoted 2 lines
This looks bogus. You're killing the
wbinvd()inflush_kernel_map()which is needed.I must admit I don't exactly know the
wbinvd()command, but as the comment says:
/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */I thought it is not NEEDED, just a COULD...
Its hinting at a possible optimisation, not saying that it is unneeded.
Dave
Signature · 2 lines
| Dave Jones. http://www.codemonkey.org.uk | SuSE Labs
Quoted 2 lines · Dave Jones
Its hinting at a possible optimisation, not saying that it is unneeded.
OK, sorry, than I just misunderstood the comment...
So here is a minimal change patch that should solve the preempt issue in flush_map().
Instead of just doing a preempt_disable() before and a preempt_enable() after the flush_kernel_map() calls I just changed the order so that the preempt point is not between them...
Thomas
arch/i386/mm/pageattr.c renamed from arch/i386/mm/pageattr.c.orig +1 -1
@@ -131,10 +131,10 @@static inline void flush_map (void ) { +flush_kernel_map (NULL );#ifdef CONFIG_SMP smp_call_function (flush_kernel_map,NULL ,1 ,1 );#endif -flush_kernel_map (NULL ); }struct deferred_page {
arch/x86_64/mm/pageattr.c renamed from arch/x86_64/mm/pageattr.c.orig +1 -1
@@ -123,10 +123,10 @@static inline void flush_map (unsigned long address) { +flush_kernel_map ((void *)address);#ifdef CONFIG_SMP smp_call_function (flush_kernel_map, (void *)address,1 ,1 );#endif -flush_kernel_map ((void *)address); }struct deferred_page {
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Quoted 8 lines · Thomas Schlichter
So here is a minimal change patch that should solve the preempt issue in
flush_map().Instead of just doing a
preempt_disable()before and apreempt_enable()after
theflush_kernel_map()calls I just changed the order so that the preempt point is not between them...
No. All that does is make sure that the cpu you start out on is flushed, once or twice, and the cpu you end up on may be missed. Use preempt_disable and preempt_enable.
Hugh
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Quoted 3 lines · Hugh Dickins
No. All that does is make sure that the cpu you start out on is flushed, once or twice, and the cpu you end up on may be missed. Use
preempt_disableandpreempt_enable.
Oh, you are right! I think I am totally stupid this morning...! Now finally I hope this is the correct patch...
Thomas Schlichter
arch/i386/mm/pageattr.c renamed from arch/i386/mm/pageattr.c.orig +6 -2
@@ -131,10 +131,14 @@static inline void flush_map (void ) { -#ifdef CONFIG_SMP +#ifdef CONFIG_SMP +preempt_disable ();smp_call_function (flush_kernel_map,NULL ,1 ,1 ); -#endif flush_kernel_map (NULL ); +preempt_enable (); +#else +flush_kernel_map (NULL ); +#endif }struct deferred_page {
arch/x86_64/mm/pageattr.c renamed from arch/x86_64/mm/pageattr.c.orig +6 -2
@@ -123,10 +123,14 @@static inline void flush_map (unsigned long address) { -#ifdef CONFIG_SMP +#ifdef CONFIG_SMP +preempt_disable ();smp_call_function (flush_kernel_map, (void *)address,1 ,1 ); -#endif flush_kernel_map ((void *)address); +preempt_enable (); +#else +flush_kernel_map ((void *)address); +#endif }struct deferred_page {
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Quoted 6 lines · Thomas Schlichter
Quoted 3 lines
No. All that does is make sure that the cpu you start out on is flushed, once or twice, and the cpu you end up on may be missed. Use
preempt_disableandpreempt_enable.Oh, you are right! I think I am totally stupid this morning...! Now finally I hope this is the correct patch...
That would appear to do what you want, but its an ugly construct to be repeating everywhere that wants to call a function on all CPUs. It would probably clean things up a lot if we had a function to do..
static inline void on_each_cpu(void *func)
{
#ifdef CONFIG_SMP preempt_disable(); smp_call_function(func, NULL, 1, 1); func(NULL); preempt_enable(); #else func(NULL);
#endif
}
Bluesmoke and agpgart could both use this to cleanup some mess, and no doubt there are others
Comments?
Dave
Signature · 1 line
| Dave Jones. http://www.codemonkey.org.uk
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Yes, you are right. I was just looking for this preempt-problem where a flush_tlb* was done, but there are many other places where this problem occours, too... Nearly everywhere where smp_call_function() is used!
I found a function in the file mm/slab.c called smp_call_function_all_cpus() which tries to do the thing we want, but I think not even this function is preempt-safe...!
But here I think I better get off my fingers, I just wanted to help a bit with this issue, but I don't think I have the time and knowledge to solve it completely in the mentioned good way...
Perhaps even the semantic of the function smp_call_function() could be changed to call the function on every CPU? Just an idea...
Best regards
Thomas Schlichter
Quoted 30 lines · Dave Jones
Quoted 6 lines · Thomas Schlichter
Quoted 3 lines
No. All that does is make sure that the cpu you start out on is flushed, once or twice, and the cpu you end up on may be missed. Use
preempt_disableandpreempt_enable.Oh, you are right! I think I am totally stupid this morning...! Now finally I hope this is the correct patch...
That would appear to do what you want, but its an ugly construct to be repeating everywhere that wants to call a function on all CPUs. It would probably clean things up a lot if we had a function to do..
static inline void
on_each_cpu(void *func)
{#ifdef CONFIG_SMP preempt_disable(); smp_call_function(func, NULL, 1, 1); func(NULL); preempt_enable(); #else func(NULL);#endif
}Bluesmoke and agpgart could both use this to cleanup some mess, and no doubt there are others
Comments?
Dave
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Quoted 3 lines · Thomas Schlichter
I found a function in the file
mm/slab.ccalledsmp_call_function_all_cpus()which tries to do the thing we want, but I think not even this function is preempt-safe...!
That certainly looks erm, odd.
Quoted 2 lines
Perhaps even the semantic of the function
smp_call_function()could be changed to call the function on every CPU? Just an idea...
Some places that use it actually rely on it not executing on the local cpu. It would also break backwards compatabiltiy for no real good reason.
Dave
Signature · 2 lines
| Dave Jones. http://www.codemonkey.org.uk | SuSE Labs
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
In reply to Dave JonesQuoted 11 lines · Dave Jones
It would probably clean things up a lot if we had a function to do..
static inline void
on_each_cpu(void *func)
{#ifdef CONFIG_SMP preempt_disable(); smp_call_function(func, NULL, 1, 1); func(NULL); preempt_enable(); #else func(NULL);
Quoted 2 lines
#endif
}
Of course that's much much better. But I think rather better as static inline void on_each_cpu(void (*func) (void *info), void *info) passing info to func instead of assuming NULL. inline? maybe.
Hugh
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
Quoted 3 lines · Hugh Dickins
Of course that's much much better. But I think rather better as static inline void
on_each_cpu(void (*func) (void *info), void *info)passing info to func instead of assuming NULL. inline? maybe.
Sounds good to me. As for inline - This thing will be generating lots of IPIs on SMP, so its hardly going to be performance critical, so we could leave it out of line IMO.
Dave
Signature · 2 lines
| Dave Jones. http://www.codemonkey.org.uk | SuSE Labs
Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()
In reply to Dave JonesQuoted 3 lines · Dave Jones
That would appear to do what you want, but its an ugly construct to be repeating everywhere that wants to call a function on all CPUs. It would probably clean things up a lot if we had a function to do..
Ok, here's a first stab at an implementation. Compiles, but is untested.. Fixes up a few preemption races Thomas highlighted, and converts a few smp_call_function() users over to on_each_cpu(), which saves quite a bit of code.
Dave
arch/i386/kernel/cpuid.c +2 -0
@@ -64,6 +64,7 @@ {struct cpuid_command cmd; +preempt_disable ();if ( cpu ==smp_processor_id () ) {cpuid (reg, &data[0 ], &data[1 ], &data[2 ], &data[3 ]); }else {
@@ -73,6 +74,7 @@ smp_call_function(cpuid_smp_cpuid, &cmd, 1, 1); } +preempt_enable (); }#else /* ! CONFIG_SMP */
arch/i386/kernel/microcode.c +1 -4
@@ -183,11 +183,8 @@int i, error =0 , err;struct microcode *m; -if (smp_call_function (do_update_one,NULL ,1 ,1 ) !=0 ) { -printk (KERN_ERR "microcode: IPI timeout, giving up\n" ); +if (on_each_cpu (do_update_one,NULL )==-1 )return -EIO ; - } -do_update_one (NULL );for (i=0 ; i<NR_CPUS ; i++) { err = update_req[i].err;
arch/i386/kernel/msr.c +6 -1
@@ -115,9 +115,13 @@static inline int do_wrmsr (int cpu,u32 reg,u32 eax,u32 edx) {struct msr_command cmd; +int ret; +preempt_disable ();if ( cpu ==smp_processor_id () ) { -return wrmsr_eio (reg, eax, edx); + ret =wrmsr_eio (reg, eax, edx); +preempt_enable (); +return ret; }else { cmd.cpu = cpu; cmd.reg = reg;
@@ -125,6 +129,7 @@ cmd.data[1 ] = edx;smp_call_function (msr_smp_wrmsr, &cmd,1 ,1 ); +preempt_enable ();return cmd.err; } }
arch/i386/kernel/smp.c +35 -0
@@ -547,6 +547,41 @@return 0 ; } +/* +* [SUMMARY] Run a function on every CPU. +* <func> The function to run. This must be fast and non-blocking. +* <info> An arbitrary pointer to pass to the function. +* <nonatomic> currently unused. +* [RETURNS] 0 on success, else a negative status code. Does not return until +* remote CPUs are nearly ready to execute <<func>> or are or have executed. +* +* You must not call this function with disabled interrupts or from a +* hardware interrupt handler or from a bottom half handler. +*/ +int on_each_cpu (void (*func) (void *info),void *info) +{ +#ifdef CONFIG_SMP +preempt_disable (); + +if (num_online_cpus () ==1 ) + goto only_one; + +if (smp_call_function (func, info,1 ,1 ) !=0 ) { +printk (KERN_ERR "%p: IPI timeout, giving up\n" , +__builtin_return_address (0 )); +preempt_enable (); +return -1 ; + } + +only_one: +func (info); +preempt_enable (); +#else +func (info); +#endif +return 0 ; +} +static void stop_this_cpu (void * dummy) {/*
arch/i386/kernel/sysenter.c +2 -2
@@ -17,6 +17,7 @@#include <asm/msr.h> #include <asm/pgtable.h> #include <asm/unistd.h> +#include <asm/smp.h> extern asmlinkage void sysenter_entry(void);
@@ -97,8 +98,7 @@return 0 ;memcpy ((void *) page, sysent,sizeof (sysent)); -enable_sep_cpu (NULL ); -smp_call_function (enable_sep_cpu,NULL ,1 ,1 ); +on_each_cpu (enable_sep_cpu,NULL );return 0 ; }
arch/i386/mm/pageattr.c +1 -4
@@ -131,10 +131,7 @@static inline void flush_map (void ) { -#ifdef CONFIG_SMP -smp_call_function (flush_kernel_map,NULL ,1 ,1 ); -#endif -flush_kernel_map (NULL ); +on_each_cpu (flush_kernel_map,NULL ); }struct deferred_page {
arch/x86_64/mm/pageattr.c +1 -4
@@ -123,10 +123,7 @@static inline void flush_map (unsigned long address) { -#ifdef CONFIG_SMP -smp_call_function (flush_kernel_map, (void *)address,1 ,1 ); -#endif -flush_kernel_map ((void *)address); +on_each_cpu (flush_kernel_map,(void *)address); }struct deferred_page {
drivers/char/agp/agp.h +1 -15
@@ -34,24 +34,10 @@#define PFX "agpgart: " -#ifdef CONFIG_SMP -static void ipi_handler (void *null) -{ -flush_agp_cache (); -} - -static void __attribute__((unused))global_cache_flush (void ) -{ -if (smp_call_function (ipi_handler,NULL ,1 ,1 ) !=0 ) -panic (PFX "timed out waiting for the other CPUs!\n" ); -flush_agp_cache (); -} -#else static inline void global_cache_flush (void ) { -flush_agp_cache (); +on_each_cpu (flush_agp_cache,NULL ); } -#endif /* !CONFIG_SMP */ enum aper_size_type {U8_APER_SIZE ,
include/asm-i386/agp.h +4 -2
@@ -18,6 +18,8 @@/* Could use CLFLUSH here if the cpu supports it. But then it would need to be called for each cacheline of the whole page so it may not be worth it. Would need a page for it. */ -#define flush_agp_cache () asm volatile("wbinvd":::"memory") - +static void flush_agp_cache (void *info) +{ + __asm__ __volatile__ ("wbinvd" : : :"memory" ); +}#endif
include/asm-i386/smp.h +2 -0
@@ -46,6 +46,8 @@extern void (*mtrr_hook) (void );extern void zap_low_mappings (void ); +extern int on_each_cpu (void (*func) (void *info),void *info); +#define MAX_APICID 256/*
include/asm-x86_64/agp.h +4 -2
@@ -18,6 +18,8 @@/* Could use CLFLUSH here if the cpu supports it. But then it would need to be called for each cacheline of the whole page so it may not be worth it. Would need a page for it. */ -#define flush_agp_cache () asm volatile("wbinvd":::"memory") - +static void flush_agp_cache (void *info) +{ + __asm__ __volatile__ ("wbinvd" : : :"memory" ); +}#endif
include/asm-x86_64/smp.h +2 -0
@@ -46,6 +46,8 @@extern void (*mtrr_hook) (void );extern void zap_low_mappings (void ); +extern int on_each_cpu (void (*func) (void *info),void *info); +#define SMP_TRAMPOLINE_BASE 0x6000/*
Signature · 2 lines
| Dave Jones. http://www.codemonkey.org.uk | SuSE Labs
PATCH 1/1 [2.5] fix preempt-issues with smp_call_function()
Quoted 4 lines · Dave Jones
Ok, here's a first stab at an implementation. Compiles, but is untested.. Fixes up a few preemption races Thomas highlighted, and converts a few
smp_call_function()users over toon_each_cpu(), which saves quite a bit of code.
Your patch was really fine, I just modified it a bit and fixed some more preempt-issues with the smp_call_function() calls. It compiles and workes with no problems so far... I hope I did not make any big mistakes... ;-)
I hope you like this one, too...
Thomas
arch/alpha/kernel/process.c +1 -4
@@ -155,10 +155,7 @@struct halt_info args; args.mode = mode; args.restart_cmd = restart_cmd; -#ifdef CONFIG_SMP -smp_call_function (common_shutdown_1, &args,1 ,0 ); -#endif -common_shutdown_1 (&args); +on_each_cpu (common_shutdown_1, &args,1 ,0 ); }void
arch/alpha/kernel/smp.c +17 -6
@@ -899,10 +899,8 @@smp_imb (void) {/* Must wait other processors to flush their icache before continue. */ -if (smp_call_function (ipi_imb,NULL ,1 ,1 )) +if (on_each_cpu (ipi_imb,NULL ,1 ,1 ))printk (KERN_CRIT "smp_imb: timed out\n" ); - -imb (); } static void
@@ -916,11 +914,9 @@ {/* Although we don't have any data to pass, we do want to synchronize with the other processors. */ -if (smp_call_function (ipi_flush_tlb_all,NULL ,1 ,1 )) { +if (on_each_cpu (ipi_flush_tlb_all,NULL ,1 ,1 )) {printk (KERN_CRIT "flush_tlb_all: timed out\n" ); } - -tbia (); }#define asn_locked () (cpu_data[smp_processor_id()].asn_lock)
@@ -938,6 +934,8 @@void flush_tlb_mm (struct mm_struct *mm) { +preempt_disable (); +if (mm == current->active_mm) {flush_tlb_current (mm);if (atomic_read (&mm->mm_users) <=1 ) {
@@ -948,6 +946,7 @@if (mm->context[cpu]) mm->context[cpu] =0 ; } +preempt_enable ();return ; } }
@@ -955,6 +954,8 @@if (smp_call_function (ipi_flush_tlb_mm, mm,1 ,1 )) {printk (KERN_CRIT "flush_tlb_mm: timed out\n" ); } + +preempt_enable (); }struct flush_tlb_page_struct {
@@ -981,6 +982,8 @@struct flush_tlb_page_struct data;struct mm_struct *mm = vma->vm_mm; +preempt_disable (); +if (mm == current->active_mm) {flush_tlb_current_page (mm, vma, addr);if (atomic_read (&mm->mm_users) <=1 ) {
@@ -991,6 +994,7 @@if (mm->context[cpu]) mm->context[cpu] =0 ; } +preempt_enable ();return ; } }
@@ -1002,6 +1006,8 @@if (smp_call_function (ipi_flush_tlb_page, &data,1 ,1 )) {printk (KERN_CRIT "flush_tlb_page: timed out\n" ); } + +preempt_enable (); }void
@@ -1030,6 +1036,8 @@if ((vma->vm_flags &VM_EXEC ) ==0 )return ; +preempt_disable (); +if (mm == current->active_mm) {__load_new_mm_context (mm);if (atomic_read (&mm->mm_users) <=1 ) {
@@ -1040,6 +1048,7 @@if (mm->context[cpu]) mm->context[cpu] =0 ; } +preempt_enable ();return ; } }
@@ -1047,6 +1056,8 @@if (smp_call_function (ipi_flush_icache_page, mm,1 ,1 )) {printk (KERN_CRIT "flush_icache_page: timed out\n" ); } + +preempt_enable (); }#ifdef CONFIG_DEBUG_SPINLOCK
arch/i386/kernel/cpuid.c +5 -19
@@ -44,8 +44,6 @@#include <asm/uaccess.h> #include <asm/system.h> -#ifdef CONFIG_SMP -struct cpuid_command {int cpu;u32 reg;
@@ -64,24 +62,12 @@ {struct cpuid_command cmd; -if ( cpu ==smp_processor_id () ) { -cpuid (reg, &data[0 ], &data[1 ], &data[2 ], &data[3 ]); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - cmd.data = data; - -smp_call_function (cpuid_smp_cpuid, &cmd,1 ,1 ); - } -} -#else /* ! CONFIG_SMP */ - -static inline void do_cpuid (int cpu,u32 reg,u32 *data) -{ -cpuid (reg, &data[0 ], &data[1 ], &data[2 ], &data[3 ]); + cmd.cpu = cpu; + cmd.reg = reg; + cmd.data = data; + +on_each_cpu (cpuid_smp_cpuid, &cmd,1 ,1 ); } - -#endif /* ! CONFIG_SMP */ static loff_t cpuid_seek (struct file *file,loff_t offset,int orig) {
arch/i386/kernel/io_apic.c +2 -4
@@ -1376,8 +1376,7 @@void print_all_local_APICs (void ) { -smp_call_function (print_local_APIC,NULL ,1 ,1 ); -print_local_APIC (NULL ); +on_each_cpu (print_local_APIC,NULL ,1 ,1 ); }void /*__init*/ print_PIC (void )
@@ -1855,8 +1854,7 @@*/ printk (KERN_INFO "activating NMI Watchdog ..." ); -smp_call_function (enable_NMI_through_LVT0,NULL ,1 ,1 ); -enable_NMI_through_LVT0 (NULL ); +on_each_cpu (enable_NMI_through_LVT0,NULL ,1 ,1 );printk (" done.\n" ); }
arch/i386/kernel/ldt.c +2 -2
@@ -55,13 +55,13 @@wmb ();if (reload) { +preempt_disable ();load_LDT (pc);#ifdef CONFIG_SMP -preempt_disable ();if (current->mm->cpu_vm_mask != (1 <<smp_processor_id ()))smp_call_function (flush_ldt,0 ,1 ,1 ); -preempt_enable ();#endif +preempt_enable (); }if (oldsize) {if (oldsize*LDT_ENTRY_SIZE >PAGE_SIZE )
arch/i386/kernel/microcode.c +1 -4
@@ -183,11 +183,8 @@int i, error =0 , err;struct microcode *m; -if (smp_call_function (do_update_one,NULL ,1 ,1 ) !=0 ) { -printk (KERN_ERR "microcode: IPI timeout, giving up\n" ); +if (on_each_cpu (do_update_one,NULL ,1 ,1 ) !=0 )return -EIO ; - } -do_update_one (NULL );for (i=0 ; i<NR_CPUS ; i++) { err = update_req[i].err;
arch/i386/kernel/msr.c +15 -23
@@ -116,36 +116,28 @@ {struct msr_command cmd; -if ( cpu ==smp_processor_id () ) { -return wrmsr_eio (reg, eax, edx); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - cmd.data[0 ] = eax; - cmd.data[1 ] = edx; - -smp_call_function (msr_smp_wrmsr, &cmd,1 ,1 ); -return cmd.err; - } + cmd.cpu = cpu; + cmd.reg = reg; + cmd.data[0 ] = eax; + cmd.data[1 ] = edx; + +on_each_cpu (msr_smp_wrmsr, &cmd,1 ,1 ); +return cmd.err; }static inline int do_rdmsr (int cpu,u32 reg,u32 *eax,u32 *edx) {struct msr_command cmd; -if ( cpu ==smp_processor_id () ) { -return rdmsr_eio (reg, eax, edx); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - -smp_call_function (msr_smp_rdmsr, &cmd,1 ,1 ); - - *eax = cmd.data[0 ]; - *edx = cmd.data[1 ]; + cmd.cpu = cpu; + cmd.reg = reg; + +on_each_cpu (msr_smp_rdmsr, &cmd,1 ,1 ); + + *eax = cmd.data[0 ]; + *edx = cmd.data[1 ]; -return cmd.err; - } +return cmd.err; }#else /* ! CONFIG_SMP */
arch/i386/kernel/smp.c +2 -11
@@ -436,7 +436,7 @@preempt_enable (); } -static inline void do_flush_tlb_all_local (void ) +void do_flush_tlb_all (void * info) {unsigned long cpu =smp_processor_id ();
@@ -445,18 +445,9 @@leave_mm (cpu); } -static void flush_tlb_all_ipi (void * info) -{ -do_flush_tlb_all_local (); -} -void flush_tlb_all (void ) { -preempt_disable (); -smp_call_function (flush_tlb_all_ipi,0 ,1 ,1 ); - -do_flush_tlb_all_local (); -preempt_enable (); +on_each_cpu (do_flush_tlb_all,0 ,1 ,1 ); }/*
arch/i386/kernel/sysenter.c +1 -2
@@ -95,8 +95,7 @@return 0 ;memcpy ((void *) page, sysent,sizeof (sysent)); -enable_sep_cpu (NULL ); -smp_call_function (enable_sep_cpu,NULL ,1 ,1 ); +on_each_cpu (enable_sep_cpu,NULL ,1 ,1 );return 0 ; }
arch/i386/mach-voyager/voyager_smp.c +3 -11
@@ -1209,8 +1209,8 @@smp_call_function_interrupt (); } -static inline void -do_flush_tlb_all_local (void ) +void +do_flush_tlb_all (void * info) {unsigned long cpu =smp_processor_id ();
@@ -1220,19 +1220,11 @@ } -static void -flush_tlb_all_function (void * info) -{ -do_flush_tlb_all_local (); -} -/* flush the TLB of every active CPU in the system */ void flush_tlb_all (void ) { -smp_call_function (flush_tlb_all_function,0 ,1 ,1 ); - -do_flush_tlb_all_local (); +on_each_cpu (do_flush_tlb_all,0 ,1 ,1 ); }/* used to set up the trampoline for other CPUs when the memory manager
arch/i386/mm/pageattr.c +2 -5
@@ -130,11 +130,8 @@ }static inline void flush_map (void ) -{ -#ifdef CONFIG_SMP -smp_call_function (flush_kernel_map,NULL ,1 ,1 ); -#endif -flush_kernel_map (NULL ); +{ +on_each_cpu (flush_kernel_map,NULL ,1 ,1 ); }struct deferred_page {
arch/i386/oprofile/nmi_int.c +4 -8
@@ -95,8 +95,7 @@* without actually triggering any NMIs as this will * break the core code horrifically. */ -smp_call_function (nmi_cpu_setup,NULL ,0 ,1 ); -nmi_cpu_setup (0 ); +on_each_cpu (nmi_cpu_setup,NULL ,0 ,1 );set_nmi_callback (nmi_callback); oprofile_pmdev =set_nmi_pm_callback (oprofile_pm_callback);return 0 ;
@@ -148,8 +147,7 @@ {unset_nmi_pm_callback (oprofile_pmdev);unset_nmi_callback (); -smp_call_function (nmi_cpu_shutdown,NULL ,0 ,1 ); -nmi_cpu_shutdown (0 ); +on_each_cpu (nmi_cpu_shutdown,NULL ,0 ,1 ); }
@@ -162,8 +160,7 @@static int nmi_start (void ) { -smp_call_function (nmi_cpu_start,NULL ,0 ,1 ); -nmi_cpu_start (0 ); +on_each_cpu (nmi_cpu_start,NULL ,0 ,1 );return 0 ; }
@@ -177,8 +174,7 @@static void nmi_stop (void ) { -smp_call_function (nmi_cpu_stop,NULL ,0 ,1 ); -nmi_cpu_stop (0 ); +on_each_cpu (nmi_cpu_stop,NULL ,0 ,1 ); }
arch/ia64/kernel/smp.c +5 -5
@@ -206,18 +206,18 @@void smp_flush_tlb_all (void ) { -smp_call_function ((void (*)(void *))local_flush_tlb_all,0 ,1 ,1 ); -local_flush_tlb_all (); +on_each_cpu ((void (*)(void *))local_flush_tlb_all,0 ,1 ,1 ); }void smp_flush_tlb_mm (struct mm_struct *mm) { -local_finish_flush_tlb_mm (mm); -/* this happens for the common case of a single-threaded fork(): */ if (likely (mm == current->active_mm &&atomic_read (&mm->mm_users) ==1 )) + { +local_finish_flush_tlb_mm (mm);return ; + }/* * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
@@ -226,7 +226,7 @@* anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is * rather trivial. */ -smp_call_function ((void (*)(void *))local_finish_flush_tlb_mm, mm,1 ,1 ); +on_each_cpu ((void (*)(void *))local_finish_flush_tlb_mm, mm,1 ,1 ); }/*
arch/mips64/kernel/smp.c +13 -2
@@ -195,8 +195,7 @@void flush_tlb_all (void ) { -smp_call_function (flush_tlb_all_ipi,0 ,1 ,1 ); -_flush_tlb_all (); +on_each_cpu (flush_tlb_all_ipi,0 ,1 ,1 ); }static void flush_tlb_mm_ipi (void *mm)
@@ -219,6 +218,8 @@void flush_tlb_mm (struct mm_struct *mm) { +preempt_disable (); +if ((atomic_read (&mm->mm_users) !=1 ) || (current->mm != mm)) {smp_call_function (flush_tlb_mm_ipi, (void *)mm,1 ,1 ); }else {
@@ -228,6 +229,8 @@CPU_CONTEXT (i, mm) =0 ; }_flush_tlb_mm (mm); + +preempt_enable (); }struct flush_tlb_data {
@@ -246,6 +249,8 @@void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end) { +preempt_disable (); +if ((atomic_read (&mm->mm_users) !=1 ) || (current->mm != mm)) {struct flush_tlb_data fd;
@@ -260,6 +265,8 @@CPU_CONTEXT (i, mm) =0 ; }_flush_tlb_range (mm, start, end); + +preempt_enable (); }static void flush_tlb_page_ipi (void *info)
@@ -271,6 +278,8 @@void flush_tlb_page (struct vm_area_struct *vma,unsigned long page) { +preempt_disable (); +if ((atomic_read (&vma->vm_mm->mm_users) !=1 ) || (current->mm != vma->vm_mm)) {struct flush_tlb_data fd;
@@ -284,5 +293,7 @@CPU_CONTEXT (i, vma->vm_mm) =0 ; }_flush_tlb_page (vma, page); + +preempt_enable (); }
arch/parisc/kernel/cache.c +1 -2
@@ -39,8 +39,7 @@void flush_data_cache (void ) { -smp_call_function ((void (*)(void *))flush_data_cache_local,NULL ,1 ,1 ); -flush_data_cache_local (); +on_each_cpu ((void (*)(void *))flush_data_cache_local,NULL ,1 ,1 ); }#endif
arch/parisc/kernel/irq.c +3 -8
@@ -61,20 +61,17 @@static spinlock_t irq_lock =SPIN_LOCK_UNLOCKED ;/* protect IRQ regions */ -#ifdef CONFIG_SMP static void cpu_set_eiem (void *info) {set_eiem ((unsigned long ) info); } -#endif static inline void disable_cpu_irq (void *unused,int irq) {unsigned long eirr_bit =EIEM_MASK (irq); cpu_eiem &= ~eirr_bit; -set_eiem (cpu_eiem); -smp_call_function (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); +on_each_cpu (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); }static void enable_cpu_irq (void *unused,int irq)
@@ -83,8 +80,7 @@mtctl (eirr_bit,23 );/* clear EIRR bit before unmasking */ cpu_eiem |= eirr_bit; -smp_call_function (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); -set_eiem (cpu_eiem); +on_each_cpu (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); }/* mask and disable are the same at the CPU level
@@ -100,8 +96,7 @@** handle *any* unmasked pending interrupts. ** ie We don't need to check for pending interrupts here. */ -smp_call_function (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); -set_eiem (cpu_eiem); +on_each_cpu (cpu_set_eiem, (void *) cpu_eiem,1 ,1 ); }/*
arch/parisc/kernel/smp.c +2 -3
@@ -401,7 +401,7 @@__setup ("maxcpus=" , maxcpus);/* -* Flush all other CPU's tlb and then mine. Do this with smp_call_function() +* Flush all other CPU's tlb and then mine. Do this with on_each_cpu() * as we want to ensure all TLB's flushed before proceeding. */
@@ -410,8 +410,7 @@void smp_flush_tlb_all (void ) { -smp_call_function ((void (*)(void *))flush_tlb_all_local,NULL ,1 ,1 ); -flush_tlb_all_local (); +on_each_cpu ((void (*)(void *))flush_tlb_all_local,NULL ,1 ,1 ); }
arch/parisc/mm/init.c +1 -2
@@ -974,8 +974,7 @@ do_recycle++; }spin_unlock (&sid_lock); -smp_call_function ((void (*)(void *))flush_tlb_all_local,NULL ,1 ,1 ); -flush_tlb_all_local (); +on_each_cpu ((void (*)(void *))flush_tlb_all_local,NULL ,1 ,1 );if (do_recycle) {spin_lock (&sid_lock);recycle_sids (recycle_ndirty,recycle_dirty_array);
arch/ppc/kernel/temp.c +2 -8
@@ -194,10 +194,7 @@/* schedule ourselves to be run again */ mod_timer (&tau_timer, jiffies + shrink_timer) ; -#ifdef CONFIG_SMP -smp_call_function (tau_timeout,NULL ,1 ,0 ); -#endif -tau_timeout (NULL ); +on_each_cpu (tau_timeout,NULL ,1 ,0 ); }/*
@@ -239,10 +236,7 @@ tau_timer.expires = jiffies + shrink_timer;add_timer (&tau_timer); -#ifdef CONFIG_SMP -smp_call_function (TAU_init_smp,NULL ,1 ,0 ); -#endif -TAU_init_smp (NULL ); +on_each_cpu (TAU_init_smp,NULL ,1 ,0 );printk ("Thermal assist unit " );#ifdef CONFIG_TAU_INT
arch/s390/kernel/smp.c +6 -12
@@ -228,8 +228,7 @@void machine_restart_smp (char * __unused) { cpu_restart_map = cpu_online_map; -smp_call_function (do_machine_restart,NULL ,0 ,0 ); -do_machine_restart (NULL ); +on_each_cpu (do_machine_restart,NULL ,0 ,0 ); }static void do_machine_halt (void * __unused)
@@ -247,8 +246,7 @@void machine_halt_smp (void ) { -smp_call_function (do_machine_halt,NULL ,0 ,0 ); -do_machine_halt (NULL ); +on_each_cpu (do_machine_halt,NULL ,0 ,0 ); }static void do_machine_power_off (void * __unused)
@@ -266,8 +264,7 @@void machine_power_off_smp (void ) { -smp_call_function (do_machine_power_off,NULL ,0 ,0 ); -do_machine_power_off (NULL ); +on_each_cpu (do_machine_power_off,NULL ,0 ,0 ); }/*
@@ -339,8 +336,7 @@void smp_ptlb_all (void ) { -smp_call_function (smp_ptlb_callback,NULL ,0 ,1 ); -local_flush_tlb (); +on_each_cpu (smp_ptlb_callback,NULL ,0 ,1 ); }/*
@@ -400,8 +396,7 @@ parms.end_ctl = cr; parms.orvals[cr] =1 << bit; parms.andvals[cr] =0xFFFFFFFF ; -smp_call_function (smp_ctl_bit_callback, &parms,0 ,1 ); -__ctl_set_bit (cr, bit); +on_each_cpu (smp_ctl_bit_callback, &parms,0 ,1 ); }/*
@@ -414,8 +409,7 @@ parms.end_ctl = cr; parms.orvals[cr] =0x00000000 ; parms.andvals[cr] = ~(1 << bit); -smp_call_function (smp_ctl_bit_callback, &parms,0 ,1 ); -__ctl_clear_bit (cr, bit); +on_each_cpu (smp_ctl_bit_callback, &parms,0 ,1 ); }/*
arch/s390x/kernel/smp.c +5 -10
@@ -227,8 +227,7 @@void machine_restart_smp (char * __unused) { cpu_restart_map = cpu_online_map; -smp_call_function (do_machine_restart,NULL ,0 ,0 ); -do_machine_restart (NULL ); +on_each_cpu (do_machine_restart,NULL ,0 ,0 ); }static void do_machine_halt (void * __unused)
@@ -246,8 +245,7 @@void machine_halt_smp (void ) { -smp_call_function (do_machine_halt,NULL ,0 ,0 ); -do_machine_halt (NULL ); +on_each_cpu (do_machine_halt,NULL ,0 ,0 ); }static void do_machine_power_off (void * __unused)
@@ -265,8 +263,7 @@void machine_power_off_smp (void ) { -smp_call_function (do_machine_power_off,NULL ,0 ,0 ); -do_machine_power_off (NULL ); +on_each_cpu (do_machine_power_off,NULL ,0 ,0 ); }/*
@@ -383,8 +380,7 @@ parms.end_ctl = cr; parms.orvals[cr] =1 << bit; parms.andvals[cr] =-1L ; -smp_call_function (smp_ctl_bit_callback, &parms,0 ,1 ); -__ctl_set_bit (cr, bit); +on_each_cpu (smp_ctl_bit_callback, &parms,0 ,1 ); }/*
@@ -397,8 +393,7 @@ parms.end_ctl = cr; parms.orvals[cr] =0 ; parms.andvals[cr] = ~(1L << bit); -smp_call_function (smp_ctl_bit_callback, &parms,0 ,1 ); -__ctl_clear_bit (cr, bit); +on_each_cpu (smp_ctl_bit_callback, &parms,0 ,1 ); }
arch/x86_64/kernel/bluesmoke.c +1 -13
@@ -111,11 +111,7 @@ {u32 low, high;int i; -unsigned int *cpu = info; -BUG_ON (*cpu !=smp_processor_id ()); - -preempt_disable ();for (i=0 ; i<banks; i++) {rdmsr (MSR_IA32_MC0_STATUS +i*4 , low, high);
@@ -130,20 +126,12 @@wmb (); } } -preempt_enable (); }static void mce_timerfunc (unsigned long data) { -unsigned int i; - -for (i=0 ; i<smp_num_cpus; i++) { -if (i ==smp_processor_id ()) -mce_checkregs (&i); -else -smp_call_function (mce_checkregs, &i,1 ,1 ); - } +on_each_cpu (mce_checkregs,NULL ,1 ,1 );/* Refresh the timer. */ mce_timer.expires = jiffies +MCE_RATE ;
arch/x86_64/kernel/cpuid.c +6 -10
@@ -63,16 +63,12 @@static inline void do_cpuid (int cpu,u32 reg,u32 *data) {struct cpuid_command cmd; - -if ( cpu ==smp_processor_id () ) { -cpuid (reg, &data[0 ], &data[1 ], &data[2 ], &data[3 ]); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - cmd.data = data; - -smp_call_function (cpuid_smp_cpuid, &cmd,1 ,1 ); - } + + cmd.cpu = cpu; + cmd.reg = reg; + cmd.data = data; + +on_each_cpu (cpuid_smp_cpuid, &cmd,1 ,1 ); }#else /* ! CONFIG_SMP */
arch/x86_64/kernel/io_apic.c +1 -2
@@ -926,8 +926,7 @@void print_all_local_APICs (void ) { -smp_call_function (print_local_APIC,NULL ,1 ,1 ); -print_local_APIC (NULL ); +on_each_cpu (print_local_APIC,NULL ,1 ,1 ); }void /*__init*/ print_PIC (void )
arch/x86_64/kernel/ldt.c +2 -2
@@ -60,13 +60,13 @@ pc->size = mincount;wmb ();if (reload) { +preempt_disable ();load_LDT (pc);#ifdef CONFIG_SMP -preempt_disable ();if (current->mm->cpu_vm_mask != (1 <<smp_processor_id ()))smp_call_function (flush_ldt,0 ,1 ,1 ); -preempt_enable ();#endif +preempt_enable (); }if (oldsize) {if (oldsize*LDT_ENTRY_SIZE >PAGE_SIZE )
arch/x86_64/kernel/msr.c +15 -23
@@ -120,36 +120,28 @@ {struct msr_command cmd; -if ( cpu ==smp_processor_id () ) { -return wrmsr_eio (reg, eax, edx); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - cmd.data[0 ] = eax; - cmd.data[1 ] = edx; - -smp_call_function (msr_smp_wrmsr, &cmd,1 ,1 ); -return cmd.err; - } + cmd.cpu = cpu; + cmd.reg = reg; + cmd.data[0 ] = eax; + cmd.data[1 ] = edx; + +on_each_cpu (msr_smp_wrmsr, &cmd,1 ,1 ); +return cmd.err; }static inline int do_rdmsr (int cpu,u32 reg,u32 *eax,u32 *edx) {struct msr_command cmd; -if ( cpu ==smp_processor_id () ) { -return rdmsr_eio (reg, eax, edx); - }else { - cmd.cpu = cpu; - cmd.reg = reg; - -smp_call_function (msr_smp_rdmsr, &cmd,1 ,1 ); - - *eax = cmd.data[0 ]; - *edx = cmd.data[1 ]; + cmd.cpu = cpu; + cmd.reg = reg; + +on_each_cpu (msr_smp_rdmsr, &cmd,1 ,1 ); + + *eax = cmd.data[0 ]; + *edx = cmd.data[1 ]; -return cmd.err; - } +return cmd.err; }#else /* ! CONFIG_SMP */
arch/x86_64/kernel/smp.c +2 -9
@@ -328,7 +328,7 @@preempt_enable (); } -static inline void do_flush_tlb_all_local (void ) +void do_flush_tlb_all (void * info) {unsigned long cpu =smp_processor_id ();
@@ -337,16 +337,9 @@leave_mm (cpu); } -static void flush_tlb_all_ipi (void * info) -{ -do_flush_tlb_all_local (); -} -void flush_tlb_all (void ) { -smp_call_function (flush_tlb_all_ipi,0 ,1 ,1 ); - -do_flush_tlb_all_local (); +on_each_cpu (do_flush_tlb_all,0 ,1 ,1 ); }void smp_kdb_stop (void )
arch/x86_64/mm/pageattr.c +2 -5
@@ -122,11 +122,8 @@ }static inline void flush_map (unsigned long address) -{ -#ifdef CONFIG_SMP -smp_call_function (flush_kernel_map, (void *)address,1 ,1 ); -#endif -flush_kernel_map ((void *)address); +{ +on_each_cpu (flush_kernel_map, (void *)address,1 ,1 ); }struct deferred_page {
drivers/char/agp/agp.h +1 -15
@@ -34,24 +34,10 @@#define PFX "agpgart: " -#ifdef CONFIG_SMP -static void ipi_handler (void *null) -{ -flush_agp_cache (); -} - -static void __attribute__((unused))global_cache_flush (void ) -{ -if (smp_call_function (ipi_handler,NULL ,1 ,1 ) !=0 ) -panic (PFX "timed out waiting for the other CPUs!\n" ); -flush_agp_cache (); -} -#else static inline void global_cache_flush (void ) { -flush_agp_cache (); +on_each_cpu (flush_agp_cache,NULL ,1 ,1 ); } -#endif /* !CONFIG_SMP */ enum aper_size_type {U8_APER_SIZE ,
drivers/s390/char/sclp.c +1 -2
@@ -481,8 +481,7 @@do_machine_quiesce (void) { cpu_quiesce_map = cpu_online_map; -smp_call_function (do_load_quiesce_psw,NULL ,0 ,0 ); -do_load_quiesce_psw (NULL ); +on_each_cpu (do_load_quiesce_psw,NULL ,0 ,0 ); }#else static void
drivers/s390/net/iucv.c +2 -8
@@ -617,10 +617,7 @@ulong b2f0_result =0x0deadbeef ;iucv_debug (1 ,"entering" ); -if (smp_processor_id () ==0 ) -iucv_declare_buffer_cpu0 (&b2f0_result); -else -smp_call_function (iucv_declare_buffer_cpu0, &b2f0_result,0 ,1 ); +on_each_cpu (iucv_declare_buffer_cpu0, &b2f0_result,0 ,1 );iucv_debug (1 ,"Address of EIB = %p" , iucv_external_int_buffer);if (b2f0_result ==0x0deadbeef ) b2f0_result =0xaa ;
@@ -639,10 +636,7 @@ {iucv_debug (1 ,"entering" );if (declare_flag) { -if (smp_processor_id () ==0 ) -iucv_retrieve_buffer_cpu0 (0 ); -else -smp_call_function (iucv_retrieve_buffer_cpu0,0 ,0 ,1 ); +on_each_cpu (iucv_retrieve_buffer_cpu0,0 ,0 ,1 ); declare_flag =0 ; }iucv_debug (1 ,"exiting" );
fs/buffer.c +1 -4
@@ -1404,10 +1404,7 @@static void invalidate_bh_lrus (void ) { -preempt_disable (); -invalidate_bh_lru (NULL ); -smp_call_function (invalidate_bh_lru,NULL ,1 ,1 ); -preempt_enable (); +on_each_cpu (invalidate_bh_lru,NULL ,1 ,1 ); } void set_bh_page(struct buffer_head *bh,
include/asm-alpha/agp.h +5 -1
@@ -8,6 +8,10 @@#define map_page_into_agp (page) #define unmap_page_from_agp(page)#define flush_agp_mappings () -#define flush_agp_cache() mb() + +static void flush_agp_cache(void* info) +{ +mb (); +}#endif
include/asm-i386/agp.h +4 -1
@@ -18,6 +18,9 @@/* Could use CLFLUSH here if the cpu supports it. But then it would need to be called for each cacheline of the whole page so it may not be worth it. Would need a page for it. */ -#define flush_agp_cache () asm volatile("wbinvd":::"memory") +static void flush_agp_cache (void *info) +{ + __asm__ __volatile__ ("wbinvd" : : :"memory" ); +}#endif
include/asm-ia64/agp.h +8 -1
@@ -16,7 +16,14 @@#define map_page_into_agp (page)/* nothing */ #define unmap_page_from_agp (page)/* nothing */ #define flush_agp_mappings ()/* nothing */ -#define flush_agp_cache () mb() + +/* Could use CLFLUSH here if the cpu supports it. But then it would +need to be called for each cacheline of the whole page so it may not be +worth it. Would need a page for it. */ +static void flush_agp_cache (void * info) +{ +mb (); +}/* Page-protection value to be used for AGP memory mapped into kernel space. */ #define PAGE_AGP PAGE_KERNEL
include/asm-parisc/cacheflush.h +1 -7
@@ -25,16 +25,10 @@extern void flush_cache_all_local (void ); -#ifdef CONFIG_SMP static inline void flush_cache_all (void ) { -smp_call_function ((void (*)(void *))flush_cache_all_local,NULL ,1 ,1 ); -flush_cache_all_local (); +on_each_cpu ((void (*)(void *))flush_cache_all_local,NULL ,1 ,1 ); } -#else -#define flush_cache_all flush_cache_all_local -#endif -/* The following value needs to be tuned and probably scaled with the * cache size.
include/asm-sparc64/agp.h +8 -1
@@ -6,6 +6,13 @@#define map_page_into_agp (page) #define unmap_page_from_agp(page)#define flush_agp_mappings () -#define flush_agp_cache() mb() + +/* Could use CLFLUSH here if the cpu supports it. But then it would +need to be called for each cacheline of the whole page so it may not be +worth it. Would need a page for it. */ +static void flush_agp_cache (void * info) +{ +mb (); +}#endif
include/asm-x86_64/agp.h +4 -1
@@ -18,6 +18,9 @@/* Could use CLFLUSH here if the cpu supports it. But then it would need to be called for each cacheline of the whole page so it may not be worth it. Would need a page for it. */ -#define flush_agp_cache () asm volatile("wbinvd":::"memory") +static void flush_agp_cache (void * info) +{ + __asm__ __volatile__ ("wbinvd" : : :"memory" ); +}#endif
include/linux/smp.h +28 -1
@@ -10,9 +10,10 @@#ifdef CONFIG_SMP +#include <linux/preempt.h> #include <linux/kernel.h> #include <linux/compiler.h> -#include <linux/threads.h> +#include <linux/thread_info.h> #include <asm/smp.h> #include <asm/bug.h>
@@ -54,6 +55,31 @@ int retry, int wait);/* +* Call a function on all processors +*/ +static inline int on_each_cpu (void (*func) (void *info),void *info, +int retry,int wait) +{ +int ret =0 ; + +preempt_disable (); + +if (num_online_cpus () ==1 ) + goto only_one; + + ret =smp_call_function (func, info, retry, wait); +if (ret !=0 ) +printk (KERN_ERR "%p: IPI timeout, giving up\n" , +__builtin_return_address (0 )); + +only_one: +func (info); +preempt_enable (); + +return ret; +} + +/* * True once the per process idle is forked */ extern int smp_threads_ready;
@@ -96,6 +122,7 @@#define hard_smp_processor_id () 0#define smp_threads_ready 1#define smp_call_function (func,info,retry,wait) ({ 0; }) +#define on_each_cpu (func,info,retry,wait) ({ func(info); 0; })static inline void smp_send_reschedule (int cpu) { }static inline void smp_send_reschedule_all (void ) { }#define cpu_online_map 1
mm/slab.c +4 -0
@@ -1116,12 +1116,16 @@static void smp_call_function_all_cpus (void (*func) (void *arg),void *arg) {check_irq_on (); +preempt_disable (); +local_irq_disable ();func (arg);local_irq_enable ();if (smp_call_function (func, arg,1 ,1 ))BUG (); + +preempt_enable (); }static void free_block (kmem_cache_t * cachep,void ** objpp,int len);