x86: remove extraneous stack cpumask variables
Subsystems (4)
cover x86: remove extraneous stack cpumask variables
Some more minor cleanups in cpumask stack usage with no new cpumask functions (except for the for_each_online_cpu_mask_nr call). Frees up more memory for the stack. Also, cleans up the genapic API to always pass pointers (for both 32 and 64) and to not return cpumask function values.
* Add for_each_online_cpu_mask_nr to eliminate a common case of needing a temporary on-stack cpumask_t variable.
* Change the genapic->send_IPI_mask function to accept cpumask_t pointer.
* Rewrite "send_IPI_allbutself" to not need a temp cpumask_t variable.
* Change send_IPI_mask function in xen to use for_each_online_cpu_mask_nr().
* Modify internal io_apic.c functions to accept cpumask_t pointer where possible.
Applies to linux-2.6.tip/master.
---
Summary from allmodconfig (NR_CPUS=4k, #NODES=512)
====== Stack (-l 500)
... files 4 vars 144 all 0 lim 500 unch 0
1 - initial
3 - send_IPI_mask
4 - stack-hogs-kernel_sched_c
.1. .3. .4. ..final..
1608 -1008 . 600 -62% setup_IO_APIC_irq 1592 -1024 . 568 -64% msi_compose_msg 1576 . -512 1064 -32% move_task_off_dead_cpu 1560 -1024 . 536 -65% setup_ioapic_dest 1560 -1008 . 552 -64% migrate_ioapic_irq 1560 -1032 . 528 -66% arch_setup_ht_irq 1560 -1008 . 552 -64% __assign_irq_vector 1552 -1024 . 528 -65% ir_set_msi_irq_affinity 1144 -512 . 632 -44% setup_IO_APIC 1048 -520 . 528 -49% setup_timer_IRQ0_pin 1048 -1048 . . -100% native_machine_crash_shutdown 1048 -512 . 536 -48% native_flush_tlb_others 1040 -512 . 528 -49% set_msi_irq_affinity 1040 -512 . 528 -49% set_ht_irq_affinity 1040 -512 . 528 -49% hpet_msi_set_affinity 1040 -512 . 528 -49% dmar_msi_set_affinity 1032 -1032 . . -100% set_ioapic_affinity_irq 1032 -520 . 512 -50% irq_complete_move 1032 -520 . 512 -50% create_irq_nr 1024 -1024 . . -100% x2apic_vector_allocation_domain 1024 -1024 . . -100% x2apic_send_IPI_allbutself 1024 -1024 . . -100% uv_vector_allocation_domain 1024 -1024 . . -100% uv_send_IPI_allbutself 1024 -1024 . . -100% physflat_send_IPI_allbutself 1024 -1024 . . -100% native_send_call_func_ipi 1024 -1024 . . -100% flat_send_IPI_allbutself 552 . -552 . -100% free_sched_groups 520 -520 . . -100% x2apic_send_IPI_all 520 -520 . . -100% uv_send_IPI_all 520 -520 . . -100% set_ir_ioapic_affinity_irq 520 -520 . . -100% physflat_send_IPI_mask 520 -520 . . -100% physflat_send_IPI_all 520 -520 . . -100% native_smp_send_reschedule 520 -520 . . -100% native_send_call_func_single_ipi 520 -520 . . -100% lapic_timer_broadcast 512 -512 . . -100% xen_smp_send_call_function_single_ipi 512 -512 . . -100% xen_smp_send_call_function_ipi 512 -512 . . -100% ioapic_retrigger_irq 512 -512 . . -100% flat_send_IPI_all 512 . -512 . -100% cpu_to_allnodes_group 512 -512 . . -100% assign_irq_vector
39064 -27192 -1576 10296 -73% Totals
====== MemInfo ()
... files 0 vars 0 all 0 lim 0 unch 0
PATCH 1/3 cpumask: add for_each_online_cpu_mask_nr function
* Add for_each_online_cpu_mask_nr() function to eliminate need for
a common use of a temporary cpumask_t variable. When the following
procedure is being used:
funcproto(cpumask_t *mask, ...)
{
cpumask_t temp;
cpus_and(temp, *mask, cpu_online_map);
for_each_cpu_mask_nr(cpu, temp)
...
If then becomes:
funcproto(cpumask_t *mask, ...)
{
for_each_online_cpu_mask_nr(cpu, *mask)
... * Note the generic __next_cpu_and (and __next_cpu_and_nr) functions allowing AND'ing with any cpumask_t variable, not just the cpu_online_map.
Applies to linux-2.6.tip/master.
--- include/linux/cpumask.h | 26 +++++++++++++++++++++++--- lib/cpumask.c | 26 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-)
include/linux/cpumask.h +23 -3
@@ -404,23 +404,32 @@ static inline void __cpus_fold(cpumask_t#define first_cpu (src) ({ (void)(src); 0; })#define next_cpu (n, src) ({ (void)(src); 1; })#define any_online_cpu (mask) 0 -#define for_each_cpu_mask (cpu, mask) \ + +#define for_each_cpu_mask (cpu, mask) \ for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) +#define for_each_online_cpu_mask (cpu, mask) \ + for_each_cpu_mask(cpu, mask)#else /* NR_CPUS > 1 */ extern int nr_cpu_ids;int __first_cpu (const cpumask_t *srcp);int __next_cpu (int n,const cpumask_t *srcp); +int __next_cpu_and (int n,const cpumask_t *srcp,const cpumask_t *andp);int __any_online_cpu (const cpumask_t *mask);#define first_cpu (src) __first_cpu(&(src))#define next_cpu (n, src) __next_cpu((n), &(src))#define any_online_cpu (mask) __any_online_cpu(&(mask)) +#define for_each_cpu_mask (cpu, mask) \ for ((cpu) = -1; \ (cpu) = next_cpu((cpu), (mask)), \ (cpu) < NR_CPUS; ) +#define for_each_online_cpu_mask (cpu, mask) \ + for ((cpu) = -1; \ + (cpu) = __next_cpu_and((cpu), &(mask), &(cpu_online_map)), \ + (cpu) < NR_CPUS; )#endif #if NR_CPUS <=64
@@ -428,17 +437,28 @@ int __any_online_cpu(const cpumask_t *ma#define next_cpu_nr (n, src) next_cpu(n, src)#define cpus_weight_nr (cpumask) cpus_weight(cpumask)#define for_each_cpu_mask_nr (cpu, mask) for_each_cpu_mask(cpu, mask) +#define for_each_online_cpu_mask_nr (cpu, mask) \ + for_each_online_cpu_mask(cpu, mask)#else /* NR_CPUS > 64 */ int __next_cpu_nr (int n,const cpumask_t *srcp); -#define next_cpu_nr (n, src) __next_cpu_nr((n), &(src)) -#define cpus_weight_nr (cpumask) __cpus_weight(&(cpumask), nr_cpu_ids) +int __next_cpu_and_nr (int n,const cpumask_t *srcp,const cpumask_t *andp); + +#define next_cpu_nr (n, src) __next_cpu_nr((n), &(src)) +#define next_cpu_and_nr (n, src, and) __next_cpu_and_nr((n), &(src), &(and)) +#define cpus_weight_nr (cpumask) __cpus_weight(&(cpumask), nr_cpu_ids) +#define for_each_cpu_mask_nr (cpu, mask) \ for ((cpu) = -1; \ (cpu) = next_cpu_nr((cpu), (mask)), \ (cpu) < nr_cpu_ids; ) +#define for_each_online_cpu_mask_nr (cpu, mask) \ + for ((cpu) = -1; \ + (cpu) = next_cpu_and_nr((cpu), (mask), (cpu_online_map)), \ + (cpu) < nr_cpu_ids; ) +#endif /* NR_CPUS > 64 */ /*
lib/cpumask.c +26 -0
@@ -14,6 +14,19 @@ int __next_cpu(int n, const cpumask_t *sreturn min_t (int,NR_CPUS ,find_next_bit (srcp->bits,NR_CPUS , n+1 )); }EXPORT_SYMBOL (__next_cpu); +int __next_cpu_and (int n,const cpumask_t *srcp,const cpumask_t *andp) +{ +int cpu; + +for (cpu = n +1 ; cpu <NR_CPUS ; cpu++) { + cpu =find_next_bit (srcp->bits,NR_CPUS , cpu); + +if (cpu <NR_CPUS &&cpu_isset (cpu, *andp)) +return cpu; + } +return NR_CPUS ; +} +EXPORT_SYMBOL (__next_cpu_and);#if NR_CPUS >64 int __next_cpu_nr (int n,const cpumask_t *srcp)
@@ -22,6 +35,19 @@ int __next_cpu_nr(int n, const cpumask_t find_next_bit(srcp->bits, nr_cpu_ids, n+1)); }EXPORT_SYMBOL (__next_cpu_nr); +int __next_cpu_and_nr (int n,const cpumask_t *srcp,const cpumask_t *andp) +{ +int cpu; + +for (cpu = n +1 ; cpu < nr_cpu_ids; cpu++) { + cpu =find_next_bit (srcp->bits, nr_cpu_ids, cpu); + +if (cpu < nr_cpu_ids &&cpu_isset (cpu, *andp)) +return cpu; + } +return nr_cpu_ids; +} +EXPORT_SYMBOL (__next_cpu_and_nr);#endif int __any_online_cpu (const cpumask_t *mask)
Re: [PATCH 1/3] cpumask: add for_each_online_cpu_mask_nr function
Quoted 19 lines · On Wednesday 10 September 2008 01:50:14 Mike Travis wrote:
* Add
for_each_online_cpu_mask_nr()function to eliminate need fora common use of a temporary cpumask_t variable. When the following procedure is being used: funcproto(cpumask_t *mask, ...) { cpumask_t temp; cpus_and(temp, *mask, cpu_online_map); for_each_cpu_mask_nr(cpu, temp) ... If then becomes: funcproto(cpumask_t *mask, ...) { for_each_online_cpu_mask_nr(cpu, *mask) ...
Quoted 3 lines
* Note the generic
__next_cpu_and(and__next_cpu_and_nr) functions allowing AND'ing with anycpumask_tvariable, not just thecpu_online_map.
Good idea! But I really dislike the _nr versions (too many names!). Do we really need them, since by definition cpus after nr_cpu_ids are never online...
(And we should initialize nr_cpu_ids to NR_CPUS so even early boot works, if we don't already...).
Cheers,
Rusty.
Re: [PATCH 1/3] cpumask: add for_each_online_cpu_mask_nr function
Quoted 28 lines · Rusty Russell
Quoted 19 lines · On Wednesday 10 September 2008 01:50:14 Mike Travis wrote:
* Add
for_each_online_cpu_mask_nr()function to eliminate need fora common use of a temporary cpumask_t variable. When the following procedure is being used: funcproto(cpumask_t *mask, ...) { cpumask_t temp; cpus_and(temp, *mask, cpu_online_map); for_each_cpu_mask_nr(cpu, temp) ... If then becomes: funcproto(cpumask_t *mask, ...) { for_each_online_cpu_mask_nr(cpu, *mask) ...Quoted 3 lines
* Note the generic
__next_cpu_and(and__next_cpu_and_nr) functions allowing AND'ing with anycpumask_tvariable, not just thecpu_online_map.Good idea! But I really dislike the
_nrversions (too many names!). Do we really need them, since by definition cpus afternr_cpu_idsare never online...
Quoted 5 lines
(And we should initialize
nr_cpu_idstoNR_CPUSso even early boot works, if we don't already...).Cheers,
Rusty.
Yes, the only reason the _nr is there is to be consistent with the current for_each_cpu_mask{,_nr} functions. What I'd like to do is convert all calls to use the nr_cpu_ids and if there's some reason to need to iterate over NR_CPUS range, then you'd use FOR_EACH_CPU_MASK.
And yes, nr_cpu_ids is init'd to NR_CPUS.
Thanks,
Mike
PATCH 2/3 x86: modify send_IPI_mask interface to accept cpumask_t pointers
In reply to Mike Travis * Change genapic interfaces to accept cpumask_t pointers and to not return cpumask function values.
* Modify external callers to use these cpumask_t pointers in their calls.
* Create new send_IPI_mask_allbutself which is the same as the send_IPI_mask functions but removes smp_processor_id() from list. This removes another common need for a temporary cpumask_t variable.
* Rewrite functions that used a temp cpumask_t variable for:
cpumask_t allbutme = cpu_online_map;
cpu_clear(smp_processor_id(), allbutme);
if (!cpus_empty(allbutme))
...
It becomes:
if (!cpus_equal(cpu_online_map, cpumask_of_cpu(cpu)))
...* Other minor code optimizations.
Applies to linux-2.6.tip/master.
--- arch/x86/kernel/apic.c | 2 arch/x86/kernel/crash.c | 5 - arch/x86/kernel/genapic_flat_64.c | 54 +++++++------ arch/x86/kernel/genx2apic_cluster.c | 38 +++++---- arch/x86/kernel/genx2apic_phys.c | 31 ++++--- arch/x86/kernel/genx2apic_uv_x.c | 30 +++---- arch/x86/kernel/io_apic.c | 124 ++++++++++++++++--------------- arch/x86/kernel/ipi.c | 26 ++++-- arch/x86/kernel/smp.c | 13 +-- arch/x86/kernel/tlb_32.c | 2 arch/x86/kernel/tlb_64.c | 2 arch/x86/mach-generic/bigsmp.c | 5 - arch/x86/mach-generic/es7000.c | 5 - arch/x86/mach-generic/numaq.c | 5 - arch/x86/mach-generic/summit.c | 5 - arch/x86/xen/smp.c | 15 +-- include/asm-x86/bigsmp/apic.h | 4 - include/asm-x86/bigsmp/ipi.h | 13 +-- include/asm-x86/es7000/apic.h | 8 +- include/asm-x86/es7000/ipi.h | 12 +-- include/asm-x86/genapic_32.h | 6 - include/asm-x86/genapic_64.h | 8 +- include/asm-x86/ipi.h | 21 ++++- include/asm-x86/mach-default/mach_apic.h | 20 ++--- include/asm-x86/mach-default/mach_ipi.h | 16 +--- include/asm-x86/mach-generic/mach_apic.h | 2 include/asm-x86/numaq/apic.h | 2 include/asm-x86/numaq/ipi.h | 13 +-- include/asm-x86/summit/apic.h | 8 +- include/asm-x86/summit/ipi.h | 13 +-- 30 files changed, 265 insertions(+), 243 deletions(-)
arch/x86/kernel/apic.c +1 -1
@@ -460,7 +460,7 @@ static void lapic_timer_setup(enum clockstatic void lapic_timer_broadcast (cpumask_t mask) {#ifdef CONFIG_SMP -send_IPI_mask (mask,LOCAL_TIMER_VECTOR ); +send_IPI_mask (&mask,LOCAL_TIMER_VECTOR );#endif }
arch/x86/kernel/crash.c +1 -4
@@ -77,10 +77,7 @@ static int crash_nmi_callback(struct notstatic void smp_send_nmi_allbutself (void ) { -cpumask_t mask = cpu_online_map; -cpu_clear (safe_smp_processor_id (), mask); -if (!cpus_empty (mask)) -send_IPI_mask (mask,NMI_VECTOR ); +send_IPI_allbutself (NMI_VECTOR ); }static struct notifier_block crash_nmi_nb = {
arch/x86/kernel/genapic_flat_64.c +30 -24
132 lines not on this page · raw message
arch/x86/kernel/genx2apic_cluster.c +21 -17
@@ -30,11 +30,10 @@ static cpumask_t x2apic_target_cpus(void/* * for now each logical cpu is in its own vector allocation domain. */ -static cpumask_t x2apic_vector_allocation_domain (int cpu) +static void x2apic_vector_allocation_domain (int cpu,cpumask_t *retmask) { -cpumask_t domain =CPU_MASK_NONE ; -cpu_set (cpu, domain); -return domain; +cpus_clear (*retmask); +cpu_set (cpu, *retmask); } static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
@@ -56,32 +55,37 @@ static void __x2apic_send_IPI_dest(unsig* at once. We have 16 cpu's in a cluster. This will minimize IPI register * writes. */ -static void x2apic_send_IPI_mask (cpumask_t mask,int vector) +static void x2apic_send_IPI_mask (const cpumask_t *mask,int vector) {unsigned long flags;unsigned long query_cpu;local_irq_save (flags); -for_each_cpu_mask (query_cpu, mask) { -__x2apic_send_IPI_dest (per_cpu (x86_cpu_to_logical_apicid, query_cpu), - vector,APIC_DEST_LOGICAL ); - } +for_each_cpu_mask_nr (query_cpu, *mask) +__x2apic_send_IPI_dest ( +per_cpu (x86_cpu_to_logical_apicid, query_cpu), + vector,APIC_DEST_LOGICAL );local_irq_restore (flags); }static void x2apic_send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; - -cpu_clear (smp_processor_id (), mask); +unsigned long flags; +unsigned long query_cpu; +unsigned long this_cpu =smp_processor_id (); -if (!cpus_empty (mask)) -x2apic_send_IPI_mask (mask, vector); +local_irq_save (flags); +for_each_online_cpu (query_cpu) +if (query_cpu != this_cpu) +__x2apic_send_IPI_dest ( +per_cpu (x86_cpu_to_logical_apicid, query_cpu), + vector,APIC_DEST_LOGICAL ); +local_irq_restore (flags); }static void x2apic_send_IPI_all (int vector) { -x2apic_send_IPI_mask (cpu_online_map, vector); +x2apic_send_IPI_mask (&cpu_online_map, vector); }static int x2apic_apic_id_registered (void )
@@ -89,7 +93,7 @@ static int x2apic_apic_id_registered(voireturn 1 ; } -static unsigned int x2apic_cpu_mask_to_apicid (cpumask_t cpumask) +static unsigned int x2apic_cpu_mask_to_apicid (const cpumask_t *cpumask) {int cpu;
@@ -97,7 +101,7 @@ static unsigned int x2apic_cpu_mask_to_a* We're using fixed IRQ delivery, can only return one phys APIC ID. * May as well be the first. */ - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask);if ((unsigned )cpu <NR_CPUS )return per_cpu (x86_cpu_to_logical_apicid, cpu);else
arch/x86/kernel/genx2apic_phys.c +17 -14
@@ -34,11 +34,10 @@ static cpumask_t x2apic_target_cpus(voidreturn cpumask_of_cpu (0 ); } -static cpumask_t x2apic_vector_allocation_domain (int cpu) +static void x2apic_vector_allocation_domain (int cpu,cpumask_t *retmask) { -cpumask_t domain =CPU_MASK_NONE ; -cpu_set (cpu, domain); -return domain; +cpus_clear (*retmask); +cpu_set (cpu, *retmask); } static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
@@ -54,13 +53,13 @@ static void __x2apic_send_IPI_dest(unsigx2apic_icr_write (cfg, apicid); } -static void x2apic_send_IPI_mask (cpumask_t mask,int vector) +static void x2apic_send_IPI_mask (const cpumask_t *mask,int vector) {unsigned long flags;unsigned long query_cpu;local_irq_save (flags); -for_each_cpu_mask (query_cpu, mask) { +for_each_cpu_mask_nr (query_cpu, *mask) {__x2apic_send_IPI_dest (per_cpu (x86_cpu_to_apicid, query_cpu), vector,APIC_DEST_PHYSICAL ); }
@@ -69,17 +68,21 @@ static void x2apic_send_IPI_mask(cpumaskstatic void x2apic_send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; - -cpu_clear (smp_processor_id (), mask); +unsigned long flags; +unsigned long query_cpu; +unsigned long this_cpu =smp_processor_id (); -if (!cpus_empty (mask)) -x2apic_send_IPI_mask (mask, vector); +local_irq_save (flags); +for_each_online_cpu (query_cpu) +if (query_cpu != this_cpu) +__x2apic_send_IPI_dest (per_cpu (x86_cpu_to_apicid, query_cpu), + vector,APIC_DEST_PHYSICAL ); +local_irq_restore (flags); }static void x2apic_send_IPI_all (int vector) { -x2apic_send_IPI_mask (cpu_online_map, vector); +x2apic_send_IPI_mask (&cpu_online_map, vector); }static int x2apic_apic_id_registered (void )
@@ -87,7 +90,7 @@ static int x2apic_apic_id_registered(voireturn 1 ; } -static unsigned int x2apic_cpu_mask_to_apicid (cpumask_t cpumask) +static unsigned int x2apic_cpu_mask_to_apicid (const cpumask_t *cpumask) {int cpu;
@@ -95,7 +98,7 @@ static unsigned int x2apic_cpu_mask_to_a* We're using fixed IRQ delivery, can only return one phys APIC ID. * May as well be the first. */ - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask);if ((unsigned )cpu <NR_CPUS )return per_cpu (x86_cpu_to_apicid, cpu);else
arch/x86/kernel/genx2apic_uv_x.c +14 -16
@@ -81,11 +81,10 @@ static cpumask_t uv_target_cpus(void)return cpumask_of_cpu (0 ); } -static cpumask_t uv_vector_allocation_domain (int cpu) +static void uv_vector_allocation_domain (int cpu,cpumask_t *retmask) { -cpumask_t domain =CPU_MASK_NONE ; -cpu_set (cpu, domain); -return domain; +cpus_clear (*retmask); +cpu_set (cpu, *retmask); }int uv_wakeup_secondary (int phys_apicid,unsigned int start_rip)
@@ -124,28 +123,27 @@ static void uv_send_IPI_one(int cpu, intuv_write_global_mmr64 (pnode,UVH_IPI_INT , val); } -static void uv_send_IPI_mask (cpumask_t mask,int vector) +static void uv_send_IPI_mask (const cpumask_t *mask,int vector) {unsigned int cpu; -for_each_possible_cpu (cpu) -if (cpu_isset (cpu, mask)) -uv_send_IPI_one (cpu, vector); +for_each_cpu_mask_nr (cpu, *mask) +uv_send_IPI_one (cpu, vector); }static void uv_send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; - -cpu_clear (smp_processor_id (), mask); +unsigned int cpu; +unsigned int this_cpu =smp_processor_id (); -if (!cpus_empty (mask)) -uv_send_IPI_mask (mask, vector); +for_each_online_cpu (cpu) +if (cpu != this_cpu) +uv_send_IPI_one (cpu, vector); }static void uv_send_IPI_all (int vector) { -uv_send_IPI_mask (cpu_online_map, vector); +uv_send_IPI_mask (&cpu_online_map, vector); }static int uv_apic_id_registered (void )
@@ -157,7 +155,7 @@ static void uv_init_apic_ldr(void) { } -static unsigned int uv_cpu_mask_to_apicid (cpumask_t cpumask) +static unsigned int uv_cpu_mask_to_apicid (const cpumask_t *cpumask) {int cpu;
@@ -165,7 +163,7 @@ static unsigned int uv_cpu_mask_to_apici* We're using fixed IRQ delivery, can only return one phys APIC ID. * May as well be the first. */ - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask);if ((unsigned )cpu < nr_cpu_ids)return per_cpu (x86_cpu_to_apicid, cpu);else
arch/x86/kernel/io_apic.c +64 -60
402 lines not on this page · raw message
arch/x86/kernel/ipi.c +19 -7
@@ -116,9 +116,9 @@ static inline void __send_IPI_dest_field/* * This is only used on smaller machines. */ -void send_IPI_mask_bitmask (cpumask_t cpumask,int vector) +void send_IPI_mask_bitmask (const cpumask_t *cpumask,int vector) { -unsigned long mask =cpus_addr (cpumask)[0 ]; +unsigned long mask =cpus_addr (*cpumask)[0 ];unsigned long flags;local_irq_save (flags);
@@ -127,7 +127,7 @@ void send_IPI_mask_bitmask(cpumask_t cpulocal_irq_restore (flags); } -void send_IPI_mask_sequence (cpumask_t mask,int vector) +void send_IPI_mask_sequence (const cpumask_t *mask,int vector) {unsigned long flags;unsigned int query_cpu;
@@ -139,12 +139,24 @@ void send_IPI_mask_sequence(cpumask_t ma*/ local_irq_save (flags); -for_each_possible_cpu (query_cpu) { -if (cpu_isset (query_cpu, mask)) { +for_each_cpu_mask_nr (query_cpu, *mask) +__send_IPI_dest_field (cpu_to_logical_apicid (query_cpu), vector); +local_irq_restore (flags); +} + +void send_IPI_mask_allbutself (const cpumask_t *mask,int vector) +{ +unsigned long flags; +unsigned int query_cpu; +unsigned int this_cpu =smp_processor_id (); + +/* See Hack comment above */ + +local_irq_save (flags); +for_each_cpu_mask_nr (query_cpu, *mask) +if (query_cpu != this_cpu)__send_IPI_dest_field (cpu_to_logical_apicid (query_cpu), vector); - } - }local_irq_restore (flags); }
arch/x86/kernel/smp.c +5 -8
@@ -118,26 +118,23 @@ static void native_smp_send_reschedule(iWARN_ON (1 );return ; } -send_IPI_mask (cpumask_of_cpu (cpu),RESCHEDULE_VECTOR ); +send_IPI_mask (&cpumask_of_cpu (cpu),RESCHEDULE_VECTOR ); }void native_send_call_func_single_ipi (int cpu) { -send_IPI_mask (cpumask_of_cpu (cpu),CALL_FUNCTION_SINGLE_VECTOR ); +send_IPI_mask (&cpumask_of_cpu (cpu),CALL_FUNCTION_SINGLE_VECTOR ); }void native_send_call_func_ipi (const cpumask_t *mask) { -cpumask_t allbutself; +int cpu =smp_processor_id (); - allbutself = cpu_online_map; -cpu_clear (smp_processor_id (), allbutself); - -if (cpus_equal (*mask, allbutself) && +if (!cpus_equal (cpu_online_map,cpumask_of_cpu (cpu)) &&cpus_equal (cpu_online_map, cpu_callout_map))send_IPI_allbutself (CALL_FUNCTION_VECTOR );else -send_IPI_mask (*mask,CALL_FUNCTION_VECTOR ); +send_IPI_mask (mask,CALL_FUNCTION_VECTOR ); }static void stop_this_cpu (void *dummy)
arch/x86/kernel/tlb_32.c +1 -1
@@ -158,7 +158,7 @@ void native_flush_tlb_others(const cpuma* We have to send the IPI only to * CPUs affected. */ -send_IPI_mask (cpumask,INVALIDATE_TLB_VECTOR ); +send_IPI_mask (&cpumask,INVALIDATE_TLB_VECTOR );while (!cpus_empty (flush_cpumask))/* nothing. lockup detection does not belong here */
arch/x86/kernel/tlb_64.c +1 -1
@@ -186,7 +186,7 @@ void native_flush_tlb_others(const cpuma* We have to send the IPI only to * CPUs affected. */ -send_IPI_mask (cpumask,INVALIDATE_TLB_VECTOR_START + sender); +send_IPI_mask (&cpumask,INVALIDATE_TLB_VECTOR_START + sender);while (!cpus_empty (f->flush_cpumask))cpu_relax ();
arch/x86/mach-generic/bigsmp.c +3 -2
@@ -41,9 +41,10 @@ static const struct dmi_system_id bigsmp { } }; -static cpumask_t vector_allocation_domain (int cpu) +static vector_allocation_domain(int cpu, cpumask_t *retmask) { -return cpumask_of_cpu (cpu); + cpus_clear(*retmask); +cpu_set (cpu, *retmask); }static int probe_bigsmp (void )
arch/x86/mach-generic/es7000.c +2 -3
@@ -65,7 +65,7 @@ static int __init acpi_madt_oem_check(ch }#endif -static cpumask_t vector_allocation_domain (int cpu) +static void vector_allocation_domain (int cpu,cpumask_t *retmask) {/* Careful. Some cpus do not strictly honor the set of cpus * specified in the interrupt destination when using lowest
@@ -75,8 +75,7 @@ static cpumask_t vector_allocation_domai* deliver interrupts to the wrong hyperthread when only one * hyperthread was specified in the interrupt desitination. */ -cpumask_t domain = { { [0 ] =APIC_ALL_CPUS , } }; -return domain; + *retmask = (cpumask_t ) { { [0 ] =APIC_ALL_CPUS , } }; } struct genapic __initdata_refok apic_es7000 = APIC_INIT("es7000", probe_es7000);
arch/x86/mach-generic/numaq.c +2 -3
@@ -38,7 +38,7 @@ static int acpi_madt_oem_check(char *oemreturn 0 ; } -static cpumask_t vector_allocation_domain (int cpu) +static void vector_allocation_domain (int cpu,cpumask_t *retmask) {/* Careful. Some cpus do not strictly honor the set of cpus * specified in the interrupt destination when using lowest
@@ -48,8 +48,7 @@ static cpumask_t vector_allocation_domai* deliver interrupts to the wrong hyperthread when only one * hyperthread was specified in the interrupt desitination. */ -cpumask_t domain = { { [0 ] =APIC_ALL_CPUS , } }; -return domain; + *retmask = (cpumask_t ) { { [0 ] =APIC_ALL_CPUS , } }; }struct genapic apic_numaq =APIC_INIT ("NUMAQ" , probe_numaq);
arch/x86/mach-generic/summit.c +2 -3
@@ -23,7 +23,7 @@ static int probe_summit(void)return 0 ; } -static cpumask_t vector_allocation_domain (int cpu) +static void vector_allocation_domain (int cpu,cpumask_t *retmask) {/* Careful. Some cpus do not strictly honor the set of cpus * specified in the interrupt destination when using lowest
@@ -33,8 +33,7 @@ static cpumask_t vector_allocation_domai* deliver interrupts to the wrong hyperthread when only one * hyperthread was specified in the interrupt desitination. */ -cpumask_t domain = { { [0 ] =APIC_ALL_CPUS , } }; -return domain; + *retmask = (cpumask_t ) { { [0 ] =APIC_ALL_CPUS , } }; }struct genapic apic_summit =APIC_INIT ("summit" , probe_summit);
arch/x86/xen/smp.c +7 -8
@@ -408,24 +408,22 @@ static void xen_smp_send_reschedule(intxen_send_IPI_one (cpu,XEN_RESCHEDULE_VECTOR ); } -static void xen_send_IPI_mask (cpumask_t mask,enum ipi_vector vector) +static void xen_send_IPI_mask (const cpumask_t *mask,enum ipi_vector vector) {unsigned cpu; -cpus_and (mask, mask, cpu_online_map); - -for_each_cpu_mask_nr (cpu, mask) +for_each_online_cpu_mask_nr (cpu, *mask)xen_send_IPI_one (cpu, vector); } -static void xen_smp_send_call_function_ipi (const cpumask_t *mask) +static void xen_smp_send_call_function_ipi (const cpumask_t mask) {int cpu; -xen_send_IPI_mask (*mask,XEN_CALL_FUNCTION_VECTOR ); +xen_send_IPI_mask (&mask,XEN_CALL_FUNCTION_VECTOR );/* Make sure other vcpus get a chance to run if they need to. */ -for_each_cpu_mask_nr (cpu, *mask) { +for_each_cpu_mask_nr (cpu, mask) {if (xen_vcpu_stolen (cpu)) {HYPERVISOR_sched_op (SCHEDOP_yield,0 );break ;
@@ -435,7 +433,8 @@ static void xen_smp_send_call_function_istatic void xen_smp_send_call_function_single_ipi (int cpu) { -xen_send_IPI_mask (cpumask_of_cpu (cpu),XEN_CALL_FUNCTION_SINGLE_VECTOR ); +xen_send_IPI_mask (&cpumask_of_cpu (cpu), +XEN_CALL_FUNCTION_SINGLE_VECTOR ); }static irqreturn_t xen_call_function_interrupt (int irq,void *dev_id)
include/asm-x86/bigsmp/apic.h +2 -2
@@ -121,12 +121,12 @@ static inline int check_phys_apicid_pres }/* As we are using single CPU as destination, pick only one CPU here */ -static inline unsigned int cpu_mask_to_apicid (cpumask_t cpumask) +static inline unsigned int cpu_mask_to_apicid (const cpumask_t *cpumask) {int cpu;int apicid; - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask); apicid =cpu_to_logical_apicid (cpu);return apicid; }
include/asm-x86/bigsmp/ipi.h +5 -8
@@ -1,25 +1,22 @@#ifndef __ASM_MACH_IPI_H#define __ASM_MACH_IPI_H -void send_IPI_mask_sequence (cpumask_t mask,int vector); +void send_IPI_mask_sequence (cpumask_t *mask,int vector); +void send_IPI_mask_allbutself (cpumask_t *mask,int vector); -static inline void send_IPI_mask (cpumask_t mask,int vector) +static inline void send_IPI_mask (cpumask_t *mask,int vector) {send_IPI_mask_sequence (mask, vector); }static inline void send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; -cpu_clear (smp_processor_id (), mask); - -if (!cpus_empty (mask)) -send_IPI_mask (mask, vector); +send_IPI_mask_allbutself (&cpu_online_map, vector); }static inline void send_IPI_all (int vector) { -send_IPI_mask (cpu_online_map, vector); +send_IPI_mask (&cpu_online_map, vector); }#endif /* __ASM_MACH_IPI_H */
include/asm-x86/es7000/apic.h +4 -4
@@ -144,14 +144,14 @@ static inline int check_phys_apicid_presreturn (1 ); } -static inline unsigned int cpu_mask_to_apicid (cpumask_t cpumask) +static inline unsigned int cpu_mask_to_apicid (const cpumask_t *cpumask) {int num_bits_set;int cpus_found =0 ;int cpu;int apicid; - num_bits_set =cpus_weight (cpumask); + num_bits_set =cpus_weight (*cpumask);/* Return id to all */ if (num_bits_set == NR_CPUS)#if definedCONFIG_ES7000_CLUSTERED_APIC
@@ -163,10 +163,10 @@ static inline unsigned int cpu_mask_to_a* The cpus in the mask must all be on the apic cluster. If are not * on the same apicid cluster return default value of TARGET_CPUS. */ - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask); apicid =cpu_to_logical_apicid (cpu);while (cpus_found < num_bits_set) { -if (cpu_isset (cpu, cpumask)) { +if (cpu_isset (cpu, *cpumask)) {int new_apicid =cpu_to_logical_apicid (cpu);if (apicid_cluster (apicid) !=apicid_cluster (new_apicid)){
include/asm-x86/es7000/ipi.h +5 -7
@@ -1,24 +1,22 @@#ifndef __ASM_ES7000_IPI_H#define __ASM_ES7000_IPI_H -void send_IPI_mask_sequence (cpumask_t mask,int vector); +void send_IPI_mask_sequence (cpumask_t *mask,int vector); +void send_IPI_mask_allbutself (cpumask_t *mask,int vector); -static inline void send_IPI_mask (cpumask_t mask,int vector) +static inline void send_IPI_mask (cpumask_t *mask,int vector) {send_IPI_mask_sequence (mask, vector); }static inline void send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; -cpu_clear (smp_processor_id (), mask); -if (!cpus_empty (mask)) -send_IPI_mask (mask, vector); +send_IPI_mask_allbutself (&cpu_online_map, vector); }static inline void send_IPI_all (int vector) { -send_IPI_mask (cpu_online_map, vector); +send_IPI_mask (&cpu_online_map, vector); }#endif /* __ASM_ES7000_IPI_H */
include/asm-x86/genapic_32.h +3 -3
@@ -56,12 +56,12 @@ struct genapic {unsigned (*get_apic_id)(unsigned long x);unsigned long apic_id_mask; -unsigned int (*cpu_mask_to_apicid)(cpumask_t cpumask); -cpumask_t (*vector_allocation_domain)(int cpu); +unsigned int (*cpu_mask_to_apicid)(const cpumask_t *cpumask); +void (*vector_allocation_domain)(int cpu,cpumask_t *retmask);#ifdef CONFIG_SMP /* ipi */ -void (*send_IPI_mask)(cpumask_t mask,int vector); +void (*send_IPI_mask)(const cpumask_t *mask,int vector);void (*send_IPI_allbutself)(int vector);void (*send_IPI_all)(int vector);#endif
include/asm-x86/genapic_64.h +4 -4
@@ -18,16 +18,16 @@ struct genapic {u32 int_delivery_mode;u32 int_dest_mode;int (*apic_id_registered)(void ); -cpumask_t (*target_cpus)(void); -cpumask_t (*vector_allocation_domain)(int cpu); +void (*target_cpus)(cpumask_t *retmask); +void (*vector_allocation_domain)(int cpu,cpumask_t *retmask);void (*init_apic_ldr)(void );/* ipi */ -void (*send_IPI_mask)(cpumask_t mask,int vector); +void (*send_IPI_mask)(const cpumask_t *mask,int vector);void (*send_IPI_allbutself)(int vector);void (*send_IPI_all)(int vector);void (*send_IPI_self)(int vector);/* */ -unsigned int (*cpu_mask_to_apicid)(cpumask_t cpumask); +unsigned int (*cpu_mask_to_apicid)(const cpumask_t *cpumask);unsigned int (*phys_pkg_id)(int index_msb);unsigned int (*get_apic_id)(unsigned long x);unsigned long (*set_apic_id)(unsigned int id);
include/asm-x86/ipi.h +19 -2
@@ -117,7 +117,7 @@ static inline void __send_IPI_dest_fieldnative_apic_mem_write (APIC_ICR , cfg); } -static inline void send_IPI_mask_sequence (cpumask_t mask,int vector) +static inline void send_IPI_mask_sequence (const cpumask_t *mask,int vector) {unsigned long flags;unsigned long query_cpu;
@@ -128,11 +128,28 @@ static inline void send_IPI_mask_sequenc* - mbligh */ local_irq_save (flags); -for_each_cpu_mask_nr (query_cpu, mask) { +for_each_cpu_mask_nr (query_cpu, *mask) {__send_IPI_dest_field (per_cpu (x86_cpu_to_apicid, query_cpu), vector,APIC_DEST_PHYSICAL ); } local_irq_restore(flags); } +static inline void send_IPI_mask_allbutself (cpumask_t *mask,int vector) +{ +unsigned long flags; +unsigned int query_cpu; +unsigned int this_cpu =smp_processor_id (); + +/* See Hack comment above */ + +local_irq_save (flags); +for_each_cpu_mask_nr (query_cpu, *mask) +if (query_cpu != this_cpu) +__send_IPI_dest_field ( +per_cpu (x86_cpu_to_apicid, query_cpu), + vector,APIC_DEST_PHYSICAL ); +local_irq_restore (flags); +} +#endif /* ASM_X86__IPI_H */
include/asm-x86/mach-default/mach_apic.h +10 -10
@@ -8,12 +8,13 @@#define APIC_DFR_VALUE (APIC_DFR_FLAT) -static inline cpumask_t target_cpus (void ) +static inline void target_cpus (cpumask_t *retmask) {#ifdef CONFIG_SMP -return cpu_online_map; + *retmask = cpu_online_map;#else -return cpumask_of_cpu (0 ); +cpus_clear (*retmask); +cpu_set (0 , *retmask);#endif }
@@ -24,7 +25,7 @@ static inline cpumask_t target_cpus(void#include <asm/genapic.h> #define INT_DELIVERY_MODE (genapic->int_delivery_mode)#define INT_DEST_MODE (genapic->int_dest_mode) -#define TARGET_CPUS (genapic->target_cpus()) +#define TARGET_CPUS (genapic->target_cpus)#define apic_id_registered (genapic->apic_id_registered)#define init_apic_ldr (genapic->init_apic_ldr)#define cpu_mask_to_apicid (genapic->cpu_mask_to_apicid)
@@ -36,7 +37,7 @@ extern void setup_apic_routing(void);#else #define INT_DELIVERY_MODE dest_LowestPrio#define INT_DEST_MODE 1/* logical delivery broadcast to all procs */ -#define TARGET_CPUS (target_cpus()) +#define TARGET_CPUS (target_cpus)/* * Set up the logical destination ID. *
@@ -59,9 +60,9 @@ static inline int apic_id_registered(voireturn physid_isset (read_apic_id (), phys_cpu_present_map); } -static inline unsigned int cpu_mask_to_apicid (cpumask_t cpumask) +static inline unsigned int cpu_mask_to_apicid (const cpumask_t *cpumask) { -return cpus_addr (cpumask)[0 ]; +return cpus_addr (*cpumask)[0 ]; }static inline u32 phys_pkg_id (u32 cpuid_apic,int index_msb)
@@ -86,7 +87,7 @@ static inline int apicid_to_node(int log#endif } -static inline cpumask_t vector_allocation_domain (int cpu) +static inline void vector_allocation_domain (int cpu,cpumask_t *retmask) {/* Careful. Some cpus do not strictly honor the set of cpus * specified in the interrupt destination when using lowest
@@ -96,8 +97,7 @@ static inline cpumask_t vector_allocatio* deliver interrupts to the wrong hyperthread when only one * hyperthread was specified in the interrupt desitination. */ -cpumask_t domain = { { [0 ] =APIC_ALL_CPUS , } }; - return domain; + *retmask = (cpumask_t) { { [0] = APIC_ALL_CPUS, } }; }#endif
include/asm-x86/mach-default/mach_ipi.h +7 -9
@@ -4,7 +4,8 @@/* Avoid include hell */ #define NMI_VECTOR 0x02 -void send_IPI_mask_bitmask (cpumask_t mask,int vector); +void send_IPI_mask_bitmask (const cpumask_t *mask,int vector); +void send_IPI_mask_allbutself (const cpumask_t *mask,int vector);void __send_IPI_shortcut (unsigned int shortcut,int vector);extern int no_broadcast;
@@ -13,7 +14,7 @@ extern int no_broadcast;#include <asm/genapic.h> #define send_IPI_mask (genapic->send_IPI_mask)#else -static inline void send_IPI_mask (cpumask_t mask,int vector) +static inline void send_IPI_mask (const cpumask_t *mask,int vector) {send_IPI_mask_bitmask (mask, vector); }
@@ -21,19 +22,16 @@ static inline void send_IPI_mask(cpumaskstatic inline void __local_send_IPI_allbutself (int vector) { -if (no_broadcast || vector ==NMI_VECTOR ) { -cpumask_t mask = cpu_online_map; - -cpu_clear (smp_processor_id (), mask); -send_IPI_mask (mask, vector); - }else +if (no_broadcast || vector ==NMI_VECTOR ) +send_IPI_mask_allbutself (&cpu_online_map, vector); +else __send_IPI_shortcut (APIC_DEST_ALLBUT , vector); }static inline void __local_send_IPI_all (int vector) {if (no_broadcast || vector ==NMI_VECTOR ) -send_IPI_mask (cpu_online_map, vector); +send_IPI_mask (&cpu_online_map, vector);else __send_IPI_shortcut (APIC_DEST_ALLINC , vector); }
include/asm-x86/mach-generic/mach_apic.h +1 -1
@@ -9,7 +9,7 @@#define INT_DEST_MODE (genapic->int_dest_mode)#undef APIC_DEST_LOGICAL#define APIC_DEST_LOGICAL (genapic->apic_destination_logical) -#define TARGET_CPUS (genapic->target_cpus()) +#define TARGET_CPUS (genapic->target_cpus)#define apic_id_registered (genapic->apic_id_registered)#define init_apic_ldr (genapic->init_apic_ldr)#define ioapic_phys_id_map (genapic->ioapic_phys_id_map)
include/asm-x86/numaq/apic.h +1 -1
@@ -122,7 +122,7 @@ static inline void enable_apic_mode(void* We use physical apicids here, not logical, so just return the default * physical broadcast to stop people from breaking us */ -static inline unsigned int cpu_mask_to_apicid (cpumask_t cpumask) +static inline unsigned int cpu_mask_to_apicid (const cpumask_t *cpumask) {return (int )0xF ; }
include/asm-x86/numaq/ipi.h +5 -8
@@ -1,25 +1,22 @@#ifndef __ASM_NUMAQ_IPI_H#define __ASM_NUMAQ_IPI_H -void send_IPI_mask_sequence (cpumask_t ,int vector); +void send_IPI_mask_sequence (const cpumask_t *mask,int vector); +void send_IPI_mask_allbutself (const cpumask_t *mask,int vector); -static inline void send_IPI_mask (cpumask_t mask,int vector) +static inline void send_IPI_mask (const cpumask_t *mask,int vector) {send_IPI_mask_sequence (mask, vector); }static inline void send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; -cpu_clear (smp_processor_id (), mask); - -if (!cpus_empty (mask)) -send_IPI_mask (mask, vector); +send_IPI_mask_allbutself (&cpu_online_map, vector); }static inline void send_IPI_all (int vector) { -send_IPI_mask (cpu_online_map, vector); +send_IPI_mask (&cpu_online_map, vector); }#endif /* __ASM_NUMAQ_IPI_H */
include/asm-x86/summit/apic.h +4 -4
@@ -137,14 +137,14 @@ static inline void enable_apic_mode(void { } -static inline unsigned int cpu_mask_to_apicid (cpumask_t cpumask) +static inline unsigned int cpu_mask_to_apicid (const cpumask_t *cpumask) {int num_bits_set;int cpus_found =0 ;int cpu;int apicid; - num_bits_set =cpus_weight (cpumask); + num_bits_set =cpus_weight (*cpumask);/* Return id to all */ if (num_bits_set ==NR_CPUS )return (int )0xFF ;
@@ -152,10 +152,10 @@ static inline unsigned int cpu_mask_to_a* The cpus in the mask must all be on the apic cluster. If are not * on the same apicid cluster return default value of TARGET_CPUS. */ - cpu =first_cpu (cpumask); + cpu =first_cpu (*cpumask); apicid =cpu_to_logical_apicid (cpu);while (cpus_found < num_bits_set) { -if (cpu_isset (cpu, cpumask)) { +if (cpu_isset (cpu, *cpumask)) {int new_apicid =cpu_to_logical_apicid (cpu);if (apicid_cluster (apicid) !=apicid_cluster (new_apicid)){
include/asm-x86/summit/ipi.h +5 -8
@@ -1,25 +1,22 @@#ifndef __ASM_SUMMIT_IPI_H#define __ASM_SUMMIT_IPI_H -void send_IPI_mask_sequence (cpumask_t mask,int vector); +void send_IPI_mask_sequence (const cpumask_t *mask,int vector); +void send_IPI_mask_allbutself (const cpumask_t *mask,int vector); -static inline void send_IPI_mask (cpumask_t mask,int vector) +static inline void send_IPI_mask (const cpumask_t *mask,int vector) {send_IPI_mask_sequence (mask, vector); }static inline void send_IPI_allbutself (int vector) { -cpumask_t mask = cpu_online_map; -cpu_clear (smp_processor_id (), mask); - -if (!cpus_empty (mask)) -send_IPI_mask (mask, vector); +send_IPI_mask_allbutself (&cpu_online_map, vector); }static inline void send_IPI_all (int vector) { -send_IPI_mask (cpu_online_map, vector); +send_IPI_mask (&cpu_online_map, vector); }#endif /* __ASM_SUMMIT_IPI_H */
PATCH 3/3 sched: Optimize cpumask temp stack usage in kernel/sched.c
In reply to Mike Travis * Make the following changes to kernel/sched.c functions:
- use node_to_cpumask_ptr in place of node_to_cpumask
* Remove special code for SCHED_CPUMASK_ALLOC and use CPUMASK_ALLOC from linux/cpumask.h.
Applies to linux-2.6.tip/master.
--- kernel/sched.c | 81 +++++++++++++++++++++++---------------------------------- 1 file changed, 33 insertions(+), 48 deletions(-)
kernel/sched.c +33 -48
@@ -6110,8 +6110,9 @@ static void move_task_off_dead_cpu(int d do { /* On same node? */ - mask = node_to_cpumask(cpu_to_node(dead_cpu)); - cpus_and(mask, mask, p->cpus_allowed); + node_to_cpumask_ptr(pnodemask, cpu_to_node(dead_cpu)); + + cpus_and(mask, *pnodemask, p->cpus_allowed); dest_cpu = any_online_cpu(mask); /* On any allowed CPU? */
@@ -7117,9 +7118,9 @@ static int cpu_to_allnodes_group(int cpu struct sched_group **sg, cpumask_t *nodemask) { int group; +node_to_cpumask_ptr (pnodemask,cpu_to_node (cpu)); - *nodemask =node_to_cpumask (cpu_to_node (cpu)); -cpus_and (*nodemask, *nodemask, *cpu_map); +cpus_and (*nodemask, *pnodemask, *cpu_map); group =first_cpu (*nodemask);if (sg)
@@ -7169,9 +7170,9 @@ static void free_sched_groups(const cpumfor (i =0 ; i < nr_node_ids; i++) {struct sched_group *oldsg, *sg = sched_group_nodes[i]; +node_to_cpumask_ptr (pnodemask, i); - *nodemask =node_to_cpumask (i); -cpus_and (*nodemask, *nodemask, *cpu_map); +cpus_and (*nodemask, *pnodemask, *cpu_map);if (cpus_empty (*nodemask))continue ;
@@ -7294,19 +7295,6 @@ struct allmasks {#endif }; -#if NR_CPUS >128 -#define SCHED_CPUMASK_ALLOC 1 -#define SCHED_CPUMASK_FREE (v) kfree(v) -#define SCHED_CPUMASK_DECLARE (v) struct allmasks *v -#else -#define SCHED_CPUMASK_ALLOC 0 -#define SCHED_CPUMASK_FREE (v) -#define SCHED_CPUMASK_DECLARE (v) struct allmasks _v, *v = &_v -#endif - -#define SCHED_CPUMASK_VAR (v, a) cpumask_t *v = (cpumask_t *) \ - ((unsigned long)(a) + offsetof(struct allmasks, v)) -static int default_relax_domain_level =-1 ;static int __init setup_relax_domain_level (char *str)
@@ -7351,8 +7339,9 @@ static int __build_sched_domains(const c {int i;struct root_domain *rd; -SCHED_CPUMASK_DECLARE (allmasks); -cpumask_t *tmpmask; +CPUMASK_ALLOC (allmasks); +CPUMASK_PTR (tmpmask, allmasks); +#ifdef CONFIG_NUMA struct sched_group **sched_group_nodes =NULL ;int sd_allnodes =0 ;
@@ -7364,6 +7353,7 @@ static int __build_sched_domains(const c GFP_KERNEL);if (!sched_group_nodes) {printk (KERN_WARNING "Can not alloc sched group node list\n" ); +CPUMASK_FREE (allmasks);return -ENOMEM ; }#endif
@@ -7374,13 +7364,11 @@ static int __build_sched_domains(const c#ifdef CONFIG_NUMA kfree (sched_group_nodes);#endif +CPUMASK_FREE (allmasks);return -ENOMEM ; } -#if SCHED_CPUMASK_ALLOC -/* get space for all scratch cpumask variables */ - allmasks =kmalloc (sizeof (*allmasks),GFP_KERNEL ); -if (!allmasks) { +if (allmasks ==NULL ) {printk (KERN_WARNING "Cannot alloc cpumask array\n" );kfree (rd);#ifdef CONFIG_NUMA
@@ -7388,9 +7376,6 @@ static int __build_sched_domains(const c#endif return -ENOMEM ; } -#endif - tmpmask = (cpumask_t *)allmasks; -#ifdef CONFIG_NUMA sched_group_nodes_bycpu[first_cpu (*cpu_map)] = sched_group_nodes;
@@ -7401,10 +7386,10 @@ static int __build_sched_domains(const c*/ for_each_cpu_mask_nr (i, *cpu_map) {struct sched_domain *sd =NULL , *p; -SCHED_CPUMASK_VAR (nodemask, allmasks); +CPUMASK_PTR (nodemask, allmasks); +node_to_cpumask_ptr (pnodemask,cpu_to_node (i)); - *nodemask =node_to_cpumask (cpu_to_node (i)); -cpus_and (*nodemask, *nodemask, *cpu_map); +cpus_and (*nodemask, *pnodemask, *cpu_map);#ifdef CONFIG_NUMA if (cpus_weight(*cpu_map) >
@@ -7467,8 +7452,8 @@ static int __build_sched_domains(const c#ifdef CONFIG_SCHED_SMT /* Set up CPU (sibling) groups */ for_each_cpu_mask_nr (i, *cpu_map) { -SCHED_CPUMASK_VAR (this_sibling_map, allmasks); -SCHED_CPUMASK_VAR (send_covered, allmasks); +CPUMASK_PTR (this_sibling_map, allmasks); +CPUMASK_PTR (send_covered, allmasks); *this_sibling_map =per_cpu (cpu_sibling_map, i);cpus_and (*this_sibling_map, *this_sibling_map, *cpu_map);
@@ -7484,8 +7469,8 @@ static int __build_sched_domains(const c#ifdef CONFIG_SCHED_MC /* Set up multi-core groups */ for_each_cpu_mask_nr (i, *cpu_map) { -SCHED_CPUMASK_VAR (this_core_map, allmasks); -SCHED_CPUMASK_VAR (send_covered, allmasks); +CPUMASK_PTR (this_core_map, allmasks); +CPUMASK_PTR (send_covered, allmasks); *this_core_map =cpu_coregroup_map (i);cpus_and (*this_core_map, *this_core_map, *cpu_map);
@@ -7500,11 +7485,11 @@ static int __build_sched_domains(const c/* Set up physical groups */ for (i =0 ; i < nr_node_ids; i++) { -SCHED_CPUMASK_VAR (nodemask, allmasks); -SCHED_CPUMASK_VAR (send_covered, allmasks); +CPUMASK_PTR (nodemask, allmasks); +CPUMASK_PTR (send_covered, allmasks); +node_to_cpumask_ptr (pnodemask, i); - *nodemask =node_to_cpumask (i); -cpus_and (*nodemask, *nodemask, *cpu_map); +cpus_and (*nodemask, *pnodemask, *cpu_map);if (cpus_empty (*nodemask))continue ;
@@ -7516,7 +7501,7 @@ static int __build_sched_domains(const c#ifdef CONFIG_NUMA /* Set up node groups */ if (sd_allnodes) { -SCHED_CPUMASK_VAR (send_covered, allmasks); +CPUMASK_PTR (send_covered, allmasks); init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
@@ -7526,15 +7511,15 @@ static int __build_sched_domains(const cfor (i =0 ; i < nr_node_ids; i++) {/* Set up node groups */ struct sched_group *sg, *prev; -SCHED_CPUMASK_VAR (nodemask, allmasks); -SCHED_CPUMASK_VAR (domainspan, allmasks); -SCHED_CPUMASK_VAR (covered, allmasks); +CPUMASK_PTR (nodemask, allmasks); +CPUMASK_PTR (domainspan, allmasks); +CPUMASK_PTR (covered, allmasks); +node_to_cpumask_ptr (pnodemask, i);int j; - *nodemask =node_to_cpumask (i);cpus_clear (*covered); -cpus_and (*nodemask, *nodemask, *cpu_map); +cpus_and (*nodemask, *pnodemask, *cpu_map);if (cpus_empty (*nodemask)) { sched_group_nodes[i] =NULL ;continue ;
@@ -7563,7 +7548,7 @@ static int __build_sched_domains(const c prev = sg;for (j =0 ; j < nr_node_ids; j++) { -SCHED_CPUMASK_VAR (notcovered, allmasks); +CPUMASK_PTR (notcovered, allmasks);int n = (i + j) % nr_node_ids;node_to_cpumask_ptr (pnodemask, n);
@@ -7642,13 +7627,13 @@ static int __build_sched_domains(const ccpu_attach_domain (sd, rd, i); } -SCHED_CPUMASK_FREE ((void *)allmasks); +CPUMASK_FREE (allmasks);return 0 ;#ifdef CONFIG_NUMA error:free_sched_groups (cpu_map, tmpmask); -SCHED_CPUMASK_FREE ((void *)allmasks); +CPUMASK_FREE (allmasks);return -ENOMEM ;#endif }