Skip to contentSkip to conversation

cpu-2.5.63-1

PATCH William Lee Irwin III 2 participants 4 messages Feb 25, 2003 – Feb 26, 2003

Subsystems (8)
lore
Share & tools
lore
#0 lore raw

Also available from:
ftp://ftp.kernel.org/pub/linux/kernel/people/wli/cpu/

-- wli

Introduce a bitmap ADT and use it for cpu bitmasks on i386.

 arch/i386/kernel/apic.c                   |    3 
 arch/i386/kernel/cpu/proc.c               |    2 
 arch/i386/kernel/io_apic.c                |   45 +++++----
 arch/i386/kernel/irq.c                    |   39 +++++--
 arch/i386/kernel/ldt.c                    |   12 +-
 arch/i386/kernel/mpparse.c                |    6 -
 arch/i386/kernel/reboot.c                 |    2 
 arch/i386/kernel/smp.c                    |   80 +++++++++-------
 arch/i386/kernel/smpboot.c                |   70 +++++++-------
 drivers/base/node.c                       |    8 +
 include/asm-i386/highmem.h                |    6 +
 include/asm-i386/mach-default/mach_apic.h |   28 +++--
 include/asm-i386/mach-default/mach_ipi.h  |    4 
 include/asm-i386/mach-numaq/mach_apic.h   |   19 ++-
 include/asm-i386/mach-numaq/mach_ipi.h    |    9 +
 include/asm-i386/mmu_context.h            |    9 -
 include/asm-i386/mpspec.h                 |    2 
 include/asm-i386/numaq.h                  |    2 
 include/asm-i386/smp.h                    |   29 +----
 include/asm-i386/topology.h               |    9 +
 include/linux/bitmap.h                    |  148 ++++++++++++++++++++++++++++++
 include/linux/cpumask.h                   |   90 ++++++++++++++++++
 include/linux/init_task.h                 |    2 
 include/linux/irq.h                       |    2 
 include/linux/node.h                      |    2 
 include/linux/rcupdate.h                  |    5 -
 include/linux/sched.h                     |    7 -
 include/linux/types.h                     |    8 -
 kernel/fork.c                             |    2 
 kernel/module.c                           |    9 +
 kernel/rcupdate.c                         |   12 +-
 kernel/sched.c                            |   64 +++++++-----
 kernel/softirq.c                          |    7 -
 kernel/workqueue.c                        |    4 
 34 files changed, 527 insertions(+), 219 deletions(-)
arch/i386/kernel/apic.c +2 -1
diff -urpN linux-2.5.63/arch/i386/kernel/apic.c cpu-2.5.63-1/arch/i386/kernel/apic.c
@@ -1154,7 +1154,8 @@ int __init APIC_init_uniprocessor (void)
 
 	connect_bsp_APIC();
 
-	phys_cpu_present_map = 1;
+	cpus_clear(phys_cpu_present_map);
+	cpu_set(0, phys_cpu_present_map);
 	apic_write_around(APIC_ID, boot_cpu_physical_apicid);
 
 	apic_pm_init2();
arch/i386/kernel/cpu/proc.c +1 -1
diff -urpN linux-2.5.63/arch/i386/kernel/cpu/proc.c cpu-2.5.63-1/arch/i386/kernel/cpu/proc.c
@@ -47,7 +47,7 @@ static int show_cpuinfo(struct seq_file 
 	int fpu_exception;
 
 #ifdef CONFIG_SMP
-	if (!(cpu_online_map & (1<<n)))
+	if (!cpu_isset(n, cpu_online_map))
 		return 0;
 #endif
 	seq_printf(m, "processor\t: %d\n"
arch/i386/kernel/io_apic.c +27 -18

128 lines not on this page · raw message

arch/i386/kernel/irq.c +26 -13
diff -urpN linux-2.5.63/arch/i386/kernel/irq.c cpu-2.5.63-1/arch/i386/kernel/irq.c
@@ -44,7 +44,7 @@
 #include <asm/desc.h>
 #include <asm/irq.h>
 
-
+#include "mach_apic.h"
 
 /*
  * Linux has a controller-independent x86 interrupt architecture.
@@ -793,13 +793,13 @@ int setup_irq(unsigned int irq, struct i
 static struct proc_dir_entry * root_irq_dir;
 static struct proc_dir_entry * irq_dir [NR_IRQS];
 
-#define HEX_DIGITS 8
+#define HEX_DIGITS (2*sizeof(cpumask_t))
 
 static unsigned int parse_hex_value (const char *buffer,
-		unsigned long count, unsigned long *ret)
+		unsigned long count, cpumask_t *ret)
 {
 	unsigned char hexnum [HEX_DIGITS];
-	unsigned long value;
+	cpumask_t value = CPU_MASK_NONE;
 	int i;
 
 	if (!count)
@@ -813,10 +813,10 @@ static unsigned int parse_hex_value (con
 	 * Parse the first 8 characters as a hex string, any non-hex char
 	 * is end-of-string. '00e1', 'e1', '00E1', 'E1' are all the same.
 	 */
-	value = 0;
 
 	for (i = 0; i < count; i++) {
 		unsigned int c = hexnum[i];
+		int k;
 
 		switch (c) {
 			case '0' ... '9': c -= '0'; break;
@@ -825,7 +825,10 @@ static unsigned int parse_hex_value (con
 		default:
 			goto out;
 		}
-		value = (value << 4) | c;
+		bitmap_shift_left(&cpus_coerce(value), &cpus_coerce(value), 4, NR_CPUS);
+		for (k = 0; k < 4; ++k)
+			if (test_bit(k, (unsigned long *)&c))
+				cpu_set(k, value);
 	}
 out:
 	*ret = value;
@@ -836,20 +839,28 @@ out:
 
 static struct proc_dir_entry * smp_affinity_entry [NR_IRQS];
 
-unsigned long irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = ~0UL };
+cpumask_t irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = CPU_MASK_ALL };
 static int irq_affinity_read_proc (char *page, char **start, off_t off,
 			int count, int *eof, void *data)
 {
+	int k, len;
 	if (count < HEX_DIGITS+1)
 		return -EINVAL;
-	return sprintf (page, "%08lx\n", irq_affinity[(long)data]);
+
+	len = 0;
+	for (k = 0; k < CPU_ARRAY_SIZE; ++k) {
+		int j = sprintf (page, "%08lx\n", irq_affinity[(long)data].mask[k]);
+		len  += j;
+		page += j;
+	}
+	return len;
 }
 
 static int irq_affinity_write_proc (struct file *file, const char *buffer,
 					unsigned long count, void *data)
 {
 	int irq = (long) data, full_count = count, err;
-	unsigned long new_value;
+	cpumask_t new_value, tmp;
 
 	if (!irq_desc[irq].handler->set_affinity)
 		return -EIO;
@@ -861,11 +872,12 @@ static int irq_affinity_write_proc (stru
 	 * way to make the system unusable accidentally :-) At least
 	 * one online CPU still has to be targeted.
 	 */
-	if (!(new_value & cpu_online_map))
+	cpus_and(tmp, new_value, cpu_online_map);
+	if (cpus_empty(tmp))
 		return -EINVAL;
 
 	irq_affinity[irq] = new_value;
-	irq_desc[irq].handler->set_affinity(irq, new_value);
+	irq_desc[irq].handler->set_affinity(irq, cpu_to_logical_apicid(first_cpu(new_value)));
 
 	return full_count;
 }
@@ -884,8 +896,9 @@ static int prof_cpu_mask_read_proc (char
 static int prof_cpu_mask_write_proc (struct file *file, const char *buffer,
 					unsigned long count, void *data)
 {
-	unsigned long *mask = (unsigned long *) data, full_count = count, err;
-	unsigned long new_value;
+	cpumask_t *mask = (cpumask_t *)data;
+	unsigned long full_count = count, err;
+	cpumask_t new_value;
 
 	err = parse_hex_value(buffer, count, &new_value);
 	if (err)
arch/i386/kernel/ldt.c +8 -4
diff -urpN linux-2.5.63/arch/i386/kernel/ldt.c cpu-2.5.63-1/arch/i386/kernel/ldt.c
@@ -57,10 +57,14 @@ static int alloc_ldt(mm_context_t *pc, i
 	if (reload) {
 		load_LDT(pc);
 #ifdef CONFIG_SMP
-		preempt_disable();
-		if (current->mm->cpu_vm_mask != (1 << smp_processor_id()))
-			smp_call_function(flush_ldt, 0, 1, 1);
-		preempt_enable();
+		{
+			cpumask_t tmp = CPU_MASK_NONE;
+			preempt_disable();
+			cpu_set(smp_processor_id(), tmp);
+			if (!cpus_equal(current->mm->cpu_vm_mask, tmp))
+				smp_call_function(flush_ldt, 0, 1, 1);
+			preempt_enable();
+		}
 #endif
 	}
 	if (oldsize) {
arch/i386/kernel/mpparse.c +4 -2
diff -urpN linux-2.5.63/arch/i386/kernel/mpparse.c cpu-2.5.63-1/arch/i386/kernel/mpparse.c
@@ -70,7 +70,7 @@ unsigned int boot_cpu_logical_apicid = -
 static unsigned int __initdata num_processors;
 
 /* Bitmask of physically existing CPUs */
-unsigned long phys_cpu_present_map;
+cpumask_t phys_cpu_present_map;
 
 int x86_summit = 0;
 u8 bios_cpu_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
@@ -106,6 +106,7 @@ static struct mpc_config_translation *tr
 void __init MP_processor_info (struct mpc_config_processor *m)
 {
  	int ver, apicid;
+	cpumask_t tmp;
  	
 	if (!(m->mpc_cpuflag & CPU_ENABLED))
 		return;
@@ -176,7 +177,8 @@ void __init MP_processor_info (struct mp
 	}
 	ver = m->mpc_apicver;
 
-	phys_cpu_present_map |= apicid_to_cpu_present(apicid);
+	tmp = apicid_to_cpu_present(apicid);
+	cpus_or(phys_cpu_present_map, phys_cpu_present_map, tmp);
 	
 	/*
 	 * Validate version
arch/i386/kernel/reboot.c +1 -1
diff -urpN linux-2.5.63/arch/i386/kernel/reboot.c cpu-2.5.63-1/arch/i386/kernel/reboot.c
@@ -234,7 +234,7 @@ void machine_restart(char * __unused)
 		   if its not, default to the BSP */
 		if ((reboot_cpu == -1) ||  
 		      (reboot_cpu > (NR_CPUS -1))  || 
-		      !(phys_cpu_present_map & (1<<cpuid))) 
+		      !(cpu_isset(cpuid, phys_cpu_present_map)))
 			reboot_cpu = boot_cpu_physical_apicid;
 
 		reboot_smp = 0;  /* use this as a flag to only go through this once*/
arch/i386/kernel/smp.c +47 -33

201 lines not on this page · raw message

arch/i386/kernel/smpboot.c +36 -34

225 lines not on this page · raw message

drivers/base/node.c +7 -1
diff -urpN linux-2.5.63/drivers/base/node.c cpu-2.5.63-1/drivers/base/node.c
@@ -31,7 +31,13 @@ struct device_driver node_driver = {
 static ssize_t node_read_cpumap(struct device * dev, char * buf)
 {
 	struct node *node_dev = to_node(to_root(dev));
-        return sprintf(buf,"%lx\n",node_dev->cpumap);
+	int k, len = 0;
+	for (k = 0; k < CPU_ARRAY_SIZE; ++k) {
+        	int j = sprintf(buf,"%lx\n",node_dev->cpumap.mask[k]);
+		len += j;
+		buf += j;
+	}
+	return len;
 }
 static DEVICE_ATTR(cpumap,S_IRUGO,node_read_cpumap,NULL);
 
include/asm-i386/highmem.h +6 -0
diff -urpN linux-2.5.63/include/asm-i386/highmem.h cpu-2.5.63-1/include/asm-i386/highmem.h
@@ -22,6 +22,7 @@
 
 #include <linux/config.h>
 #include <linux/interrupt.h>
+#include <linux/threads.h>
 #include <asm/kmap_types.h>
 #include <asm/tlbflush.h>
 
@@ -39,7 +40,12 @@ extern void kmap_init(void);
  * easily, subsequent pte tables have to be allocated in one physical
  * chunk of RAM.
  */
+#if NR_CPUS <= 32
 #define PKMAP_BASE (0xff800000UL)
+#else
+#define PKMAP_BASE (0xff600000UL)
+#endif
+
 #ifdef CONFIG_X86_PAE
 #define LAST_PKMAP 512
 #else
include/asm-i386/mach-default/mach_apic.h +19 -9
diff -urpN linux-2.5.63/include/asm-i386/mach-default/mach_apic.h cpu-2.5.63-1/include/asm-i386/mach-default/mach_apic.h
@@ -3,8 +3,12 @@
 
 #define APIC_DFR_VALUE	(APIC_DFR_FLAT)
 
+/*
+ * Flat mode can't support large numbers of cpus.
+ * The first word of cpu_online_map should cover us.
+ */
 #ifdef CONFIG_SMP
- #define TARGET_CPUS (cpu_online_map)
+ #define TARGET_CPUS (cpu_online_map.mask[0])
 #else
  #define TARGET_CPUS 0x01
 #endif
@@ -17,12 +21,12 @@
 
 #define APIC_BROADCAST_ID      0x0F
 #define check_apicid_used(bitmap, apicid) (bitmap & (1 << apicid))
-#define check_apicid_present(bit) (phys_cpu_present_map & (1 << bit))
+#define check_apicid_present(bit) cpu_isset(bit, phys_cpu_present_map)
 
 static inline int apic_id_registered(void)
 {
-	return (test_bit(GET_APIC_ID(apic_read(APIC_ID)), 
-						&phys_cpu_present_map));
+	return cpu_isset(GET_APIC_ID(apic_read(APIC_ID)),
+				phys_cpu_present_map);
 }
 
 /*
@@ -42,9 +46,12 @@ static inline void init_apic_ldr(void)
 	apic_write_around(APIC_LDR, val);
 }
 
-static inline ulong ioapic_phys_id_map(ulong phys_map)
+/*
+ * Only small machines use this APIC mode.
+ */
+static inline unsigned long ioapic_phys_id_map(cpumask_t phys_map)
 {
-	return phys_map;
+	return cpus_coerce(phys_map);
 }
 
 static inline void clustered_apic_check(void)
@@ -74,9 +81,12 @@ static inline int cpu_present_to_apicid(
 	return  mps_cpu;
 }
 
-static inline unsigned long apicid_to_cpu_present(int phys_apicid)
+static inline cpumask_t apicid_to_cpu_present(int phys_apicid)
 {
-	return (1ul << phys_apicid);
+	cpumask_t mask;
+	cpus_clear(mask);
+	cpu_set(phys_apicid, mask);
+	return mask;
 }
 
 static inline int mpc_apic_id(struct mpc_config_processor *m, int quad)
@@ -95,7 +105,7 @@ static inline void setup_portio_remap(vo
 
 static inline int check_phys_apicid_present(int boot_cpu_physical_apicid)
 {
-	return test_bit(boot_cpu_physical_apicid, &phys_cpu_present_map);
+	return cpu_isset(boot_cpu_physical_apicid, phys_cpu_present_map);
 }
 
 #endif /* __ASM_MACH_APIC_H */
include/asm-i386/mach-default/mach_ipi.h +2 -2
diff -urpN linux-2.5.63/include/asm-i386/mach-default/mach_ipi.h cpu-2.5.63-1/include/asm-i386/mach-default/mach_ipi.h
@@ -1,10 +1,10 @@
 #ifndef __ASM_MACH_IPI_H
 #define __ASM_MACH_IPI_H
 
-static inline void send_IPI_mask_bitmask(int mask, int vector);
+static inline void send_IPI_mask_bitmask(cpumask_t cpumask, int vector);
 static inline void __send_IPI_shortcut(unsigned int shortcut, int vector);
 
-static inline void send_IPI_mask(int mask, int vector)
+static inline void send_IPI_mask(cpumask_t mask, int vector)
 {
 	send_IPI_mask_bitmask(mask, vector);
 }
include/asm-i386/mach-numaq/mach_apic.h +12 -7
diff -urpN linux-2.5.63/include/asm-i386/mach-numaq/mach_apic.h cpu-2.5.63-1/include/asm-i386/mach-numaq/mach_apic.h
@@ -12,12 +12,12 @@
 #define INT_DEST_MODE 0     /* physical delivery on LOCAL quad */
  
 #define APIC_BROADCAST_ID      0x0F
-#define check_apicid_used(bitmap, apicid) ((bitmap) & (1 << (apicid)))
-#define check_apicid_present(bit) (phys_cpu_present_map & (1 << bit))
+#define check_apicid_used(bitmap, apicid) cpu_isset(apicid, bitmap)
+#define check_apicid_present(bit)	cpu_isset(bit, phys_cpu_present_map)
 
 static inline int apic_id_registered(void)
 {
-	return (1);
+	return 1;
 }
 
 static inline void init_apic_ldr(void)
@@ -40,10 +40,10 @@ static inline int multi_timer_check(int 
 	return (apic != 0 && irq == 0);
 }
 
-static inline ulong ioapic_phys_id_map(ulong phys_map)
+static inline unsigned long ioapic_phys_id_map(cpumask_t phys_map)
 {
 	/* We don't have a good way to do this yet - hack */
-	return 0xf;
+	return 0xFUL;
 }
 
 /* Mapping from cpu number to logical apicid */
@@ -68,9 +68,14 @@ static inline int apicid_to_node(int log
 	return (logical_apicid >> 4);
 }
 
-static inline unsigned long apicid_to_cpu_present(int logical_apicid)
+static inline cpumask_t apicid_to_cpu_present(int logical_apicid)
 {
-	return ( (logical_apicid&0xf) << (4*apicid_to_node(logical_apicid)) );
+	cpumask_t mask = CPU_MASK_NONE;
+	int node = apicid_to_node(logical_apicid);
+	int cpu = __ffs(logical_apicid & 0xf);
+
+	cpu_set(cpu + 4*node, mask);
+	return mask;
 }
 
 static inline int mpc_apic_id(struct mpc_config_processor *m, int quad)
include/asm-i386/mach-numaq/mach_ipi.h +5 -4
diff -urpN linux-2.5.63/include/asm-i386/mach-numaq/mach_ipi.h cpu-2.5.63-1/include/asm-i386/mach-numaq/mach_ipi.h
@@ -1,18 +1,19 @@
 #ifndef __ASM_MACH_IPI_H
 #define __ASM_MACH_IPI_H
 
-static inline void send_IPI_mask_sequence(int mask, int vector);
+static inline void send_IPI_mask_sequence(cpumask_t, int vector);
 
-static inline void send_IPI_mask(int mask, int vector)
+static inline void send_IPI_mask(cpumask_t mask, int vector)
 {
 	send_IPI_mask_sequence(mask, vector);
 }
 
 static inline void send_IPI_allbutself(int vector)
 {
-	unsigned long mask = cpu_online_map & ~(1 << smp_processor_id());
+	cpumask_t mask = cpu_online_map;
+	cpu_clear(smp_processor_id(), mask);
 
-	if (mask)
+	if (!cpus_empty(mask))
 		send_IPI_mask(mask, vector);
 }
 
include/asm-i386/mmu_context.h +4 -5
diff -urpN linux-2.5.63/include/asm-i386/mmu_context.h cpu-2.5.63-1/include/asm-i386/mmu_context.h
@@ -26,12 +26,12 @@ static inline void switch_mm(struct mm_s
 {
 	if (likely(prev != next)) {
 		/* stop flush ipis for the previous mm */
-		clear_bit(cpu, &prev->cpu_vm_mask);
+		cpu_clear(cpu, prev->cpu_vm_mask);
 #ifdef CONFIG_SMP
 		cpu_tlbstate[cpu].state = TLBSTATE_OK;
 		cpu_tlbstate[cpu].active_mm = next;
 #endif
-		set_bit(cpu, &next->cpu_vm_mask);
+		cpu_set(cpu, next->cpu_vm_mask);
 
 		/* Re-load page tables */
 		load_cr3(next->pgd);
@@ -45,9 +45,8 @@ static inline void switch_mm(struct mm_s
 #ifdef CONFIG_SMP
 	else {
 		cpu_tlbstate[cpu].state = TLBSTATE_OK;
-		if (cpu_tlbstate[cpu].active_mm != next)
-			BUG();
-		if (!test_and_set_bit(cpu, &next->cpu_vm_mask)) {
+		BUG_ON(cpu_tlbstate[cpu].active_mm != next);
+		if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
 			/* We were in lazy tlb mode and leave_mm disabled 
 			 * tlb flush IPI delivery. We must reload %cr3.
 			 */
include/asm-i386/mpspec.h +1 -1
diff -urpN linux-2.5.63/include/asm-i386/mpspec.h cpu-2.5.63-1/include/asm-i386/mpspec.h
@@ -206,7 +206,7 @@ extern int quad_local_to_mp_bus_id [NR_C
 extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES];
 
 extern unsigned int boot_cpu_physical_apicid;
-extern unsigned long phys_cpu_present_map;
+extern cpumask_t phys_cpu_present_map;
 extern int smp_found_config;
 extern void find_smp_config (void);
 extern void get_smp_config (void);
include/asm-i386/numaq.h +1 -1
diff -urpN linux-2.5.63/include/asm-i386/numaq.h cpu-2.5.63-1/include/asm-i386/numaq.h
@@ -38,7 +38,7 @@
 
 #define pfn_to_pgdat(pfn) NODE_DATA(pfn_to_nid(pfn))
 #define PHYSADDR_TO_NID(pa) pfn_to_nid(pa >> PAGE_SHIFT)
-#define MAX_NUMNODES		8
+#define MAX_NUMNODES		16
 extern int pfn_to_nid(unsigned long);
 extern void get_memcfg_numaq(void);
 #define get_memcfg_numa() get_memcfg_numaq()
include/asm-i386/smp.h +6 -23
diff -urpN linux-2.5.63/include/asm-i386/smp.h cpu-2.5.63-1/include/asm-i386/smp.h
@@ -8,6 +8,7 @@
 #include <linux/config.h>
 #include <linux/kernel.h>
 #include <linux/threads.h>
+#include <linux/cpumask.h>
 #endif
 
 #ifdef CONFIG_X86_LOCAL_APIC
@@ -31,8 +32,8 @@
  */
  
 extern void smp_alloc_memory(void);
-extern unsigned long phys_cpu_present_map;
-extern unsigned long cpu_online_map;
+extern cpumask_t phys_cpu_present_map;
+extern cpumask_t cpu_online_map;
 extern volatile unsigned long smp_invalidate_needed;
 extern int pic_mode;
 extern int smp_num_siblings;
@@ -55,37 +56,19 @@ extern void zap_low_mappings (void);
  */
 #define smp_processor_id() (current_thread_info()->cpu)
 
-extern volatile unsigned long cpu_callout_map;
+extern volatile cpumask_t cpu_callout_map;
 
-#define cpu_possible(cpu) (cpu_callout_map & (1<<(cpu)))
-#define cpu_online(cpu) (cpu_online_map & (1<<(cpu)))
-
-#define for_each_cpu(cpu, mask) \
-	for(mask = cpu_online_map; \
-	    cpu = __ffs(mask), mask != 0; \
-	    mask &= ~(1<<cpu))
-
-extern inline unsigned int num_online_cpus(void)
-{
-	return hweight32(cpu_online_map);
-}
+#define cpu_possible(cpu) test_bit(cpu, cpu_callout_map.mask)
 
 /* We don't mark CPUs online until __cpu_up(), so we need another measure */
 static inline int num_booting_cpus(void)
 {
-	return hweight32(cpu_callout_map);
+	return bitmap_weight((unsigned long *)cpu_callout_map.mask, NR_CPUS);
 }
 
 extern void map_cpu_to_logical_apicid(void);
 extern void unmap_cpu_to_logical_apicid(int cpu);
 
-extern inline int any_online_cpu(unsigned int mask)
-{
-	if (mask & cpu_online_map)
-		return __ffs(mask & cpu_online_map);
-
-	return -1;
-}
 #ifdef CONFIG_X86_LOCAL_APIC
 static __inline int hard_smp_processor_id(void)
 {
include/asm-i386/topology.h +6 -3
diff -urpN linux-2.5.63/include/asm-i386/topology.h cpu-2.5.63-1/include/asm-i386/topology.h
@@ -29,8 +29,10 @@
 
 #ifdef CONFIG_NUMA
 
+#include <linux/cpumask.h>
+
 /* Mappings between logical cpu number and node number */
-extern volatile unsigned long node_2_cpu_mask[];
+extern volatile cpumask_t node_2_cpu_mask[];
 extern volatile int cpu_2_node[];
 
 /* Returns the number of the node containing CPU 'cpu' */
@@ -47,7 +49,7 @@ static inline int cpu_to_node(int cpu)
 #define parent_node(node) (node)
 
 /* Returns a bitmask of CPUs on Node 'node'. */
-static inline unsigned long node_to_cpumask(int node)
+static inline cpumask_t node_to_cpumask(int node)
 {
 	return node_2_cpu_mask[node];
 }
@@ -55,7 +57,8 @@ static inline unsigned long node_to_cpum
 /* Returns the number of the first CPU on Node 'node'. */
 static inline int node_to_first_cpu(int node)
 { 
-	return __ffs(node_to_cpumask(node));
+	cpumask_t mask = node_to_cpumask(node);
+	return first_cpu(mask);
 }
 
 /* Returns the number of the first MemBlk on Node 'node' */
include/linux/bitmap.h +148 -0

148 lines not on this page · raw message

include/linux/cpumask.h +90 -0

90 lines not on this page · raw message

include/linux/init_task.h +1 -1
diff -urpN linux-2.5.63/include/linux/init_task.h cpu-2.5.63-1/include/linux/init_task.h
@@ -68,7 +68,7 @@
 	.prio		= MAX_PRIO-20,					\
 	.static_prio	= MAX_PRIO-20,					\
 	.policy		= SCHED_NORMAL,					\
-	.cpus_allowed	= ~0UL,						\
+	.cpus_allowed	= CPU_MASK_ALL,					\
 	.mm		= NULL,						\
 	.active_mm	= &init_mm,					\
 	.run_list	= LIST_HEAD_INIT(tsk.run_list),			\
include/linux/irq.h +1 -1
diff -urpN linux-2.5.63/include/linux/irq.h cpu-2.5.63-1/include/linux/irq.h
@@ -44,7 +44,7 @@ struct hw_interrupt_type {
 	void (*disable)(unsigned int irq);
 	void (*ack)(unsigned int irq);
 	void (*end)(unsigned int irq);
-	void (*set_affinity)(unsigned int irq, unsigned long mask);
+	void (*set_affinity)(unsigned int irq, unsigned long dest);
 };
 
 typedef struct hw_interrupt_type  hw_irq_controller;
include/linux/node.h +1 -1
diff -urpN linux-2.5.63/include/linux/node.h cpu-2.5.63-1/include/linux/node.h
@@ -22,7 +22,7 @@
 #include <linux/device.h>
 
 struct node {
-	unsigned long cpumap;	/* Bitmap of CPUs on the Node */
+	cpumask_t cpumap;	/* Bitmap of CPUs on the Node */
 	struct sys_root sysroot;
 };
 
include/linux/rcupdate.h +3 -2
diff -urpN linux-2.5.63/include/linux/rcupdate.h cpu-2.5.63-1/include/linux/rcupdate.h
@@ -40,6 +40,7 @@
 #include <linux/spinlock.h>
 #include <linux/threads.h>
 #include <linux/percpu.h>
+#include <linux/cpumask.h>
 
 /**
  * struct rcu_head - callback structure for use with RCU
@@ -67,7 +68,7 @@ struct rcu_ctrlblk {
 	spinlock_t	mutex;		/* Guard this struct                  */
 	long		curbatch;	/* Current batch number.	      */
 	long		maxbatch;	/* Max requested batch number.        */
-	unsigned long	rcu_cpu_mask; 	/* CPUs that need to switch in order  */
+	cpumask_t	rcu_cpu_mask; 	/* CPUs that need to switch in order  */
 					/* for current batch to proceed.      */
 };
 
@@ -114,7 +115,7 @@ static inline int rcu_pending(int cpu) 
 	     rcu_batch_before(RCU_batch(cpu), rcu_ctrlblk.curbatch)) ||
 	    (list_empty(&RCU_curlist(cpu)) &&
 			 !list_empty(&RCU_nxtlist(cpu))) ||
-	    test_bit(cpu, &rcu_ctrlblk.rcu_cpu_mask))
+	    cpu_isset(cpu, rcu_ctrlblk.rcu_cpu_mask))
 		return 1;
 	else
 		return 0;
include/linux/sched.h +4 -3
diff -urpN linux-2.5.63/include/linux/sched.h cpu-2.5.63-1/include/linux/sched.h
@@ -12,6 +12,7 @@
 #include <linux/jiffies.h>
 #include <linux/rbtree.h>
 #include <linux/thread_info.h>
+#include <linux/cpumask.h>
 
 #include <asm/system.h>
 #include <asm/semaphore.h>
@@ -197,7 +198,7 @@ struct mm_struct {
 	unsigned long arg_start, arg_end, env_start, env_end;
 	unsigned long rss, total_vm, locked_vm;
 	unsigned long def_flags;
-	unsigned long cpu_vm_mask;
+	cpumask_t cpu_vm_mask;
 	unsigned long swap_address;
 
 	unsigned dumpable:1;
@@ -331,7 +332,7 @@ struct task_struct {
 	unsigned long sleep_timestamp;
 
 	unsigned long policy;
-	unsigned long cpus_allowed;
+	cpumask_t cpus_allowed;
 	unsigned int time_slice, first_time_slice;
 
 	struct list_head tasks;
@@ -467,7 +468,7 @@ do { if (atomic_dec_and_test(&(tsk)->usa
 #define PF_KSWAPD	0x00040000	/* I am kswapd */
 
 #if CONFIG_SMP
-extern void set_cpus_allowed(task_t *p, unsigned long new_mask);
+extern void set_cpus_allowed(task_t *p, cpumask_t new_mask);
 #else
 # define set_cpus_allowed(p, new_mask) do { } while (0)
 #endif
include/linux/types.h +1 -7
diff -urpN linux-2.5.63/include/linux/types.h cpu-2.5.63-1/include/linux/types.h
@@ -3,17 +3,11 @@
 
 #ifdef	__KERNEL__
 #include <linux/config.h>
-
-#define BITS_TO_LONGS(bits) \
-	(((bits)+BITS_PER_LONG-1)/BITS_PER_LONG)
-#define DECLARE_BITMAP(name,bits) \
-	unsigned long name[BITS_TO_LONGS(bits)]
-#define CLEAR_BITMAP(name,bits) \
-	memset(name, 0, BITS_TO_LONGS(bits)*sizeof(unsigned long))
 #endif
 
 #include <linux/posix_types.h>
 #include <asm/types.h>
+#include <linux/bitmap.h>
 
 #ifndef __KERNEL_STRICT_NAMES
 
kernel/fork.c +1 -1
diff -urpN linux-2.5.63/kernel/fork.c cpu-2.5.63-1/kernel/fork.c
@@ -251,7 +251,7 @@ static inline int dup_mmap(struct mm_str
 	mm->free_area_cache = TASK_UNMAPPED_BASE;
 	mm->map_count = 0;
 	mm->rss = 0;
-	mm->cpu_vm_mask = 0;
+	cpus_clear(mm->cpu_vm_mask);
 	pprev = &mm->mmap;
 
 	/*
kernel/module.c +6 -3
diff -urpN linux-2.5.63/kernel/module.c cpu-2.5.63-1/kernel/module.c
@@ -224,6 +224,7 @@ static int stopref(void *cpu)
 {
 	int irqs_disabled = 0;
 	int prepared = 0;
+	cpumask_t allowed_mask = CPU_MASK_NONE;
 
 	sprintf(current->comm, "kmodule%lu\n", (unsigned long)cpu);
 
@@ -232,7 +233,8 @@ static int stopref(void *cpu)
 	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 	setscheduler(current->pid, SCHED_FIFO, &param);
 #endif
-	set_cpus_allowed(current, 1UL << (unsigned long)cpu);
+	cpu_set((int)cpu, allowed_mask);
+	set_cpus_allowed(current, allowed_mask);
 
 	/* Ack: we are alive */
 	atomic_inc(&stopref_thread_ack);
@@ -285,7 +287,7 @@ static void stopref_set_state(enum stopr
 static int stop_refcounts(void)
 {
 	unsigned int i, cpu;
-	unsigned long old_allowed;
+	cpumask_t old_allowed, allowed_mask = CPU_MASK_NONE;
 	int ret = 0;
 
 	/* One thread per cpu.  We'll do our own. */
@@ -293,7 +295,8 @@ static int stop_refcounts(void)
 
 	/* FIXME: racy with set_cpus_allowed. */
 	old_allowed = current->cpus_allowed;
-	set_cpus_allowed(current, 1UL << (unsigned long)cpu);
+	cpu_set(cpu, allowed_mask);
+	set_cpus_allowed(current, allowed_mask);
 
 	atomic_set(&stopref_thread_ack, 0);
 	stopref_num_threads = 0;
kernel/rcupdate.c +6 -6
diff -urpN linux-2.5.63/kernel/rcupdate.c cpu-2.5.63-1/kernel/rcupdate.c
@@ -47,7 +47,7 @@
 /* Definition for rcupdate control block. */
 struct rcu_ctrlblk rcu_ctrlblk = 
 	{ .mutex = SPIN_LOCK_UNLOCKED, .curbatch = 1, 
-	  .maxbatch = 1, .rcu_cpu_mask = 0 };
+	  .maxbatch = 1, .rcu_cpu_mask = CPU_MASK_NONE };
 DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
 
 /* Fake initialization required by compiler */
@@ -106,7 +106,7 @@ static void rcu_start_batch(long newbatc
 		rcu_ctrlblk.maxbatch = newbatch;
 	}
 	if (rcu_batch_before(rcu_ctrlblk.maxbatch, rcu_ctrlblk.curbatch) ||
-	    (rcu_ctrlblk.rcu_cpu_mask != 0)) {
+	    !cpus_empty(rcu_ctrlblk.rcu_cpu_mask)) {
 		return;
 	}
 	rcu_ctrlblk.rcu_cpu_mask = cpu_online_map;
@@ -121,7 +121,7 @@ static void rcu_check_quiescent_state(vo
 {
 	int cpu = smp_processor_id();
 
-	if (!test_bit(cpu, &rcu_ctrlblk.rcu_cpu_mask)) {
+	if (!cpu_isset(cpu, rcu_ctrlblk.rcu_cpu_mask)) {
 		return;
 	}
 
@@ -139,13 +139,13 @@ static void rcu_check_quiescent_state(vo
 	}
 
 	spin_lock(&rcu_ctrlblk.mutex);
-	if (!test_bit(cpu, &rcu_ctrlblk.rcu_cpu_mask)) {
+	if (!cpu_isset(cpu, rcu_ctrlblk.rcu_cpu_mask)) {
 		spin_unlock(&rcu_ctrlblk.mutex);
 		return;
 	}
-	clear_bit(cpu, &rcu_ctrlblk.rcu_cpu_mask);
+	cpu_clear(cpu, rcu_ctrlblk.rcu_cpu_mask);
 	RCU_last_qsctr(cpu) = RCU_QSCTR_INVALID;
-	if (rcu_ctrlblk.rcu_cpu_mask != 0) {
+	if (!cpus_empty(rcu_ctrlblk.rcu_cpu_mask)) {
 		spin_unlock(&rcu_ctrlblk.mutex);
 		return;
 	}
kernel/sched.c +37 -27

197 lines not on this page · raw message

kernel/softirq.c +4 -3
diff -urpN linux-2.5.63/kernel/softirq.c cpu-2.5.63-1/kernel/softirq.c
@@ -299,15 +299,16 @@ void __init softirq_init()
 static int ksoftirqd(void * __bind_cpu)
 {
 	int cpu = (int) (long) __bind_cpu;
+	cpumask_t allowed_mask = CPU_MASK_NONE;
 
 	daemonize("ksoftirqd/%d", cpu);
 	set_user_nice(current, 19);
 	current->flags |= PF_IOTHREAD;
 
 	/* Migrate to the right CPU */
-	set_cpus_allowed(current, 1UL << cpu);
-	if (smp_processor_id() != cpu)
-		BUG();
+	cpu_set(cpu, allowed_mask);
+	set_cpus_allowed(current, allowed_mask);
+	BUG_ON(smp_processor_id() != cpu);
 
 	__set_current_state(TASK_INTERRUPTIBLE);
 	mb();
kernel/workqueue.c +3 -1
diff -urpN linux-2.5.63/kernel/workqueue.c cpu-2.5.63-1/kernel/workqueue.c
@@ -171,6 +171,7 @@ static int worker_thread(void *__startup
 	int cpu = cwq - cwq->wq->cpu_wq;
 	DECLARE_WAITQUEUE(wait, current);
 	struct k_sigaction sa;
+	cpumask_t allowed_mask = CPU_MASK_NONE;
 
 	daemonize("%s/%d", startup->name, cpu);
 	allow_signal(SIGCHLD);
@@ -178,7 +179,8 @@ static int worker_thread(void *__startup
 	cwq->thread = current;
 
 	set_user_nice(current, -10);
-	set_cpus_allowed(current, 1UL << cpu);
+	cpu_set(cpu, allowed_mask);
+	set_cpus_allowed(current, allowed_mask);
 
 	complete(&startup->done);
 
#1 lore raw

On Tue, 25 Feb 2003 09:54:56 -0800
William Lee Irwin III <[email protected]> wrote:

# My comments on this begin with '#'.

arch/i386/kernel/io_apic.c +5 -3
diff -urpN linux-2.5.63/arch/i386/kernel/io_apic.c cpu-2.5.63-1/arch/i386/kernel/io_apic.c
@@ -433,10 +432,12 @@ tryanotherirq:
 
-	allowed_mask = cpu_online_map & irq_affinity[selected_irq];
-	target_cpu_mask = 1 << min_loaded;
+	cpus_and(allowed_mask, cpu_online_map, irq_affinity[selected_irq]);
+	cpus_clear(target_cpu_mask);
+	cpu_set(min_loaded, target_cpu_mask);
+	cpus_and(tmp, target_cpu_mask, allowed_mask);
 
-	if (target_cpu_mask & allowed_mask) {
+	if (cpus_empty(tmp)) {

# ? if (!cpus_empty(tmp)) {
irq_desc_t *desc = irq_desc + selected_irq; Dprintk("irq = %d moved to cpu = %d\n", selected_irq, min_loaded); /* mark for change destination */

include/linux/bitmap.h +6 -0
diff -urpN linux-2.5.63/include/linux/bitmap.h cpu-2.5.63-1/include/linux/bitmap.h
@@ -0,0 +1,148 @@
+#ifndef __LINUX_BITMAP_H
+#define __LINUX_BITMAP_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/bitops.h>

-#define BITS_TO_LONGS(bits) \
- (((bits)+BITS_PER_LONG-1)/BITS_PER_LONG)

# keep this and use it.  (but moved from another file)

+#define DECLARE_BITMAP(name,bits) \
+ unsigned long name[((bits)+BITS_PER_LONG-1)/BITS_PER_LONG]
# unsigned long name[BITS_TO_LONGS(bits)]

+#define CLEAR_BITMAP(name,bits) \
+ memset(name, 0, ((bits)+BITS_PER_LONG-1)/8)
# memset(name, 0, BITS_TO_LONGS(bits) * (BITS_PER_LONG / 8))
# This clears all longs in <name>, so that extra code below can disappear.

+static inline int bitmap_empty(unsigned long *bitmap, int bits)
+{
+ int k;
# int m = BITS_TO_LONGS(bits);
+ for (k = 0; k < bits/BITS_PER_LONG; ++k)
# for (k = 0; k < m; ++k)
+ if (bitmap[k])
+ return 0;
+
+ if (bits % BITS_PER_LONG)
+ return (bitmap[k+1] & (~0UL >> (bits % BITS_PER_LONG))) == 0;
# don't need this 'if' any more.
+
+ return 1;
+}
+
+static inline int bitmap_full(unsigned long *bitmap, int bits)
+{
+ int k;
# int m = BITS_TO_LONGS(bits);
+ for (k = 0; k < bits/BITS_PER_LONG; ++k)
# for (k = 0; k < m; ++k)
+ if (~bitmap[k])
+ return 0;
+
+ if (bits % BITS_PER_LONG)
+ return (~bitmap[k+1] & (~0UL >> (bits % BITS_PER_LONG))) == 0;
# don't need this 'if' any more.
+
+ return 1;
+}
+
+static inline int bitmap_equal(unsigned long *bitmap1, unsigned long *bitmap2, int bits) +{
+ int k;
# int m = BITS_TO_LONGS(bits);
+ for (k = 0; k < bits/BITS_PER_LONG; ++k)
# for (k = 0; k < m; ++k)
+ if (bitmap1[k] != bitmap2[k])
+ return 0;
+
+ if (bits % BITS_PER_LONG) {
+ unsigned long mask = ~0UL >> (bits % BITS_PER_LONG);
+ return (bitmap1[k+1] & mask) == (bitmap2[k+1] & mask);
+ }
# don't need this 'if' any more.
+ return 1;
+}
+
+static inline void bitmap_fill(unsigned long *bitmap, int bits)
+{
+ memset(bitmap, 0xf, (bits+BITS_PER_LONG/8-1)/8);
# 0xf: should be 0xff; however, it's not used (as wli pointed out to me); +}
+
+static inline void bitmap_copy(unsigned long *dst, unsigned long *src, int bits) +{
+ memcpy(dst, src, (bits+BITS_PER_LONG/8-1)/8);
# Change to copy entire longs:
# memcpy(dst, src, BITS_TO_LONGS(bits) * (BITS_PER_LONG / 8));
+}
+
+static inline void bitmap_shift_left(unsigned long *,unsigned long *,int,int);

# why this line above?  not needed.

+static inline void bitmap_shift_right(unsigned long *dst, unsigned long *src, int shift, int bits) +{
...
+}
+
+static inline void bitmap_shift_left(unsigned long *dst, unsigned long *src, int shift, int bits) +{
...
+}

+#elif BITS_PER_LONG == 64
+static inline int bitmap_weight(unsigned long *bitmap, int bits) +{
+ int k, w = 0;
+ u32 *map = (u32 *)bitmap;
+
+ for (k = 0; k < bits/32; ++k)
+ w += hweight32(map[k]);
+
+ if (bits % 32)
+ w += hweight32(map[k] & (~0UL >> (bits % 32)));
+
+ return w;
+}
+#endif

# use hweight64() above?
include/linux/cpumask.h +8 -0
diff -urpN linux-2.5.63/include/linux/cpumask.h cpu-2.5.63-1/include/linux/cpumask.h
@@ -0,0 +1,90 @@
+#ifndef __LINUX_CPUMASK_H
+#define __LINUX_CPUMASK_H
+
+#ifdef CONFIG_SMP
+
+#include <linux/bitmap.h>
+
+#define CPU_ARRAY_SIZE		((NR_CPUS + BITS_PER_LONG - 1)/BITS_PER_LONG)
# define CPU_ARRAY_SIZE		BITS_TO_LONGS(NR_CPUS)

+struct cpumask
+{
+ unsigned long mask[(NR_CPUS+BITS_PER_LONG-1)/BITS_PER_LONG];
# [CPU_ARRAY_SIZE];
+};

--
~Randy

#2 lore raw

On Tue, Feb 25, 2003 at 01:32:07PM -0800, Randy.Dunlap wrote:
- if (target_cpu_mask & allowed_mask) {
+ if (cpus_empty(tmp)) {
# ? if (!cpus_empty(tmp)) {

Yep, that's a bug.

On Tue, Feb 25, 2003 at 01:32:07PM -0800, Randy.Dunlap wrote:
-#define BITS_TO_LONGS(bits) \
- (((bits)+BITS_PER_LONG-1)/BITS_PER_LONG)

# keep this and use it.  (but moved from another file)
+#define DECLARE_BITMAP(name,bits) \
+	unsigned long name[((bits)+BITS_PER_LONG-1)/BITS_PER_LONG]
#	unsigned long name[BITS_TO_LONGS(bits)]
+#define CLEAR_BITMAP(name,bits) \
+	memset(name, 0, ((bits)+BITS_PER_LONG-1)/8)
#	memset(name, 0, BITS_TO_LONGS(bits) * (BITS_PER_LONG / 8))
# This clears all longs in <name>, so that extra code below can disappear.

Header ordering stuff. A wee bit brute-force. All good cleanups. You originally spotted the bitmap_fill() typo too.

Looks like time for a cpu-2.5.63-2 with your changes.

-- wli

#3 lore raw
In reply to Randy.Dunlap
Quoted 1 line · Randy.Dunlap

# My comments on this begin with '#'.

All suggested changes implemented.

cpu-2.5.63-1A, the diff between cpu-2.5.63-1 and cpu-2.5.63-2, follows, both cpu-2.5.63-1A and cpu-2.5.63-2 are also available from:
ftp://ftp.kernel.org/pub/linux/kernel/people/wli/cpu/

(well, uploaded 30s ago, you might have to wait for mirroring)

-- wli

Include Randy Dunlaps BITS_TO_LONGS() cleanups and his irqbalance bugfix.

 arch/i386/kernel/io_apic.c |    2 +-
 include/linux/bitmap.h     |   44 ++++++++++++++------------------------------
 include/linux/cpumask.h    |    4 ++--
 3 files changed, 17 insertions(+), 33 deletions(-)
arch/i386/kernel/io_apic.c +1 -1
diff -urpN cpu-2.5.63-1/arch/i386/kernel/io_apic.c cpu-2.5.63-2/arch/i386/kernel/io_apic.c
@@ -437,7 +437,7 @@ tryanotherirq:
 	cpu_set(min_loaded, target_cpu_mask);
 	cpus_and(tmp, target_cpu_mask, allowed_mask);
 
-	if (cpus_empty(tmp)) {
+	if (!cpus_empty(tmp)) {
 		irq_desc_t *desc = irq_desc + selected_irq;
 		Dprintk("irq = %d moved to cpu = %d\n", selected_irq, min_loaded);
 		/* mark for change destination */
include/linux/bitmap.h +14 -30

120 lines not on this page · raw message

include/linux/cpumask.h +2 -2
diff -urpN cpu-2.5.63-1/include/linux/cpumask.h cpu-2.5.63-2/include/linux/cpumask.h
@@ -5,11 +5,11 @@
 
 #include <linux/bitmap.h>
 
-#define CPU_ARRAY_SIZE		((NR_CPUS + BITS_PER_LONG - 1)/BITS_PER_LONG)
+#define CPU_ARRAY_SIZE		BITS_TO_LONGS(NR_CPUS)
 
 struct cpumask
 {
-	unsigned long mask[(NR_CPUS+BITS_PER_LONG-1)/BITS_PER_LONG];
+	unsigned long mask[CPU_ARRAY_SIZE];
 };
 
 typedef struct cpumask cpumask_t;