Skip to contentSkip to conversation

(resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

PATCH Ashok Raj 5 participants 21 messages May 5, 2004 – May 12, 2004

Subsystems (7)
lore
Share & tools
lore

Ashok Raj

author
#0 lore raw

Hi Andrew/DavidM

as i mentioned earlier, i broke the smp build for i386 in the take3 patches i sent this morning. This patch is tested on i386 and ia64 as well.

thanks to Anil Keshavamurthy for taking initiative to test and bringup the issue.

Cheers,
Ashok

Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.

---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++++---------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 +-
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 ++--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 ---
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++++
 linux-2.6.6-rc3-root/init/main.c                |    4 ++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 +-
 linux-2.6.6-rc3-root/kernel/sched.c             |   17 ++++++++++++++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 +-
 10 files changed, 54 insertions(+), 23 deletions(-)
arch/ia64/kernel/smpboot.c renamed from arch/ia64/kernel/smpboot.c~cpu_present +13 -9
diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
include/linux/cpumask.h renamed from include/linux/cpumask.h~cpu_present +11 -0
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
init/main.c renamed from init/main.c~cpu_present +2 -2
diff -puN init/main.c~cpu_present init/main.c
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
kernel/cpu.c renamed from kernel/cpu.c~cpu_present +9 -1
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
fs/proc/proc_misc.c renamed from fs/proc/proc_misc.c~cpu_present +2 -2
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
kernel/sched.c renamed from kernel/sched.c~cpu_present +14 -3
diff -puN kernel/sched.c~cpu_present kernel/sched.c
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
@@ -2944,6 +2944,17 @@ void __init sched_init(void)
 	runqueue_t *rq;
 	int i, j, k;
 
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+
 	for (i = 0; i < NR_CPUS; i++) {
 		prio_array_t *array;
 
fs/buffer.c renamed from fs/buffer.c~cpu_present +1 -1
diff -puN fs/buffer.c~cpu_present fs/buffer.c
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
kernel/fork.c renamed from kernel/fork.c~cpu_present +1 -1
diff -puN kernel/fork.c~cpu_present kernel/fork.c
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
kernel/timer.c renamed from kernel/timer.c~cpu_present +1 -1
diff -puN kernel/timer.c~cpu_present kernel/timer.c
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
include/asm-ia64/smp.h renamed from include/asm-ia64/smp.h~cpu_present +0 -3
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

#1 lore raw
Quoted 2 lines · Ashok Raj

Name: cpu_present_map.patch

This does not play happily with the sched_domains patches.

Program received signal SIGEMT, Emulation trap. task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
1087 return p->thread_info->cpu;
(gdb) bt

#0  task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
#1  0xc01187cd in try_to_wake_up (p=0x0, state=7, sync=0) at kernel/sched.c:844
#2  0xc0118a1d in wake_up_process (p=0x0) at kernel/sched.c:981
#3  0xc011b688 in cpu_attach_domain (sd=0xc120b060, cpu=-805728492) at kernel/sched.c:3804
#4  0xc05db60b in arch_init_sched_domains () at arch/i386/kernel/smpboot.c:1327
#5  0xc05dfb44 in sched_init_smp () at kernel/sched.c:4013
#6  0xc010045d in init (unused=0x0) at init/main.c:643
#7  0xc010428d in kernel_thread_helper () at arch/i386/kernel/process.c:246

(gdb) p cpu_online_map
$1 = 1
(gdb) f 3
#3 0xc011b688 in cpu_attach_domain (sd=0xc120b060, cpu=-805728492) at kernel/sched.c:3804
3804 wake_up_process(rq->migration_thread);
(gdb) p cpu
$2 = -805728492
(gdb) up
#4 0xc05db60b in arch_init_sched_domains () at arch/i386/kernel/smpboot.c:1327
1327 cpu_attach_domain(cpu_domain, i);
(gdb) p i
$3 = 1
(gdb) info thr

  5 Thread 32769 (swapper)  start_secondary (unused=0x0) at arch/i386/kernel/smpboot.c:446
* 4 Thread 32768 (swapper)  0xc0398ae1 in schedule () at kernel/sched.c:1198
  3 Thread 3 (ksoftirqd/0)  0xc0398ae1 in schedule () at kernel/sched.c:1198
  2 Thread 2 (migration/0)  migration_thread (data=0x0) at include/asm/current.h:10

1 Thread 1 (swapper) task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
(gdb)

It appears that x86's arch_init_sched_domains() is blindly assuming that each CPU's migration thread is running and is pointed to by rq->migration_thread. But from the above you'll see that CPU0's migration thread is running, but CPU1's has never been created, hence the null-pointer deref.

I'll drop this final patch. Please reissue it against next -mm. Thanks.

#2 lore raw

various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

Hmmm .. we're getting backed up here. Sure glad Andrew's got the job of conducting this, not me ...

The purpose of this message is to list several patches that are interacting, and the order that I'm guessing Andrew will end up taking them. I claim no authority, and at best very limited forecasting ability.

This is just to put a source of possible confusions on the table, by running it up the flagpole, and inviting the shooting to begin.

The following patches are colliding or interacting, listed in the order that I'm guessing they will end up going into *-mm, and with my guess of their current status:

1. Nick Piggin's sched_domains patches (in *-mm now)
2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
6. Andi Kleen's numa placement (being reviewed now, I think)

Actually, bssprot is independent, once it resolves some ia64 issue, and the last two (nodemask and numa) are sufficiently independent to go in parallel or reverse order.

If you _wanted_, Andrew, to add to your current patch load, I'd be delighted to submit my final bitmap/cpumask cleanup now - I'm just guessing that you will politely ignore my offer.

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
#3 lore raw

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

Quoted 8 lines · Paul Jackson

Hmmm .. we're getting backed up here. Sure glad Andrew's got the job of conducting this, not me ...

The purpose of this message is to list several patches that are interacting, and the order that I'm guessing Andrew will end up taking them. I claim no authority, and at best very limited forecasting ability.

I try to keep patches in the expected merge-up order. The patching order is in the series file - see the `patch-series' symlink in the kerne.org directory.

Quoted 7 lines

This is just to put a source of possible confusions on the table, by running it up the flagpole, and inviting the shooting to begin.

The following patches are colliding or interacting, listed in the order that I'm guessing they will end up going into *-mm, and with my guess of their current status:

Quoted 6 lines

1. Nick Piggin's sched_domains patches (in *-mm now)
2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
6. Andi Kleen's numa placement (being reviewed now, I think)

Planned for shortly after 2.6.6 are:

sched-domains
rmap 7-14 (15-23 probably a week later)
numa api

Quoted 3 lines

Actually, bssprot is independent, once it resolves some ia64 issue, and the last two (nodemask and numa) are sufficiently independent to go in parallel or reverse order.

I dropped bssprot - it was causing too much grief.

Quoted 3 lines

If you _wanted_, Andrew, to add to your current patch load, I'd be delighted to submit my final bitmap/cpumask cleanup now - I'm just guessing that you will politely ignore my offer.

I confess to being vaguely stunned that it is possible to generate a ~20 patch series againt the bitmap code. I'll pay more attention next time it flies past.

#4 lore raw

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

Quoted 1 line

The patching order is in the series file ...

Yes - I am aware of your series file. So far as I know, only Nick's patch is still in it, of the patches I listed.

Quoted 2 lines

I confess to being vaguely stunned that it is possible to generate a ~20 patch series againt the bitmap code.

Perhaps I sliced and diced it too finely ;).

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
#5 lore raw

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

In reply to Paul Jackson
Quoted 9 lines · Paul Jackson

Hmmm .. we're getting backed up here. Sure glad Andrew's got the job of conducting this, not me ...
The purpose of this message is to list several patches that are interacting, and the order that I'm guessing Andrew will end up taking them. I claim no authority, and at best very limited forecasting ability.
This is just to put a source of possible confusions on the table, by running it up the flagpole, and inviting the shooting to begin. The following patches are colliding or interacting, listed in the order

Quoted 8 lines

that I'm guessing they will end up going into *-mm, and with my guess of their current status:
1. Nick Piggin's sched_domains patches (in *-mm now)
2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
6. Andi Kleen's numa placement (being reviewed now, I think)

I don't see any essential interaction between these patches. This appears to be 100% mergewerk. I don't think there's any essential conflict, just potential PITA rediffing for the (re)senders if/when they touch the same lines of code and things get accepted in a different order from what ppl merged their stuff against. As far as I'm concerned, business as usual.

-- wli

#6 lore raw

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

Quoted 1 line

I don't see any essential interaction between these patches.

Essentially, yes. The interactions were:

Ashok's Hotplug added an essential arch-specific ifdef to a
arch-specific cpumask file that my cleanup removes - so I had to work with Ashok to remove that arch dependency.

Reiser's bssprot broke ia64, so I couldn't test with that patch.

Matthew has recoded his nodemask patch to presume my cpumask cleanup.

Andi's numa patch is so far as we know independent, but close enough
to all this other activity to be at risk for something.

The interactions were enough that I probably couldn't expect Andrew to wade through them to adapt my cleanup. Rather I pretty much have to deliver my cleanup ready to go, with whatever Andrew thinks is his current *-mm series.

I'll be doing well just to get Andrew to look at it, despite what I take to be endorsements from Matthew, Rusty and Joe Korty, and conditional acceptance from yourself. Andrew would probably drop it in a heartbeat if it collided non-trivially.

So merge work - yes. But merge work that mostly I need to do, which makes me very sensitive to the order in which things happen.

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
#7 lore raw

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

Quoted 10 lines · Paul Jackson

The interactions were enough that I probably couldn't expect Andrew to wade through them to adapt my cleanup. Rather I pretty much have to deliver my cleanup ready to go, with whatever Andrew thinks is his current *-mm series.
I'll be doing well just to get Andrew to look at it, despite what I take to be endorsements from Matthew, Rusty and Joe Korty, and conditional acceptance from yourself. Andrew would probably drop it in a heartbeat if it collided non-trivially.
So merge work - yes. But merge work that mostly I need to do, which makes me very sensitive to the order in which things happen.

Yeah, merging is a pain in the butt. I make vague attempts to merge on a minute-to-minute basis, but that has me, well, basically livelocked. So it is with Linux in general, I suppose.

-- wli

#8 lore raw
In reply to Andrew Morton
Quoted 8 lines · Andrew Morton

Ashok Raj <[email protected]> wrote:

It appears that x86's arch_init_sched_domains() is blindly assuming that each CPU's migration thread is running and is pointed to by rq->migration_thread. But from the above you'll see that CPU0's migration thread is running, but CPU1's has never been created, hence the null-pointer deref.

The assumption is that the migration thread is running unless cpu_is_offline is true. I think this is still OK, I can have another look at it if anyone thinks otherwise.

As far as arch_init_sched_domains goes, it can be run as late as you like if it needs to be moved.

author
#9 lore raw
In reply to Andrew Morton
Quoted 8 lines · Andrew Morton
Quoted 2 lines · Ashok Raj

Name: cpu_present_map.patch

This does not play happily with the sched_domains patches.

I'll drop this final patch. Please reissue it against next -mm. Thanks.

Thanks, i will rework the patch and resend soon.

Cheers,
Ashok
-- Linux Core Software Group

author
#10 lore raw

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

In reply to Andrew Morton
Quoted 7 lines · Andrew Morton
Quoted 2 lines · Ashok Raj

Name: cpu_present_map.patch

This does not play happily with the sched_domains patches.

I'll drop this final patch. Please reissue it against next -mm. Thanks.

Ooopsy Daisy...

apparently i picked up the patches from a different directory. This was the exact problem i fixed with the (supposed resend patch)... this one should work with sched domain, and i also tested with that on i386 MP as well.

below is patch 6 resend-2

cheers,
ashok

Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.

---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++++---------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 +-
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 ++--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 ---
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++++
 linux-2.6.6-rc3-root/init/main.c                |   21 +++++++++++++++++++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 +-
 linux-2.6.6-rc3-root/kernel/sched.c             |    6 +++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 +-
 10 files changed, 60 insertions(+), 23 deletions(-)
arch/ia64/kernel/smpboot.c renamed from arch/ia64/kernel/smpboot.c~cpu_present +13 -9
diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
include/linux/cpumask.h renamed from include/linux/cpumask.h~cpu_present +11 -0
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
init/main.c renamed from init/main.c~cpu_present +19 -2
diff -puN init/main.c~cpu_present init/main.c
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
@@ -577,6 +577,22 @@ static void run_init_process(char *init_
 	execve(init_filename, argv_init, envp_init);
 }
 
+static void fixup_cpu_present_map(void)
+{
+	int i;
+
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+}
+
 static int init(void * unused)
 {
 	lock_kernel();
@@ -595,6 +611,7 @@ static int init(void * unused)
 
 	do_pre_smp_initcalls();
 
+	fixup_cpu_present_map();
 	smp_init();
 	do_basic_setup();
 
kernel/cpu.c renamed from kernel/cpu.c~cpu_present +9 -1
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
fs/proc/proc_misc.c renamed from fs/proc/proc_misc.c~cpu_present +2 -2
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
kernel/sched.c renamed from kernel/sched.c~cpu_present +3 -3
diff -puN kernel/sched.c~cpu_present kernel/sched.c
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
fs/buffer.c renamed from fs/buffer.c~cpu_present +1 -1
diff -puN fs/buffer.c~cpu_present fs/buffer.c
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
kernel/fork.c renamed from kernel/fork.c~cpu_present +1 -1
diff -puN kernel/fork.c~cpu_present kernel/fork.c
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
kernel/timer.c renamed from kernel/timer.c~cpu_present +1 -1
diff -puN kernel/timer.c~cpu_present kernel/timer.c
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
include/asm-ia64/smp.h renamed from include/asm-ia64/smp.h~cpu_present +0 -3
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

#11 lore raw

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

Quoted 2 lines · Ashok Raj

Name: cpu_present_map.patch

Ho-hum. Please at least compile-test non-trivial patches with CONFIG_SMP=n, especially when they dink with bitmasks, bitmaps and SMP-specific features.

init/main.c: In function `fixup_cpu_present_map':
init/main.c:636: warning: use of compound expressions as lvalues is deprecated init/main.c:636: error: invalid lvalue in assignment

Due to:

			cpu_set(i, cpu_present_map);

It appears that cpu_set() is simply broken on UP:

#define cpu_present_map cpumask_of_cpu(0)

#define cpumask_of_cpu(cpu) ({ ((cpumask_t)1) << (cpu); })

#define cpu_set(cpu, map) do { (void)(cpu); cpus_coerce(map) = 1UL; } while (0)

Put those things together and there's no way it can work. It's not even conceptually right: cpu_present_map is a "constant" on UP and we have no business trying to modify it. So perhaps a build error is the appropriate response.

I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about the more general details.

author
#12 lore raw

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

Quoted 11 lines · Andrew Morton
Quoted 2 lines · Ashok Raj

Name: cpu_present_map.patch

Ho-hum. Please at least compile-test non-trivial patches with CONFIG_SMP=n, especially when they dink with bitmasks, bitmaps and SMP-specific features.

I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about the more general details.

Now under the protective covers of CONFIG_SMP, init/main.c compiles now
but ran into another link error, seems like some workaround required in kthread_bind as well as kthread_wait_task_inactive is defined in sched.c only for CONFIG_SMP case.

btw; maybe i should rename this patch and all the problems will go away... :-)

kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
: undefined reference to `kthread_wait_task_inactive'
make: *** [.tmp_vmlinux1] Error 1

Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.

---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++-------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 -
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 +--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 --
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++
 linux-2.6.6-rc3-root/init/main.c                |   28 ++++++++++++++++++++++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 -
 linux-2.6.6-rc3-root/kernel/sched.c             |    6 ++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 -
 10 files changed, 67 insertions(+), 23 deletions(-)
arch/ia64/kernel/smpboot.c renamed from arch/ia64/kernel/smpboot.c~cpu_present +13 -9
diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
include/linux/cpumask.h renamed from include/linux/cpumask.h~cpu_present +11 -0
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
init/main.c renamed from init/main.c~cpu_present +26 -2
diff -puN init/main.c~cpu_present init/main.c
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
@@ -577,6 +577,29 @@ static void run_init_process(char *init_
 	execve(init_filename, argv_init, envp_init);
 }
 
+#ifdef CONFIG_SMP
+/*
+ * Dirty Hack now, maybe cpu_set for UP needs to be redefined for UP??
+ */
+static void fixup_cpu_present_map(void)
+{
+	int i;
+
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+}
+#else
+#define fixup_cpu_present_map()
+#endif
+
 static int init(void * unused)
 {
 	lock_kernel();
@@ -595,6 +618,7 @@ static int init(void * unused)
 
 	do_pre_smp_initcalls();
 
+	fixup_cpu_present_map();
 	smp_init();
 	do_basic_setup();
 
kernel/cpu.c renamed from kernel/cpu.c~cpu_present +9 -1
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
fs/proc/proc_misc.c renamed from fs/proc/proc_misc.c~cpu_present +2 -2
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
kernel/sched.c renamed from kernel/sched.c~cpu_present +3 -3
diff -puN kernel/sched.c~cpu_present kernel/sched.c
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
fs/buffer.c renamed from fs/buffer.c~cpu_present +1 -1
diff -puN fs/buffer.c~cpu_present fs/buffer.c
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
kernel/fork.c renamed from kernel/fork.c~cpu_present +1 -1
diff -puN kernel/fork.c~cpu_present kernel/fork.c
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
kernel/timer.c renamed from kernel/timer.c~cpu_present +1 -1
diff -puN kernel/timer.c~cpu_present kernel/timer.c
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
include/asm-ia64/smp.h renamed from include/asm-ia64/smp.h~cpu_present +0 -3
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

#13 lore raw

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

Quoted 4 lines · Ashok Raj
Quoted 3 lines

I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about the more general details.

Now under the protective covers of CONFIG_SMP, init/main.c compiles now

I already fixed it up, thanks. With an incremental patch, which is very much preferred over a wholesale replacement.

Quoted 3 lines

kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
: undefined reference to `kthread_wait_task_inactive'
make: *** [.tmp_vmlinux1] Error 1

There is no symbol `kthread_wait_task_inactive' in the tree.

author
#14 lore raw

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

Quoted 9 lines · Andrew Morton
Quoted 4 lines · Ashok Raj
Quoted 3 lines

I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about the more general details.

Now under the protective covers of CONFIG_SMP, init/main.c compiles now

I already fixed it up, thanks. With an incremental patch, which is very much preferred over a wholesale replacement.

I will in future.

Quoted 6 lines
Quoted 3 lines

kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
: undefined reference to `kthread_wait_task_inactive'
make: *** [.tmp_vmlinux1] Error 1

There is no symbol `kthread_wait_task_inactive' in the tree.

I think i put the experimental i386 hotplug patch, and that might have brought something with it.. i see the -mm compile clean now.

Cheers,
ashok

author
#15 lore raw

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [7/7]

In reply to Andrew Morton
Quoted 2 lines · Andrew Morton

Ashok Raj <[email protected]> wrote:
I'll drop this final patch. Please reissue it against next -mm. Thanks.

Below is resend of IA64 arch hotplug sections.. tested with 2.6.6-rc3-mm2

Name: hotcpu_ia64.patch
Author: Ashok Raj (Intel Corporation)
D: Supports basic ability to enable hotplug functions for IA64.
D: Code is just evolving, and there are several loose ends to tie up. D:
D: What this code drop does
D: - Support logical online and offline
D: - Handles interrupt migration without loss of interrupts.
D: - Handles stress fine > 24+ hrs with make -j/ftp/rcp workloads
D: - Handles irq migration from a dying cpu without loss of interrupts.
D: What needs to be done
D: - Boot CPU removal support, with platform level authentication
D: - Putting cpu being removed in BOOT_RENDEZ mode.

---

 linux-2.6.6-rc3-root/arch/ia64/Kconfig           |    9 +
 linux-2.6.6-rc3-root/arch/ia64/kernel/iosapic.c  |    7 -
 linux-2.6.6-rc3-root/arch/ia64/kernel/irq.c      |  103 +++++++++++++++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/irq_ia64.c |   60 +++++++++-
 linux-2.6.6-rc3-root/arch/ia64/kernel/process.c  |   43 +++++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/sal.c      |   13 ++
 linux-2.6.6-rc3-root/arch/ia64/kernel/smp.c      |   26 ++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c  |  136 +++++++++++++++++++++--
 linux-2.6.6-rc3-root/arch/ia64/kernel/time.c     |    5 
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h      |    2 
 10 files changed, 386 insertions(+), 18 deletions(-)
arch/ia64/Kconfig renamed from arch/ia64/Kconfig~hotcpu_ia64 +9 -0
diff -puN arch/ia64/Kconfig~hotcpu_ia64 arch/ia64/Kconfig
@@ -78,6 +78,15 @@ config IA64_HP_SIM
 
 endchoice
 
+config HOTPLUG_CPU
+    bool "Support for hot-pluggable CPUs (EXPERIMENTAL)"
+    depends on SMP && HOTPLUG && EXPERIMENTAL
+	default n
+    ---help---
+      Say Y here to experiment with turning CPUs off and on.  CPUs
+      can be controlled through /sys/devices/system/cpu/cpu#.
+      Say N if you want to disable cpu hotplug.
+
 choice
 	prompt "Processor type"
 	default ITANIUM
arch/ia64/kernel/irq.c renamed from arch/ia64/kernel/irq.c~hotcpu_ia64 +103 -0

135 lines not on this page · raw message

arch/ia64/kernel/process.c renamed from arch/ia64/kernel/process.c~hotcpu_ia64 +42 -1

86 lines not on this page · raw message

arch/ia64/kernel/smpboot.c renamed from arch/ia64/kernel/smpboot.c~hotcpu_ia64 +127 -9

216 lines not on this page · raw message

arch/ia64/kernel/smp.c renamed from arch/ia64/kernel/smp.c~hotcpu_ia64 +25 -1
diff -puN arch/ia64/kernel/smp.c~hotcpu_ia64 arch/ia64/kernel/smp.c
@@ -71,10 +71,23 @@ static volatile struct call_data_struct 
 /* This needs to be cacheline aligned because it is written to by *other* CPUs.  */
 static DEFINE_PER_CPU(u64, ipi_operation) ____cacheline_aligned;
 
+extern void cpu_halt (void);
+
+void
+lock_ipi_calllock(void)
+{
+	spin_lock_irq(&call_lock);
+}
+
+void
+unlock_ipi_calllock(void)
+{
+	spin_unlock_irq(&call_lock);
+}
+
 static void
 stop_this_cpu (void)
 {
-	extern void cpu_halt (void);
 	/*
 	 * Remove this CPU:
 	 */
@@ -84,6 +97,17 @@ stop_this_cpu (void)
 	cpu_halt();
 }
 
+void
+cpu_die(void)
+{
+	max_xtp();
+	local_irq_disable();
+	cpu_halt();
+	/* Should never be here */
+	BUG();
+	for (;;);
+}
+
 irqreturn_t
 handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
 {
arch/ia64/kernel/time.c renamed from arch/ia64/kernel/time.c~hotcpu_ia64 +5 -0
diff -puN arch/ia64/kernel/time.c~hotcpu_ia64 arch/ia64/kernel/time.c
@@ -10,6 +10,7 @@
  */
 #include <linux/config.h>
 
+#include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -244,6 +245,10 @@ timer_interrupt (int irq, void *dev_id, 
 {
 	unsigned long new_itm;
 
+	if (unlikely(cpu_is_offline(smp_processor_id()))) {
+		return IRQ_HANDLED;
+	}
+
 	platform_timer_interrupt(irq, dev_id, regs);
 
 	new_itm = local_cpu_data->itm_next;
include/asm-ia64/smp.h renamed from include/asm-ia64/smp.h~hotcpu_ia64 +2 -0
diff -puN include/asm-ia64/smp.h~hotcpu_ia64 include/asm-ia64/smp.h
@@ -120,6 +120,8 @@ extern void smp_do_timer (struct pt_regs
 extern int smp_call_function_single (int cpuid, void (*func) (void *info), void *info,
 				     int retry, int wait);
 extern void smp_send_reschedule (int cpu);
+extern void lock_ipi_calllock(void);
+extern void unlock_ipi_calllock(void);
 
 #endif /* CONFIG_SMP */
 #endif /* _ASM_IA64_SMP_H */
arch/ia64/kernel/iosapic.c renamed from arch/ia64/kernel/iosapic.c~hotcpu_ia64 +5 -2
diff -puN arch/ia64/kernel/iosapic.c~hotcpu_ia64 arch/ia64/kernel/iosapic.c
@@ -32,6 +32,8 @@
  * 03/02/19	B. Helgaas	Make pcat_compat system-wide, not per-IOSAPIC.
  *				Remove iosapic_address & gsi_base from external interfaces.
  *				Rationalize __init/__devinit attributes.
+ * 04/12/04 Ashok Raj	<[email protected]> Intel Corporation 2004
+ *				Updated to work with irq migration necessary for CPU Hotplug
  */
 /*
  * Here is what the interrupt logic between a PCI device and the kernel looks like:
@@ -189,8 +191,10 @@ set_rte (unsigned int vector, unsigned i
 	pol     = iosapic_intr_info[vector].polarity;
 	trigger = iosapic_intr_info[vector].trigger;
 	dmode   = iosapic_intr_info[vector].dmode;
+	vector &= (~IA64_IRQ_REDIRECTED);
 
 	redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
+
 #ifdef CONFIG_SMP
 	{
 		unsigned int irq;
@@ -312,9 +316,8 @@ iosapic_set_affinity (unsigned int irq, 
 
 	spin_lock_irqsave(&iosapic_lock, flags);
 	{
-		/* get current delivery mode by reading the low32 */
-		writel(IOSAPIC_RTE_LOW(rte_index), addr + IOSAPIC_REG_SELECT);
 		low32 = iosapic_intr_info[vec].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
+
 		if (redir)
 		        /* change delivery mode to lowest priority */
 			low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
arch/ia64/kernel/irq_ia64.c renamed from arch/ia64/kernel/irq_ia64.c~hotcpu_ia64 +55 -5

84 lines not on this page · raw message

arch/ia64/kernel/sal.c renamed from arch/ia64/kernel/sal.c~hotcpu_ia64 +13 -0
diff -puN arch/ia64/kernel/sal.c~hotcpu_ia64 arch/ia64/kernel/sal.c
@@ -122,10 +122,23 @@ sal_desc_entry_point (void *p)
 static void __init
 set_smp_redirect (int flag)
 {
+#ifndef CONFIG_HOTPLUG_CPU
 	if (no_int_routing)
 		smp_int_redirect &= ~flag;
 	else
 		smp_int_redirect |= flag;
+#else
+	/*
+	 * For CPU Hotplug we dont want to do any chipset supported
+	 * interrupt redirection. The reason is this would require that
+	 * All interrupts be stopped and hard bind the irq to a cpu.
+	 * Later when the interrupt is fired we need to set the redir hint
+	 * on again in the vector. This is combersome for something that the
+	 * user mode irq balancer will solve anyways.
+	 */
+	no_int_routing=1;
+	smp_int_redirect &= ~flag;
+#endif
 }
 #else
 #define set_smp_redirect(flag)	do { } while (0)

_

#16 lore raw
In reply to Ashok Raj

Being somewhat thick in the head at times, it took me a lengthy private email thread with Ashok to understand why he needed to add a cpu_present_map for his cpu hotplug work, and just how that new map related to the existing ones.

After patient and repeated explanations from Ashok, I worked out the following explanation that he found accurate. On the off chance that others find this way of phrasing it helpful, I post it here for the record.

===

Ashok's cpu hot plug patch adds a cpu_present_map, resulting in the following cpu maps being available. All the following maps are fixed size bitmaps of size NR_CPUS.

#ifdef CONFIG_HOTPLUG_CPU
cpu_possible_map - map with all NR_CPUS bits set cpu_present_map - map with bit 'cpu' set iff cpu is populated cpu_online_map - map with bit 'cpu' set iff cpu available to scheduler
#else
cpu_possible_map - map with bit 'cpu' set iff cpu is populated cpu_present_map - copy of cpu_possible_map
cpu_online_map - map with bit 'cpu' set iff cpu available to scheduler
#endif

In either case, NR_CPUS is fixed at compile time, as the static size of these bitmaps. The cpu_possible_map is fixed at boot time, as the set of CPU id's that it is possible might ever be plugged in at anytime during the life of that system boot. The cpu_present_map is dynamic(*), representing which CPUs are currently plugged in. And cpu_online_map is the dynamic subset of cpu_present_map, indicating those CPUs available for scheduling.

If HOTPLUG is enabled, then cpu_possible_map is forced to have all NR_CPUS bits set, otherwise it is just the set of CPUs that ACPI reports present at boot.

If HOTPLUG is enabled, then cpu_present_map varies dynamically, depending on what ACPI reports as currently plugged in, otherwise cpu_present_map is just a copy of cpu_possible_map.

(*) Well, cpu_present_map is dynamic in the hotplug case.
If not hotplug, it's the same as cpu_possible_map, hence fixed at boot.

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
#17 lore raw
Quoted 4 lines · Paul Jackson

On the off chance that
others find this way of phrasing it helpful, I post it here for the record.

Thanks, I added that to the changelog. If you think additional code commentary is needed, please send patches.

I guess I'm not doing anything useful with these patches. Is it OK with everyone if I scoot them over to davidm?

#18 lore raw
Quoted 1 line

Thanks, I added that to the changelog.

Good idea - thanks.

Quoted 2 lines

If you think additional code commentary is needed, please send patches.

Ok - I've no plans to add any code comments at this time to this code. Patches make sense if I did.

Quoted 1 line

Is it OK with everyone if I scoot them over to davidm?

Either way ... doesn't matter to me.

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
author
#19 lore raw
In reply to Andrew Morton
Quoted 11 lines · Andrew Morton
Quoted 4 lines · Paul Jackson

On the off chance that
others find this way of phrasing it helpful, I post it here for the record.

Thanks, I added that to the changelog. If you think additional code commentary is needed, please send patches.

I guess I'm not doing anything useful with these patches. Is it OK with everyone if I scoot them over to davidm?

I had sent a private mail to davidm for inclusion in ia64 tree, he mentioned that it would be preferable to cook akpm tree to make sure no build breaks, and then he would consider those for inclusion in his tree. especially -mm gets more testing, but iam not sure how may would really try this hotplug :-)

anytime now seems appropriate...

until David pickup's these it would be great to leave the here for anyone interested in testing...

Cheers,
Ashok

#20 lore raw
Quoted 5 lines · Ashok Raj

I had sent a private mail to davidm for inclusion in ia64 tree, he mentioned that it would be preferable to cook akpm tree to make sure no build breaks, and then he would consider those for inclusion in his tree. especially -mm gets more testing, but iam not sure how may would really try this hotplug :-)

Oh the patches are fine wrt side-effects. I've had them for a couple of weeks, they're tested on at least x86, ppc64, ppc, ia64, x86_64 and sparc64.

It's just a question of whether David would prefer to handle them?

Affected files are:

arch/ia64/dig/Makefile
arch/ia64/dig/topology.c
arch/ia64/Kconfig
arch/ia64/kernel/iosapic.c
arch/ia64/kernel/irq.c
arch/ia64/kernel/irq_ia64.c
arch/ia64/kernel/palinfo.c
arch/ia64/kernel/process.c
arch/ia64/kernel/sal.c
arch/ia64/kernel/setup.c
arch/ia64/kernel/smpboot.c
arch/ia64/kernel/smp.c
arch/ia64/kernel/time.c
arch/ia64/mm/init.c
arch/ia64/mm/tlb.c
fs/proc/proc_misc.c
include/asm-ia64/cpu.h
include/asm-ia64/cpumask.h
include/asm-ia64/smp.h
include/linux/cpumask.h
init/main.c
kernel/cpu.c
kernel/sched.c