Skip to contentSkip to conversation

Subject:[ PATCH 001:003] OFFSCHED CORE

PATCH NAK raz ben yehuda 2 participants 3 messages Aug 22, 2009 – Aug 26, 2009 x86

lore
Share & tools
lore
#0 lore raw

OFFSCHED support.

. A callback invoked just before halting the processor.
. SMP alternative work-around. we do not run smp alternatives if we are
left with a single processor.
. A set of utilities used by the offsched driver.

process.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
smpboot.c | 13 +++++--

2 files changed, 118 insertions(+), 4 deletions(-)
Signed-off-by [email protected]

---

arch/x86/kernel/process.c renamed from linux-2.6.30/arch/x86/kernel/process.c +109 -0
diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/process.c 2.6.30-offsched/arch/x86/kernel/process.c
@@ -613,3 +613,112 @@
 }
 early_param("idle", idle_setup);
 
+#ifdef CONFIG_HOTPLUG_CPU
+struct hotplug_cpu{
+	long flags;
+	void (*hotplug_cpu_dead)(void);
+};
+
+static long 	offsched_services;
+
+#define CPU_OFFSCHED		31
+#define CPU_OFFSCHED_KMEM	30
+#define SERVICE_TIMER		01
+#define SERVICE_NAPI		02
+
+DEFINE_PER_CPU(struct hotplug_cpu, offschedcpu);
+
+void unregister_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead = NULL;
+	clear_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unregister_offsched);
+
+int is_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched);
+
+int is_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched_kmem);
+
+void set_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	set_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(set_offsched_kmem);
+
+void unset_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	clear_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unset_offsched_kmem);
+
+int register_offsched(void (*offsched_callback)(void), int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	if (is_offsched(cpuid))
+		return -1;
+	cpu->hotplug_cpu_dead = offsched_callback;
+	set_bit(CPU_OFFSCHED, &cpu->flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_offsched);
+
+void run_offsched(void)
+{
+	int cpuid = raw_smp_processor_id();
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead();
+}
+
+int  offsched_is_napi_avail(void)
+{
+	return SERVICE_NAPI & offsched_services;
+}
+
+int (*offsched_rule)(void *);
+EXPORT_SYMBOL(offsched_rule);
+/*
+* if we a have filter than apply filter.
+* return 0 if packet should go to kernel
+* return 1 if packet should go to offsched.
+*/
+int offsched_pkt(void *s)
+{
+	if (offsched_rule)
+		return offsched_rule(s);
+	return 0;
+}
+
+void offsched_register_napi(void)
+{
+	offsched_services |= SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_register_napi);
+
+void offsched_unregister_napi(void)
+{
+	offsched_services &= ~SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_unregister_napi);
+
+/*
+ * napi scheudule is called in offsched context.
+*/
+void offsched_napi_schedule(void *n)
+{
+
+}
+
+#endif
arch/x86/kernel/smpboot.c renamed from linux-2.6.30/arch/x86/kernel/smpboot.c +9 -4
diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/smpboot.c 2.6.30-offsched/arch/x86/kernel/smpboot.c
@@ -39,6 +39,8 @@
  *	Glauber Costa		:	i386 and x86_64 integration
  */
 
+#define DEBUG
+
 #include <linux/init.h>
 #include <linux/smp.h>
 #include <linux/module.h>
@@ -686,8 +688,8 @@
 	};
 
 	INIT_WORK(&c_idle.work, do_fork_idle);
-
-	alternatives_smp_switch(1);
+	if (!is_offsched(cpu))
+		alternatives_smp_switch(1);
 
 	c_idle.idle = get_idle_for_cpu(cpu);
 
@@ -1283,8 +1285,9 @@
 	for (i = 0; i < 10; i++) {
 		/* They ack this in play_dead by setting CPU_DEAD */
 		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
-			printk(KERN_INFO "CPU %d is now offline\n", cpu);
-			if (1 == num_online_cpus())
+			printk(KERN_INFO "CPU %d is now offline %s\n", cpu,
+				is_offsched(cpu) ? "and OFFSCHED" : "");
+			if (1 == num_online_cpus() && !is_offsched(cpu))
 				alternatives_smp_switch(0);
 			return;
 		}
@@ -1313,6 +1316,8 @@
 void native_play_dead(void)
 {
 	play_dead_common();
+	if (is_offsched(raw_smp_processor_id()))
+		run_offsched();
 	wbinvd_halt();
 }
 
Nacked-by NAK
#1 lore raw
Quoted 6 lines · raz ben yehuda

OFFSCHED support.

. A callback invoked just before halting the processor.
. SMP alternative work-around. we do not run smp alternatives if we are
left with a single processor.
. A set of utilities used by the offsched driver.

Cpu hotunplug is mostly broken when it comes to irqs. Building on and extending broken code is just asking for trouble, especially for out of tree code.

Quoted 173 lines

process.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
smpboot.c | 13 +++++--

2 files changed, 118 insertions(+), 4 deletions(-)
Signed-off-by [email protected]

---
diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/process.c 2.6.30-offsched/arch/x86/kernel/process.c

arch/x86/kernel/process.c renamed from linux-2.6.30/arch/x86/kernel/process.c +109 -0
@@ -613,3 +613,112 @@
 }
 early_param("idle", idle_setup);
 
+#ifdef CONFIG_HOTPLUG_CPU
+struct hotplug_cpu{
+	long flags;
+	void (*hotplug_cpu_dead)(void);
+};
+
+static long 	offsched_services;
+
+#define CPU_OFFSCHED		31
+#define CPU_OFFSCHED_KMEM	30
+#define SERVICE_TIMER		01
+#define SERVICE_NAPI		02
+
+DEFINE_PER_CPU(struct hotplug_cpu, offschedcpu);
+
+void unregister_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead = NULL;
+	clear_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unregister_offsched);
+
+int is_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched);
+
+int is_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched_kmem);
+
+void set_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	set_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(set_offsched_kmem);
+
+void unset_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	clear_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unset_offsched_kmem);
+
+int register_offsched(void (*offsched_callback)(void), int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	if (is_offsched(cpuid))
+		return -1;
+	cpu->hotplug_cpu_dead = offsched_callback;
+	set_bit(CPU_OFFSCHED, &cpu->flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_offsched);
+
+void run_offsched(void)
+{
+	int cpuid = raw_smp_processor_id();
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead();
+}
+
+int  offsched_is_napi_avail(void)
+{
+	return SERVICE_NAPI & offsched_services;
+}
+
+int (*offsched_rule)(void *);
+EXPORT_SYMBOL(offsched_rule);
+/*
+* if we a have filter than apply filter.
+* return 0 if packet should go to kernel
+* return 1 if packet should go to offsched.
+*/
+int offsched_pkt(void *s)
+{
+	if (offsched_rule)
+		return offsched_rule(s);
+	return 0;
+}
+
+void offsched_register_napi(void)
+{
+	offsched_services |= SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_register_napi);
+
+void offsched_unregister_napi(void)
+{
+	offsched_services &= ~SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_unregister_napi);
+
+/*
+ * napi scheudule is called in offsched context.
+*/
+void offsched_napi_schedule(void *n)
+{
+
+}
+
+#endif

diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/smpboot.c 2.6.30-offsched/arch/x86/kernel/smpboot.c

arch/x86/kernel/smpboot.c renamed from linux-2.6.30/arch/x86/kernel/smpboot.c +9 -5
@@ -39,6 +39,8 @@
  *	Glauber Costa		:	i386 and x86_64 integration
  */
 
+#define DEBUG
+
 #include <linux/init.h>
 #include <linux/smp.h>
 #include <linux/module.h>
@@ -686,8 +688,8 @@
 	};
 
 	INIT_WORK(&c_idle.work, do_fork_idle);
-
-	alternatives_smp_switch(1);
+	if (!is_offsched(cpu))
+		alternatives_smp_switch(1);
 
 	c_idle.idle = get_idle_for_cpu(cpu);
 
@@ -1283,8 +1285,9 @@
 	for (i = 0; i < 10; i++) {
 		/* They ack this in play_dead by setting CPU_DEAD */
 		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
-			printk(KERN_INFO "CPU %d is now offline\n", cpu);
-			if (1 == num_online_cpus())
+			printk(KERN_INFO "CPU %d is now offline %s\n", cpu,
+				is_offsched(cpu) ? "and OFFSCHED" : "");
+			if (1 == num_online_cpus() && !is_offsched(cpu))
 				alternatives_smp_switch(0);
 			return;
 		}
@@ -1313,6 +1316,8 @@
 void native_play_dead(void)
 {
 	play_dead_common();
+	if (is_offsched(raw_smp_processor_id()))
+		run_offsched();
 	wbinvd_halt();
 }
 
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#2 lore raw
Quoted 9 lines · Eric W. Biederman
Quoted 6 lines · raz ben yehuda

OFFSCHED support.

. A callback invoked just before halting the processor.
. SMP alternative work-around. we do not run smp alternatives if we are
left with a single processor.
. A set of utilities used by the offsched driver.

Quoted 6 lines

Cpu hotunplug is mostly broken when it comes to irqs. Building on and extending broken code is just asking for trouble, especially for out of tree code.

I have this code for 4 years. The least of my problems was the hot-unplugging. as for the tree, I will use git. much depends if the patch is excepted. you say - i do.

Quoted 173 lines
Quoted 173 lines

process.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
smpboot.c | 13 +++++--

2 files changed, 118 insertions(+), 4 deletions(-)
Signed-off-by [email protected]

---
diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/process.c 2.6.30-offsched/arch/x86/kernel/process.c

arch/x86/kernel/process.c renamed from linux-2.6.30/arch/x86/kernel/process.c +109 -0
@@ -613,3 +613,112 @@
 }
 early_param("idle", idle_setup);
 
+#ifdef CONFIG_HOTPLUG_CPU
+struct hotplug_cpu{
+	long flags;
+	void (*hotplug_cpu_dead)(void);
+};
+
+static long 	offsched_services;
+
+#define CPU_OFFSCHED		31
+#define CPU_OFFSCHED_KMEM	30
+#define SERVICE_TIMER		01
+#define SERVICE_NAPI		02
+
+DEFINE_PER_CPU(struct hotplug_cpu, offschedcpu);
+
+void unregister_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead = NULL;
+	clear_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unregister_offsched);
+
+int is_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched);
+
+int is_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(is_offsched_kmem);
+
+void set_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	set_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(set_offsched_kmem);
+
+void unset_offsched_kmem(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	clear_bit(CPU_OFFSCHED_KMEM, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unset_offsched_kmem);
+
+int register_offsched(void (*offsched_callback)(void), int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	if (is_offsched(cpuid))
+		return -1;
+	cpu->hotplug_cpu_dead = offsched_callback;
+	set_bit(CPU_OFFSCHED, &cpu->flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_offsched);
+
+void run_offsched(void)
+{
+	int cpuid = raw_smp_processor_id();
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead();
+}
+
+int  offsched_is_napi_avail(void)
+{
+	return SERVICE_NAPI & offsched_services;
+}
+
+int (*offsched_rule)(void *);
+EXPORT_SYMBOL(offsched_rule);
+/*
+* if we a have filter than apply filter.
+* return 0 if packet should go to kernel
+* return 1 if packet should go to offsched.
+*/
+int offsched_pkt(void *s)
+{
+	if (offsched_rule)
+		return offsched_rule(s);
+	return 0;
+}
+
+void offsched_register_napi(void)
+{
+	offsched_services |= SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_register_napi);
+
+void offsched_unregister_napi(void)
+{
+	offsched_services &= ~SERVICE_NAPI;
+}
+EXPORT_SYMBOL_GPL(offsched_unregister_napi);
+
+/*
+ * napi scheudule is called in offsched context.
+*/
+void offsched_napi_schedule(void *n)
+{
+
+}
+
+#endif

diff -X 2.6.30-offsched/Documentation/dontdiff -urN tmp/linux-2.6.30/arch/x86/kernel/smpboot.c 2.6.30-offsched/arch/x86/kernel/smpboot.c

arch/x86/kernel/smpboot.c renamed from linux-2.6.30/arch/x86/kernel/smpboot.c +9 -5
@@ -39,6 +39,8 @@
  *	Glauber Costa		:	i386 and x86_64 integration
  */
 
+#define DEBUG
+
 #include <linux/init.h>
 #include <linux/smp.h>
 #include <linux/module.h>
@@ -686,8 +688,8 @@
 	};
 
 	INIT_WORK(&c_idle.work, do_fork_idle);
-
-	alternatives_smp_switch(1);
+	if (!is_offsched(cpu))
+		alternatives_smp_switch(1);
 
 	c_idle.idle = get_idle_for_cpu(cpu);
 
@@ -1283,8 +1285,9 @@
 	for (i = 0; i < 10; i++) {
 		/* They ack this in play_dead by setting CPU_DEAD */
 		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
-			printk(KERN_INFO "CPU %d is now offline\n", cpu);
-			if (1 == num_online_cpus())
+			printk(KERN_INFO "CPU %d is now offline %s\n", cpu,
+				is_offsched(cpu) ? "and OFFSCHED" : "");
+			if (1 == num_online_cpus() && !is_offsched(cpu))
 				alternatives_smp_switch(0);
 			return;
 		}
@@ -1313,6 +1316,8 @@
 void native_play_dead(void)
 {
 	play_dead_common();
+	if (is_offsched(raw_smp_processor_id()))
+		run_offsched();
 	wbinvd_halt();
 }
 
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/