Subject: PATCH : offline scheduler ( some refer to it as a CPU isolation )

PATCH raz ben yehuda 2 participants 3 messages Apr 18, 2009 – Apr 19, 2009

Subsystems (5)

x86 MM core Scheduler exec and binfmt +1 more subsystems

lore

Share & tools

lore

raz ben yehuda

author

2009-04-18 16:12 UTC #0 lore raw

offsched is a platform aimed to assign a service to an off-lined processor. Motivation is explained in:
http://sos-linux.svn.sourceforge.net/viewvc/sos-linux/offsched/trunk/Documentation/OFFSCHED.pdf
Currently I have implemented two services:

HYBRID REAL TIME LINUX
Implemented as a A 1us timer. This timer shows how a true real time system may co-exist with a regular linux server. This way there is no enforcement of a real time system requirements on the entire kernel. Full documentation ( and the notion of an offlet ) is at:
http://sos-linux.svn.sourceforge.net/viewvc/sos-linux/offsched/trunk/Documentation/OFFSCHED-RT.pdf

RTOP
This piece of software send system statistics to a remove server.
The user benefit is that even if the machine is un-accessible (remotely or locally) RTOP still sends system statistics to a remote server. I have showed in a small paper what RTOP is:
http://sos-linux.svn.sourceforge.net/viewvc/sos-linux/offsched/trunk/Documentation/OFFSCHED-RTOP.pdf

The patch is mostly a facility for offsched. The exporting of tasklist_lock is because RTOP is implemented as a driver and it must lock the tasks list.

Signed-off-by Raz Ben Yehuda <[email protected]>

arch/x86/kernel/process.c |   42 ++++++++++++++++++++++++++++++++++++++++++
arch/x86/kernel/smpboot.c |   11 +++++++----
include/linux/cpu.h       |   20 ++++++++++++++++++++
include/linux/sched.h     |    2 +-
kernel/cpu.c              |    1 +
kernel/fork.c             |    6 ++++++
6 files changed, 77 insertions(+), 5 deletions(-)

arch/x86/kernel/process.c +42 -0

index ca98915..123b32d 100644
@@ -613,3 +613,45 @@ static int __init idle_setup(char *str)
 }
 early_param("idle", idle_setup);
 
+#ifdef CONFIG_HOTPLUG_CPU
+struct hotplug_cpu{
+	long flags;
+	void (*hotplug_cpu_dead)(void);
+};
+
+#define CPU_OFFSCHED 31
+
+DEFINE_PER_CPU(struct hotplug_cpu, offschedcpu);
+
+void unregister_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead = NULL;
+	clear_bit(CPU_OFFSCHED, &cpu->flags);
+}
+EXPORT_SYMBOL_GPL(unregister_offsched);
+
+int is_offsched(int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	return test_bit(CPU_OFFSCHED, &cpu->flags);
+}
+
+int register_offsched(void (*offsched_callback)(void), int cpuid)
+{
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	if (is_offsched(cpuid))
+		return -1;
+	cpu->hotplug_cpu_dead = offsched_callback;
+	set_bit(CPU_OFFSCHED, &cpu->flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_offsched);
+
+void run_offsched(void)
+{
+	int cpuid = raw_smp_processor_id();
+	struct hotplug_cpu *cpu = &per_cpu(offschedcpu, cpuid);
+	cpu->hotplug_cpu_dead();
+}
+#endif

arch/x86/kernel/smpboot.c +7 -4

index 58d24ef..25e70e0 100644
@@ -686,8 +686,8 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
 	};
 
 	INIT_WORK(&c_idle.work, do_fork_idle);
-
-	alternatives_smp_switch(1);
+	if (!is_offsched(cpu))
+		alternatives_smp_switch(1);
 
 	c_idle.idle = get_idle_for_cpu(cpu);

@@ -1283,8 +1283,9 @@ void native_cpu_die(unsigned int cpu)
 	for (i = 0; i < 10; i++) {
 		/* They ack this in play_dead by setting CPU_DEAD */
 		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
-			printk(KERN_INFO "CPU %d is now offline\n", cpu);
-			if (1 == num_online_cpus())
+			printk(KERN_INFO "CPU %d is now offline %s\n", cpu,
+				is_offsched(cpu) ? "and OFFSCHED" : "");
+			if (1 == num_online_cpus() && !is_offsched(cpu))
 				alternatives_smp_switch(0);
 			return;
 		}

@@ -1313,6 +1314,8 @@ void play_dead_common(void)
 void native_play_dead(void)
 {
 	play_dead_common();
+	if (is_offsched(raw_smp_processor_id()))
+		run_offsched();
 	wbinvd_halt();
 }

include/linux/cpu.h +20 -0

index 2643d84..f2048ca 100644
@@ -43,6 +43,7 @@ extern int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls);
 
 #ifdef CONFIG_HOTPLUG_CPU
 extern void unregister_cpu(struct cpu *cpu);
+extern void unregister_offsched(int cpuid);
 #endif
 struct notifier_block;

@@ -51,6 +52,9 @@ struct notifier_block;
 #ifdef CONFIG_HOTPLUG_CPU
 extern int register_cpu_notifier(struct notifier_block *nb);
 extern void unregister_cpu_notifier(struct notifier_block *nb);
+extern int register_offsched(void (*hotplug_cpu_dead)(void), int cpuid);
+extern int is_offsched(int cpuid);
+extern void run_offsched(void);
 #else
 
 #ifndef MODULE

@@ -60,11 +64,27 @@ static inline int register_cpu_notifier(struct notifier_block *nb)
 {
 	return 0;
 }
+
+static inline int register_offsched(void (*hotplug_cpu_dead)(void), int cpuid);
+{
+	return 0;
+}
 #endif
 
 static inline void unregister_cpu_notifier(struct notifier_block *nb)
 {
 }
+
+static inline void unregister_offsched(int cpuid)
+{
+}
+static inline int is_offsched(int cpuid)
+{
+	return 0;
+}
+static inline void run_offsched(void)
+{
+}
 #endif
 
 int cpu_up(unsigned int cpu);

include/linux/sched.h +1 -1

index b4c38bc..8039ee7 100644
@@ -1999,7 +1999,7 @@ struct task_struct *fork_idle(int);
 
 extern void set_task_comm(struct task_struct *tsk, char *from);
 extern char *get_task_comm(char *to, struct task_struct *tsk);
-
+extern rwlock_t *get_tasklist_lock(void);
 #ifdef CONFIG_SMP
 extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
 #else

kernel/cpu.c +1 -0

index 395b697..e67190d 100644
@@ -373,6 +373,7 @@ out:
 	cpu_maps_update_done();
 	return err;
 }
+EXPORT_SYMBOL(cpu_up);
 
 #ifdef CONFIG_PM_SLEEP_SMP
 static cpumask_var_t frozen_cpus;

kernel/fork.c +6 -0

index b9e2edd..4de4142 100644
@@ -85,6 +85,12 @@ __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
 
 DEFINE_TRACE(sched_process_fork);
 
+rwlock_t	*get_tasklist_lock(void)
+{
+	return &tasklist_lock;
+}
+EXPORT_SYMBOL_GPL(get_tasklist_lock);
+
 int nr_processes(void)
 {
 	int cpu;

Leon Woestenberg

2009-04-19 01:23 UTC #1 lore raw

Raz,

Quoted 3 lines · raz ben yehuda

offsched is a platform aimed to assign a service to an off-lined processor. Motivation is explained in:
http://sos-linux.svn.sourceforge.net/viewvc/sos-linux/offsched/trunk/Documentation/OFFSCHED.pdf

I quickly glanced over but could not find a comparison with other methods?

Suppose I run /bin/init with a certain CPU affinity (not covering all CPUs) and then run my own specific service on the remaining CPUs, how would this differ from your approach?

Quoted 1 line

From what I quickly gathered, you are running parts of the kernel

space on the second CPU set, or is that too simple a statement?

Looks like a very fresh approach, nice project!

Regards,

Signature · 1 line

Leon

raz ben yehuda

author

2009-04-19 07:09 UTC #2 lore raw

Quoted 7 lines · Leon Woestenberg

Raz,
Quoted 3 lines · raz ben yehuda
offsched is a platform aimed to assign a service to an off-lined processor. Motivation is explained in:
http://sos-linux.svn.sourceforge.net/viewvc/sos-linux/offsched/trunk/Documentation/OFFSCHED.pdf

Quoted 6 lines

I quickly glanced over but could not find a comparison with other methods?
Suppose I run /bin/init with a certain CPU affinity (not covering all CPUs) and then run my own specific service on the remaining CPUs, how would this differ from your approach?

First , not all services are children of init. there are workqueues, kernel daemons, softirqs and so on. offlet provides hard real time,in the sense that one can pre-determine how long an operation cost, it is NMI mode after all. Also, this is why i chose to implement a 1us timer, because i can do it.

Quoted 2 lines

Quoted 1 line
From what I quickly gathered, you are running parts of the kernel
space on the second CPU set, or is that too simple a statement?

I am running **no kernel** on the offloaded processors, but offsched code, and the rest of the processors run the "kernel".

Quoted 2 lines

Looks like a very fresh approach, nice project!

Thank you. i appreciate it.

Quoted 1 line

Regards,