sched: prepare for cmwq, take#2
Subsystems (6)
PATCH 1/1 sched: prepare for cmwq, take#2
Hello,
This is the second take of repare-for-cmwq patchset. Changes from the first take[L] are
* consult-online-mask-instead-of-active-in-select_fallback_rq and implement-__set_cpus_allowed replaced by updates to when cpu_active and cpuset configuration are updated as suggested by Peter.
* Patch description updated in 0004 as suggested by Peter.
0001-sched-define-and-use-CPU_PRI_-enums-for-cpu-notifier.patch 0002-sched-adjust-when-cpu_active-and-cpuset-configuratio.patch 0003-sched-refactor-try_to_wake_up.patch 0004-sched-add-hooks-for-workqueue.patch
This patchset is available in the following git tree.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git sched-wq
and contains the following changes.
include/linux/cpu.h | 25 ++++++ include/linux/perf_event.h | 2 include/linux/sched.h | 1 kernel/cpu.c | 6 - kernel/cpuset.c | 46 +++++++----- kernel/fork.c | 2 kernel/sched.c | 169 ++++++++++++++++++++++++++++++++++----------- kernel/workqueue_sched.h | 16 ++++ 8 files changed, 203 insertions(+), 64 deletions(-)
Thanks.
--
tejun
PATCH 1/4 sched: define and use CPU_PRI_* enums for cpu notifier priorities
Instead of hardcoding priority 10 and 20 in sched and perf, collect them into CPU_PRI_* enums.
--- include/linux/cpu.h | 9 +++++++++ include/linux/perf_event.h | 2 +- kernel/sched.c | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-)
include/linux/cpu.h +9 -0
@@ -48,6 +48,15 @@ extern ssize_t arch_cpu_release(const char *, size_t);#endif struct notifier_block ; +/* +* CPU notifier priorities. +*/ +enum { +/* migration should happen before other stuff but after perf */ +CPU_PRI_PERF =20 , +CPU_PRI_MIGRATION =10 , +}; +#ifdef CONFIG_SMP /* Need to know about CPUs going up/down? */ #if defined(CONFIG_HOTPLUG_CPU ) || !defined(MODULE )
include/linux/perf_event.h +1 -1
@@ -990,7 +990,7 @@ static inline void perf_event_disable(struct perf_event *event) { }#define perf_cpu_notifier (fn) \ do { \ static struct notifier_block fn##_nb __cpuinitdata = \ - { .notifier_call = fn, .priority = 20 }; \ + { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ (void *)(unsigned long)smp_processor_id()); \ fn(&fn##_nb, (unsigned long)CPU_STARTING, \
kernel/sched.c +1 -1
@@ -5837,7 +5837,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)*/ static struct notifier_block __cpuinitdata migration_notifier = { .notifier_call = migration_call, - .priority =10 + .priority =CPU_PRI_MIGRATION , };static int __init migration_init (void )
Signature · 1 line
1.6.4.2
PATCH 2/4 sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
In reply to Tejun HeoCurrently, when a cpu goes down, cpu_active is cleared before CPU_DOWN_PREPARE starts and cpuset configuration is updated from a default priority cpu notifier. When a cpu is coming up, it's set before CPU_ONLINE but cpuset configuration again is updated from the same cpu notifier.
For cpu notifiers, this presents an inconsistent state. Threads which a CPU_DOWN_PREPARE notifier expects to be bound to the CPU can be migrated to other cpus because the cpu is no more inactive.
Fix it by updating cpu_active in the highest priority cpu notifier and cpuset configuration in the second highest when a cpu is coming up. Down path is updated similarly. This guarantees that all other cpu notifiers see consistent cpu_active and cpuset configuration.
This problem is triggered by cmwq. During CPU_DOWN_PREPARE, hotplug callback creates a kthread and kthread_bind()s it to the target cpu, and the thread is expected to run on that cpu.
--- include/linux/cpu.h | 16 ++++++++++++++++ kernel/cpu.c | 6 ------ kernel/cpuset.c | 46 ++++++++++++++++++++++++++++------------------ kernel/sched.c | 31 ++++++++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 25 deletions(-)
include/linux/cpu.h +16 -0
@@ -52,6 +52,22 @@ struct notifier_block;* CPU notifier priorities. */ enum { +/* +* SCHED_ACTIVE marks a cpu which is coming up active during +* CPU_ONLINE and CPU_DOWN_FAILED and must be the first +* notifier. CPUSET_ACTIVE adjusts cpuset according to +* cpu_active mask right after SCHED_ACTIVE. During +* CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are +* ordered in the similar way. +* +* This ordering guarantees consistent cpu_active mask and +* migration behavior to all cpu notifiers. +*/ +CPU_PRI_SCHED_ACTIVE =INT_MAX , +CPU_PRI_CPUSET_ACTIVE =INT_MAX -1 , +CPU_PRI_SCHED_INACTIVE =INT_MIN +1 , +CPU_PRI_CPUSET_INACTIVE =INT_MIN , +/* migration should happen before other stuff but after perf */ CPU_PRI_PERF =20 ,CPU_PRI_MIGRATION =10 ,
kernel/cpu.c +0 -6
@@ -211,12 +211,9 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)return -EINVAL ;cpu_hotplug_begin (); -set_cpu_active (cpu, false); err =__raw_notifier_call_chain (&cpu_chain,CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls);if (err ==NOTIFY_BAD ) { -set_cpu_active (cpu, true); - nr_calls--;__raw_notifier_call_chain (&cpu_chain,CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL );
@@ -228,7 +225,6 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) { -set_cpu_active (cpu, true);/* CPU didn't die: tell everyone. Can't complain. */ if (raw_notifier_call_chain (&cpu_chain,CPU_DOWN_FAILED | mod, hcpu) ==NOTIFY_BAD )
@@ -309,8 +305,6 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) goto out_notify;BUG_ON (!cpu_online (cpu)); -set_cpu_active (cpu, true); -/* Now call notifier in preparation. */ raw_notifier_call_chain (&cpu_chain,CPU_ONLINE | mod, hcpu);
kernel/cpuset.c +28 -18
74 lines not on this page · raw message
kernel/sched.c +30 -1
@@ -5840,17 +5840,46 @@ static struct notifier_block __cpuinitdata migration_notifier = { .priority = CPU_PRI_MIGRATION, }; +static int __cpuinit sched_cpu_active (struct notifier_block *nfb, +unsigned long action,void *hcpu) +{ +switch (action & ~CPU_TASKS_FROZEN ) { +case CPU_ONLINE : +case CPU_DOWN_FAILED : +set_cpu_active ((long )hcpu, true); +return NOTIFY_OK ; +default : +return NOTIFY_DONE ; + } +} + +static int __cpuexit sched_cpu_inactive(struct notifier_block *nfb, +unsigned long action,void *hcpu) +{ +switch (action & ~CPU_TASKS_FROZEN ) { +case CPU_DOWN_PREPARE : +set_cpu_active ((long )hcpu, false); +return NOTIFY_OK ; +default : +return NOTIFY_DONE ; + } +} +static int __init migration_init (void ) {void *cpu = (void *)(long )smp_processor_id ();int err; -/* Start one for the boot CPU: */ +/* Initialize migration for the boot CPU */ err =migration_call (&migration_notifier,CPU_UP_PREPARE , cpu);BUG_ON (err ==NOTIFY_BAD );migration_call (&migration_notifier,CPU_ONLINE , cpu);register_cpu_notifier (&migration_notifier); +/* Register cpu active notifiers */ +cpu_notifier (sched_cpu_active,CPU_PRI_SCHED_ACTIVE ); +cpu_notifier (sched_cpu_inactive,CPU_PRI_SCHED_INACTIVE ); +return 0 ; }early_initcall (migration_init);
Signature · 1 line
1.6.4.2
Peter Zijlstra
maintainerRe: [PATCH 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Quoted 10 lines · Tejun Heo
Currently, when a cpu goes down,
cpu_activeis cleared beforeCPU_DOWN_PREPAREstarts and cpuset configuration is updated from a default priority cpu notifier. When a cpu is coming up, it's set beforeCPU_ONLINEbut cpuset configuration again is updated from the same cpu notifier.For cpu notifiers, this presents an inconsistent state. Threads which a
CPU_DOWN_PREPAREnotifier expects to be bound to the CPU can be migrated to other cpus because the cpu is no more inactive.
Quoted 8 lines
Fix it by updating
cpu_activein the highest priority cpu notifier and cpuset configuration in the second highest when a cpu is coming up. Down path is updated similarly. This guarantees that all other cpu notifiers see consistentcpu_activeand cpuset configuration.This problem is triggered by cmwq. During
CPU_DOWN_PREPARE, hotplug callback creates a kthread and kthread_bind()s it to the target cpu, and the thread is expected to run on that cpu.
I know we all love notifier lists, but doesn't the below code get lots more readable if we don't play tricks with notifier priorities and simply hardcode the few (perf/sched/cpuset) callbacks into the hotplug paths?
Also, I'm afraid you've now inverted the relation between cpu_active_mask and parition_sched_domains().
You need to first set/clear the active mask, then rebuild the domain. But with your patch parition_sched_domains() gets called in the regular DOWN_PREPARE path, while we only clear active at the very end, which means we build the wrong domains.
Re: [PATCH 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Hello,
Quoted 4 lines · Peter Zijlstra
I know we all love notifier lists, but doesn't the below code get lots more readable if we don't play tricks with notifier priorities and simply hardcode the few (perf/sched/cpuset) callbacks into the hotplug paths?
Maybe, maybe not. In this case, I kind of like that sleep failure case doesn't have to be explicitly rolled back but if you like hard coding that's fine too.
Quoted 7 lines
Also, I'm afraid you've now inverted the relation between
cpu_active_maskandparition_sched_domains().You need to first set/clear the active mask, then rebuild the domain. But with your patch
parition_sched_domains()gets called in the regularDOWN_PREPAREpath, while we only clear active at the very end, which means we build the wrong domains.
Ah, right forgot about that. So, the things that need to be ordered are cpu_active mask update, cpuset configuration and sched domains, right?
Thanks.
Signature · 1 line
tejun
Peter Zijlstra
maintainerRe: [PATCH 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Quoted 11 lines · Tejun Heo
Hello,
Quoted 4 lines · Peter Zijlstra
I know we all love notifier lists, but doesn't the below code get lots more readable if we don't play tricks with notifier priorities and simply hardcode the few (perf/sched/cpuset) callbacks into the hotplug paths?
Maybe, maybe not. In this case, I kind of like that sleep failure case doesn't have to be explicitly rolled back but if you like hard coding that's fine too.
Hurm, yeah that rollback might make things messy indeed.
Quoted 11 lines
Quoted 7 lines
Also, I'm afraid you've now inverted the relation between
cpu_active_maskandparition_sched_domains().You need to first set/clear the active mask, then rebuild the domain. But with your patch
parition_sched_domains()gets called in the regularDOWN_PREPAREpath, while we only clear active at the very end, which means we build the wrong domains.Ah, right forgot about that. So, the things that need to be ordered are
cpu_activemask update, cpuset configuration and sched domains, right?
Right, I think that should cover it.
PATCH 2/4 sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Currently, when a cpu goes down, cpu_active is cleared before CPU_DOWN_PREPARE starts and cpuset configuration is updated from a default priority cpu notifier. When a cpu is coming up, it's set before CPU_ONLINE but cpuset configuration again is updated from the same cpu notifier.
For cpu notifiers, this presents an inconsistent state. Threads which a CPU_DOWN_PREPARE notifier expects to be bound to the CPU can be migrated to other cpus because the cpu is no more inactive.
Fix it by updating cpu_active in the highest priority cpu notifier and cpuset configuration in the second highest when a cpu is coming up. Down path is updated similarly. This guarantees that all other cpu notifiers see consistent cpu_active and cpuset configuration.
cpuset_track_online_cpus() notifier is converted to cpuset_update_active_cpus() which just updates the configuration and now called from cpuset_cpu_[in]active() notifiers registered from sched_init_smp(). If cpuset is disabled, cpuset_update_active_cpus() degenerates into partition_sched_domains() making separate notifier for !CONFIG_CPUSETS unnecessary.
This problem is triggered by cmwq. During CPU_DOWN_PREPARE, hotplug callback creates a kthread and kthread_bind()s it to the target cpu, and the thread is expected to run on that cpu.
---
Okay, how about this one? Using notifiers seems better for the following reasons.
* Rollback on failure.
* cpuset/sched_domain don't expect to be called before smp configuration is complete. If hardcoded into cpu_up/down(), condition checks need to be added so that they're skipped if the system is bringing up the cpus for the first time.
Works fine here w/ CPUSET enabled and disabled.
Thanks.
include/linux/cpu.h | 16 +++++++++++ include/linux/cpuset.h | 6 ++++ kernel/cpu.c | 6 ---- kernel/cpuset.c | 21 +-------------- kernel/sched.c | 67 ++++++++++++++++++++++++++++++++++++------------- 5 files changed, 74 insertions(+), 42 deletions(-)
include/linux/cpu.h +16 -0
@@ -52,6 +52,22 @@ struct notifier_block;* CPU notifier priorities. */ enum { +/* +* SCHED_ACTIVE marks a cpu which is coming up active during +* CPU_ONLINE and CPU_DOWN_FAILED and must be the first +* notifier. CPUSET_ACTIVE adjusts cpuset according to +* cpu_active mask right after SCHED_ACTIVE. During +* CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are +* ordered in the similar way. +* +* This ordering guarantees consistent cpu_active mask and +* migration behavior to all cpu notifiers. +*/ +CPU_PRI_SCHED_ACTIVE =INT_MAX , +CPU_PRI_CPUSET_ACTIVE =INT_MAX -1 , +CPU_PRI_SCHED_INACTIVE =INT_MIN +1 , +CPU_PRI_CPUSET_INACTIVE =INT_MIN , +/* migration should happen before other stuff but after perf */ CPU_PRI_PERF =20 ,CPU_PRI_MIGRATION =10 ,
kernel/cpu.c +0 -6
@@ -211,12 +211,9 @@ static int __ref _cpu_down(unsigned intreturn -EINVAL ;cpu_hotplug_begin (); -set_cpu_active (cpu, false); err =__raw_notifier_call_chain (&cpu_chain,CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls);if (err ==NOTIFY_BAD ) { -set_cpu_active (cpu, true); - nr_calls--;__raw_notifier_call_chain (&cpu_chain,CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL );
@@ -228,7 +225,6 @@ static int __ref _cpu_down(unsigned int err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) { -set_cpu_active (cpu, true);/* CPU didn't die: tell everyone. Can't complain. */ if (raw_notifier_call_chain (&cpu_chain,CPU_DOWN_FAILED | mod, hcpu) ==NOTIFY_BAD )
@@ -309,8 +305,6 @@ static int __cpuinit _cpu_up(unsigned in goto out_notify;BUG_ON (!cpu_online (cpu)); -set_cpu_active (cpu, true); -/* Now call notifier in preparation. */ raw_notifier_call_chain (&cpu_chain,CPU_ONLINE | mod, hcpu);
kernel/cpuset.c +2 -19
@@ -2071,31 +2071,17 @@ static void scan_for_empty_cpusets(struc* but making no active use of cpusets. * * This routine ensures that top_cpuset.cpus_allowed tracks -* cpu_online_map on each CPU hotplug (cpuhp) event. +* cpu_active_mask on each CPU hotplug (cpuhp) event. * * Called within get_online_cpus(). Needs to call cgroup_lock() * before calling generate_sched_domains(). */ -static int cpuset_track_online_cpus (struct notifier_block *unused_nb, -unsigned long phase,void *unused_cpu) +void cpuset_update_active_cpus (void ) {struct sched_domain_attr *attr;cpumask_var_t *doms;int ndoms; -switch (phase) { -case CPU_ONLINE : -case CPU_ONLINE_FROZEN : -case CPU_DOWN_PREPARE : -case CPU_DOWN_PREPARE_FROZEN : -case CPU_DOWN_FAILED : -case CPU_DOWN_FAILED_FROZEN : -break ; - -default : -return NOTIFY_DONE ; - } -cgroup_lock ();mutex_lock (&callback_mutex);cpumask_copy (top_cpuset.cpus_allowed, cpu_active_mask);
@@ -2106,8 +2092,6 @@ static int cpuset_track_online_cpus(stru/* Have scheduler rebuild the domains */ partition_sched_domains (ndoms, doms, attr); - -return NOTIFY_OK ; }#ifdef CONFIG_MEMORY_HOTPLUG
@@ -2161,7 +2145,6 @@ void __init cpuset_init_smp(void)cpumask_copy (top_cpuset.cpus_allowed, cpu_active_mask); top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY ]; -hotcpu_notifier (cpuset_track_online_cpus,0 );hotplug_memory_notifier (cpuset_track_online_nodes,10 ); cpuset_wq =create_singlethread_workqueue ("cpuset" );
kernel/sched.c +50 -17
106 lines not on this page · raw message
include/linux/cpuset.h +6 -0
@@ -20,6 +20,7 @@ extern int number_of_cpusets; /* How manextern int cpuset_init (void );extern void cpuset_init_smp (void ); +extern void cpuset_update_active_cpus (void );extern void cpuset_cpus_allowed (struct task_struct *p,struct cpumask *mask);extern int cpuset_cpus_allowed_fallback (struct task_struct *p);extern nodemask_t cpuset_mems_allowed (struct task_struct *p);
@@ -96,6 +97,11 @@ static inline void set_mems_allowed(nodestatic inline int cpuset_init (void ) {return 0 ; }static inline void cpuset_init_smp (void ) {} +static inline void cpuset_update_active_cpus (void ) +{ +partition_sched_domains (1 ,NULL ,NULL ); +} +static inline void cpuset_cpus_allowed (struct task_struct *p,struct cpumask *mask) {
Re: [PATCH UPDATED 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Quoted 11 lines · Tejun Heo
Okay, how about this one? Using notifiers seems better for the following reasons.
* Rollback on failure.
* cpuset/sched_domain don't expect to be called before smp configuration is complete. If hardcoded into cpu_up/down(), condition checks need to be added so that they're skipped if the system is bringing up the cpus for the first time.
Works fine here w/ CPUSET enabled and disabled.
Ping. Peter.
Signature · 1 line
tejun
Peter Zijlstra
maintainerRe: [PATCH UPDATED 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
In reply to Tejun HeoQuoted 35 lines · Tejun Heo
Currently, when a cpu goes down,
cpu_activeis cleared beforeCPU_DOWN_PREPAREstarts and cpuset configuration is updated from a default priority cpu notifier. When a cpu is coming up, it's set beforeCPU_ONLINEbut cpuset configuration again is updated from the same cpu notifier.For cpu notifiers, this presents an inconsistent state. Threads which a
CPU_DOWN_PREPAREnotifier expects to be bound to the CPU can be migrated to other cpus because the cpu is no more inactive.Fix it by updating
cpu_activein the highest priority cpu notifier and cpuset configuration in the second highest when a cpu is coming up. Down path is updated similarly. This guarantees that all other cpu notifiers see consistentcpu_activeand cpuset configuration.
cpuset_track_online_cpus()notifier is converted tocpuset_update_active_cpus()which just updates the configuration and now called from cpuset_cpu_[in]active() notifiers registered fromsched_init_smp(). If cpuset is disabled,cpuset_update_active_cpus()degenerates intopartition_sched_domains()making separate notifier for !CONFIG_CPUSETS unnecessary.This problem is triggered by cmwq. During
CPU_DOWN_PREPARE, hotplug callback creates a kthread and kthread_bind()s it to the target cpu, and the thread is expected to run on that cpu.Signed-off-by Tejun Heo <[email protected]>Cc Rusty Russell <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Paul Menage <[email protected]>---
Okay, how about this one? Using notifiers seems better for the following reasons.
Quoted 8 lines
* Rollback on failure.
* cpuset/sched_domain don't expect to be called before smp configuration is complete. If hardcoded into cpu_up/down(), condition checks need to be added so that they're skipped if the system is bringing up the cpus for the first time.
Works fine here w/ CPUSET enabled and disabled.
OK, this looks good.
Thanks Tejun.
Re: [PATCH UPDATED 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Quoted 10 lines · Peter Zijlstra
Quoted 8 lines
* Rollback on failure.
* cpuset/sched_domain don't expect to be called before smp configuration is complete. If hardcoded into cpu_up/down(), condition checks need to be added so that they're skipped if the system is bringing up the cpus for the first time.
Works fine here w/ CPUSET enabled and disabled.
OK, this looks good.
Cool, can I add your acked-by's to and send them towards Ingo?
Thanks.
Signature · 1 line
tejun
Peter Zijlstra
maintainerRe: [PATCH UPDATED 2/4] sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
Quoted 8 lines · Tejun Heo
Quoted 7 lines · Peter Zijlstra
Quoted 7 lines
* Rollback on failure.
* cpuset/sched_domain don't expect to be called before smp configuration is complete. If hardcoded into cpu_up/down(), condition checks need to be added so that they're skipped if the system is bringing up the cpus for the first time.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Works fine here w/ CPUSET enabled and disabled.
OK, this looks good.
Cool, can I add your acked-by's to and send them towards Ingo?
Yes.
PATCH 3/4 sched: refactor try_to_wake_up()
In reply to Tejun HeoFactor ttwu_activate() and ttwu_woken_up() out of try_to_wake_up(). The factoring out doesn't affect try_to_wake_up() much code-generation-wise. Depending on configuration options, it ends up generating the same object code as before or slightly different one due to different register assignment.
This is to help future implementation of try_to_wake_up_local().
Mike Galbraith suggested rename to ttwu_post_activation() from ttwu_woken_up() and comment update in try_to_wake_up().
--- kernel/sched.c | 83 +++++++++++++++++++++++++++++++++----------------------- 1 files changed, 49 insertions(+), 34 deletions(-)
kernel/sched.c +49 -34
@@ -2321,11 +2321,52 @@ static void update_avg(u64 *avg, u64 sample) }#endif -/*** +static inline void ttwu_activate (struct task_struct *p,struct rq *rq, +bool is_sync,bool is_migrate,bool is_local, +unsigned long en_flags) +{ +schedstat_inc (p, se.statistics.nr_wakeups); +if (is_sync) +schedstat_inc (p, se.statistics.nr_wakeups_sync); +if (is_migrate) +schedstat_inc (p, se.statistics.nr_wakeups_migrate); +if (is_local) +schedstat_inc (p, se.statistics.nr_wakeups_local); +else +schedstat_inc (p, se.statistics.nr_wakeups_remote); + +activate_task (rq, p, en_flags); +} + +static inline void ttwu_post_activation (struct task_struct *p,struct rq *rq, +int wake_flags,bool success) +{ +trace_sched_wakeup (p, success); +check_preempt_curr (rq, p, wake_flags); + + p->state =TASK_RUNNING ; +#ifdef CONFIG_SMP +if (p->sched_class->task_woken) + p->sched_class->task_woken (rq, p); + +if (unlikely (rq->idle_stamp)) { +u64 delta = rq->clock - rq->idle_stamp; +u64 max =2 *sysctl_sched_migration_cost; + +if (delta > max) + rq->avg_idle = max; +else +update_avg (&rq->avg_idle, delta); + rq->idle_stamp =0 ; + } +#endif +} + +/** * try_to_wake_up - wake up a thread -* @p: the to-be-woken-up thread +* @p: the thread to be awakened * @state: the mask of task states that can be woken -* @sync: do a synchronous wakeup? +* @wake_flags: wake modifier flags (WF_*) * * Put it on the run-queue if it's not already there. The "current" * thread is always on the run-queue (except when the actual
@@ -2333,7 +2374,8 @@ static void update_avg(u64 *avg, u64 sample)* the simpler "current->state = TASK_RUNNING" to mark yourself * runnable without the overhead of this. * -* returns failure only if the task is already active. +* Returns %true if @p was woken up, %false if it was already running +* or @state didn't match @p's state. */ static int try_to_wake_up (struct task_struct *p,unsigned int state,int wake_flags)
@@ -2413,38 +2455,11 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, out_activate:#endif /* CONFIG_SMP */ -schedstat_inc (p, se.statistics.nr_wakeups); -if (wake_flags &WF_SYNC ) -schedstat_inc (p, se.statistics.nr_wakeups_sync); -if (orig_cpu != cpu) -schedstat_inc (p, se.statistics.nr_wakeups_migrate); -if (cpu == this_cpu) -schedstat_inc (p, se.statistics.nr_wakeups_local); -else -schedstat_inc (p, se.statistics.nr_wakeups_remote); -activate_task (rq, p, en_flags); +ttwu_activate (p, rq, wake_flags &WF_SYNC , orig_cpu != cpu, + cpu == this_cpu, en_flags); success =1 ; - out_running: -trace_sched_wakeup (p, success); -check_preempt_curr (rq, p, wake_flags); - - p->state =TASK_RUNNING ; -#ifdef CONFIG_SMP -if (p->sched_class->task_woken) - p->sched_class->task_woken (rq, p); - -if (unlikely (rq->idle_stamp)) { -u64 delta = rq->clock - rq->idle_stamp; -u64 max =2 *sysctl_sched_migration_cost; - -if (delta > max) - rq->avg_idle = max; -else -update_avg (&rq->avg_idle, delta); - rq->idle_stamp =0 ; - } -#endif +ttwu_post_activation (p, rq, wake_flags, success); out:task_rq_unlock (rq, &flags);put_cpu ();
Signature · 1 line
1.6.4.2
PATCH 4/4 sched: add hooks for workqueue
In reply to Tejun HeoConcurrency managed workqueue needs to know when workers are going to sleep and waking up. Using these two hooks, cmwq keeps track of the current concurrency level and throttles execution of new works if it's too high and wakes up another worker from the sleep hook if it becomes too low.
This patch introduces PF_WQ_WORKER to identify workqueue workers and adds the following two hooks.
* wq_worker_waking_up(): called when a worker is woken up.
* wq_worker_sleeping(): called when a worker is going to sleep and may return a pointer to a local task which should be woken up. The returned task is woken up using try_to_wake_up_local() which is simplified ttwu which is called under rq lock and can only wake up local tasks.
Both hooks are currently defined as noop in kernel/workqueue_sched.h. Later cmwq implementation will replace them with proper implementation.
These hooks are hard coded as they'll always be enabled.
--- include/linux/sched.h | 1 + kernel/fork.c | 2 +- kernel/sched.c | 53 ++++++++++++++++++++++++++++++++++++++++++++- kernel/workqueue_sched.h | 16 +++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 kernel/workqueue_sched.h
include/linux/sched.h +1 -0
@@ -1702,6 +1702,7 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *#define PF_EXITING 0x00000004/* getting shut down */ #define PF_EXITPIDONE 0x00000008/* pi exit done on shut down */ #define PF_VCPU 0x00000010/* I'm a virtual CPU */ +#define PF_WQ_WORKER 0x00000020/* I'm a workqueue worker */ #define PF_FORKNOEXEC 0x00000040/* forked but didn't exec */ #define PF_MCE_PROCESS 0x00000080/* process policy on mce errors */ #define PF_SUPERPRIV 0x00000100/* used super-user privileges */
kernel/fork.c +1 -1
@@ -900,7 +900,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p) {unsigned long new_flags = p->flags; - new_flags &= ~PF_SUPERPRIV ; + new_flags &= ~(PF_SUPERPRIV |PF_WQ_WORKER ); new_flags |=PF_FORKNOEXEC ; new_flags |=PF_STARTING ; p->flags = new_flags;
kernel/sched.c +51 -2
79 lines not on this page · raw message
kernel/workqueue_sched.h new file +16 -0
@@ -0,0 +1,16 @@ +/* +* kernel/workqueue_sched.h +* +* Scheduler hooks for concurrency managed workqueue. Only to be +* included from sched.c and workqueue.c. +*/ +static inline void wq_worker_waking_up (struct task_struct *task, +unsigned int cpu) +{ +} + +static inline struct task_struct *wq_worker_sleeping (struct task_struct *task, +unsigned int cpu) +{ +return NULL ; +}
Signature · 1 line
1.6.4.2