select idle cpu from idle cpumask in sched domain
cover select idle cpu from idle cpumask in sched domain
I'm writting to see if it makes sense to track idle cpus in a shared cpumask in sched domain, then a task wakes up it can select idle cpu from this cpumask instead of scanning all the cpus in the last level cache domain, especially when the system is heavily loaded, the scanning cost could be significantly reduced. The price is that the atomic cpumask ops are added to the idle entry and exit paths.
I tested the following benchmarks on a x86 4 socket system with 24 cores per socket and 2 hyperthreads per core, total 192 CPUs:
uperf throughput: netperf workload, tcp_nodelay, r/w size = 90
threads baseline-avg %std patch-avg %std
96 1 1.24 0.98 2.76
144 1 1.13 1.35 4.01
192 1 0.58 1.67 3.25
240 1 2.49 1.68 3.55
hackbench: process mode, 100000 loops, 40 file descriptors per group
group baseline-avg %std patch-avg %std
2(80) 1 12.05 0.97 9.88
3(120) 1 12.48 0.95 11.62
4(160) 1 13.83 0.97 13.22
5(200) 1 2.76 1.01 2.94
schbench: 99th percentile latency, 16 workers per message thread
mthread baseline-avg %std patch-avg %std
6(96) 1 1.24 0.993 1.73
9(144) 1 0.38 0.998 0.39
12(192) 1 1.58 0.995 1.64
15(240) 1 51.71 0.606 37.41
sysbench mysql throughput: read/write, table size = 10,000,000
thread baseline-avg %std patch-avg %std
96 1 1.77 1.015 1.71
144 1 3.39 0.998 4.05
192 1 2.88 1.002 2.81
240 1 2.07 1.011 2.09
kbuild: kexec reboot every time
baseline-avg patch-avg
1 1
Any suggestions are highly appreciated!
Thanks,
-Aubrey
Aubrey Li (1):
sched/fair: select idle cpu from idle cpumask in sched domain
include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)
Signature · 1 line
2.25.1
RFC PATCH 1/1 sched/fair: select idle cpu from idle cpumask in sched domain
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)
include/linux/sched/topology.h +13 -0
@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */
kernel/sched/fair.c +3 -1
@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)
@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }
@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }
kernel/sched/topology.c +1 -1
@@ -1769,7 +1769,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr (sdd->sd, j) = sd; - sds =kzalloc_node (sizeof (struct sched_domain_shared ), + sds =kzalloc_node (sizeof (struct sched_domain_shared ) +cpumask_size (),GFP_KERNEL ,cpu_to_node (j));if (!sds)return -ENOMEM ;
Signature · 1 line
2.25.1
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 21 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +0
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 10 lines
include/linux/sched/topology.h +8
-0atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];
Can't you use cpumask_var_t and zalloc_cpumask_var() instead?
The patch looks useful. Did it help you with any particular workload? It'd be good to expand on that in the commit message.
Thanks
--
Qais Yousef
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 26 lines · Qais Yousef
Quoted 25 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +2
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain.
Quoted 8 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 6 lines
include/linux/sched/topology.h +6
-0+* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?
I can use the existing free code. Do we have a problem of this?
Quoted 4 lines
The patch looks useful. Did it help you with any particular workload? It'd be good to expand on that in the commit message.
Odd, that included in patch v1 0/1, did you receive it?
Thanks,
-Aubrey
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 36 lines · Aubrey
Quoted 26 lines · Qais Yousef
Quoted 25 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +2
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. Quoted 8 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 6 lines
include/linux/sched/topology.h +6
-0+* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?
Quoted 7 lines
I can use the existing free code. Do we have a problem of this?
Quoted 4 lines
The patch looks useful. Did it help you with any particular workload? It'd be good to expand on that in the commit message.
Odd, that included in patch v1 0/1, did you receive it?
I found it at here:
https://lkml.org/lkml/2020/9/11/645
Quoted 4 lines
Thanks,
-Aubrey
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
In reply to Li, AubreyQuoted 16 lines · Aubrey
Quoted 7 lines
Quoted 7 lines
+++
b/include/linux/sched/topology.hinclude/linux/sched/topology.h +2
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. Quoted 8 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 6 lines
include/linux/sched/topology.h +6
-0+* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?
Quoted 2 lines
I can use the existing free code. Do we have a problem of this?
Nah, flexible array members are the preferred approach here; this also means we don't let CONFIG_CPUMASK_OFFSTACK dictate where this gets allocated.
See struct numa_group, struct sched_group, struct sched_domain, struct em_perf_domain...
Quoted 8 lines
Quoted 4 lines
The patch looks useful. Did it help you with any particular workload? It'd be good to expand on that in the commit message.
Odd, that included in patch v1 0/1, did you receive it?
Thanks,
-Aubrey
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 18 lines · Valentin Schneider
Quoted 16 lines · Aubrey
Quoted 15 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 13 lines
+++
b/include/linux/sched/topology.hinclude/linux/sched/topology.h +8
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?
Quoted 4 lines
Quoted 2 lines
I can use the existing free code. Do we have a problem of this?
Nah, flexible array members are the preferred approach here; this also
Is this your opinion or a rule written somewhere I missed?
Quoted 5 lines
means we don't let
CONFIG_CPUMASK_OFFSTACKdictate where this gets allocated.See struct
numa_group, structsched_group, structsched_domain, structem_perf_domain...
struct root_domain, struct cpupri_vec, struct generic_pm_domain, struct irq_common_data..
Use cpumask_var_t.
Both approach look correct to me, so no objection in principle. cpumask_var_t looks neater IMO and will be necessary once more than one cpumask are required in a struct.
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
The patch looks useful. Did it help you with any particular workload? It'd be good to expand on that in the commit message.
Odd, that included in patch v1 0/1, did you receive it?
Aubrey,
Sorry I didn't see that no. It's important justification to be part of the commit message, I think worth adding it.
Thanks
--
Qais Yousef
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 19 lines · Qais Yousef
Quoted 18 lines · Valentin Schneider
Quoted 16 lines · Aubrey
Quoted 15 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 13 lines
+++
b/include/linux/sched/topology.hinclude/linux/sched/topology.h +8
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
I can use the existing free code. Do we have a problem of this?
Nah, flexible array members are the preferred approach here; this also
Is this your opinion or a rule written somewhere I missed?
I don't think there's a written rule, but AIUI it is preferred by at least Peter:
https://lore.kernel.org/linux-pm/[email protected]/
https://lore.kernel.org/lkml/[email protected]/
And my opinion is that, if you can, having fewer separate allocation is better.
Quoted 7 lines
Quoted 6 lines
means we don't let
CONFIG_CPUMASK_OFFSTACKdictate where this gets allocated.See struct
numa_group, structsched_group, structsched_domain, structem_perf_domain...
Quoted 9 lines
struct
root_domain, structcpupri_vec, structgeneric_pm_domain, structirq_common_data..Use
cpumask_var_t.Both approach look correct to me, so no objection in principle.
cpumask_var_tlooks neater IMO and will be necessary once more than one cpumask are required in a struct.
You're right in that cpumask_var_t becomes necessary when you need more than one mask. For those that use it despite requiring only one mask (cpupri stuff, struct nohz too), I'm not sure.
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 28 lines · Valentin Schneider
Quoted 19 lines · Qais Yousef
Quoted 18 lines · Valentin Schneider
Quoted 16 lines · Aubrey
Quoted 15 lines
include/linux/sched/topology.h struct sched_domain_shared { · line 75
Quoted 13 lines
+++
b/include/linux/sched/topology.hinclude/linux/sched/topology.h +8
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[];Can't you use
cpumask_var_tandzalloc_cpumask_var()instead?Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
I can use the existing free code. Do we have a problem of this?
Nah, flexible array members are the preferred approach here; this also
Is this your opinion or a rule written somewhere I missed?
Quoted 7 lines
I don't think there's a written rule, but AIUI it is preferred by at least Peter:
https://lore.kernel.org/linux-pm/[email protected]/
https://lore.kernel.org/lkml/[email protected]/And my opinion is that, if you can, having fewer separate allocation is better.
+1
Quoted 13 lines
Quoted 13 lines
Quoted 7 lines
means we don't let
CONFIG_CPUMASK_OFFSTACKdictate where this gets allocated.See struct
numa_group, structsched_group, structsched_domain, structem_perf_domain...struct
root_domain, structcpupri_vec, structgeneric_pm_domain, structirq_common_data..Use
cpumask_var_t.
Quoted 8 lines
Quoted 4 lines
Both approach look correct to me, so no objection in principle.
cpumask_var_tlooks neater IMO and will be necessary once more than one cpumask are required in a struct.You're right in that
cpumask_var_tbecomes necessary when you need more than one mask. For those that use it despite requiring only one mask (cpupri stuff, struct nohz too), I'm not sure.
I don't have a strong opinoin. cpumask_var_t is more readable and maintainble IMO. But it's not a big deal. Any form can be easily changed.
Thanks
--
Qais Yousef
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
In reply to Aubrey LiHi, Aubrey
Quoted 47 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 5 lines
kernel/sched/fair.c +1
-1@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);
Is the sds_idle_cpus() always empty if nohz=off?
Do we need to initialize the idle_cpus_span with sched_domain_span(sd)?
Quoted 12 lines
kernel/sched/fair.c +1
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }
kernel/sched/fair.c static void set_cpu_sd_state_idle(int cpu) · line 10216
Quoted 4 lines
kernel/sched/fair.c +1
-0@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared));
This only works when entering/exiting tickless mode? :) Why not update idle_cpus_span during tick_nohz_idle_enter()/exit()?
Thx.
Regards,
Jiang
Quoted 19 lines
kernel/sched/fair.c +0
-0unlock:rcu_read_unlock (); }kernel/sched/topology.c +1
-2@@ -1769,7 +1769,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr (sdd->sd, j) = sd; - sds =kzalloc_node (sizeof (struct sched_domain_shared ), + sds =kzalloc_node (sizeof (struct sched_domain_shared ) +cpumask_size (),GFP_KERNEL ,cpu_to_node (j));if (!sds)return -ENOMEM ; --2.25.1
Vincent Guittot
maintainerRe: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 52 lines · Jiang Biao
Hi, Aubrey
Quoted 48 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
Quoted 5 lines
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 4 lines
kernel/sched/fair.c +1
-1time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);Is the
sds_idle_cpus()always empty if nohz=off?
Good point
Quoted 16 lines
Do we need to initialize the
idle_cpus_spanwithsched_domain_span(sd)?Quoted 14 lines
kernel/sched/fair.c +1
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;
Quoted 4 lines
kernel/sched/fair.c static void set_cpu_sd_state_idle(int cpu) · line 10216
Quoted 2 lines
kernel/sched/fair.c +1
-0atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared));This only works when entering/exiting tickless mode? :) Why not update
idle_cpus_spanduring tick_nohz_idle_enter()/exit()?
set_cpu_sd_state_busy is only called during a tick in order to limit the rate of the update to once per tick per cpu at most and prevents any kind of storm of update if short running tasks wake/sleep all the time. We don't want to update a cpumask at each and every enter/leave idle.
Quoted 24 lines
Thx.
Regards,
JiangQuoted 19 lines
kernel/sched/fair.c +0
-0unlock:rcu_read_unlock (); }kernel/sched/topology.c +1
-2@@ -1769,7 +1769,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr (sdd->sd, j) = sd; - sds =kzalloc_node (sizeof (struct sched_domain_shared ), + sds =kzalloc_node (sizeof (struct sched_domain_shared ) +cpumask_size (),GFP_KERNEL ,cpu_to_node (j));if (!sds)return -ENOMEM ; --2.25.1
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Hi, Vincent
Quoted 83 lines · Vincent Guittot
Quoted 52 lines · Jiang Biao
Hi, Aubrey
Quoted 48 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int tQuoted 5 lines
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 4 lines
kernel/sched/fair.c +1
-1time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);Is the
sds_idle_cpus()always empty if nohz=off?Good point
Quoted 16 lines
Do we need to initialize the
idle_cpus_spanwithsched_domain_span(sd)?Quoted 14 lines
kernel/sched/fair.c +1
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;Quoted 4 lines
kernel/sched/fair.c static void set_cpu_sd_state_idle(int cpu) · line 10216
Quoted 2 lines
kernel/sched/fair.c +1
-0atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared));This only works when entering/exiting tickless mode? :) Why not update
idle_cpus_spanduring tick_nohz_idle_enter()/exit()?
Quoted 6 lines
set_cpu_sd_state_busyis only called during a tick in order to limit the rate of the update to once per tick per cpu at most and prevents any kind of storm of update if short running tasks wake/sleep all the time. We don't want to update a cpumask at each and every enter/leave idle.
Agree. But set_cpu_sd_state_busy seems not being reached when nohz=off, which means it will not work for that case? :)
Thx.
Regards,
Jiang
Vincent Guittot
maintainerRe: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 88 lines · Jiang Biao
Hi, Vincent
Quoted 83 lines · Vincent Guittot
Quoted 52 lines · Jiang Biao
Hi, Aubrey
Quoted 48 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int tQuoted 5 lines
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 4 lines
kernel/sched/fair.c +1
-1time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);Is the
sds_idle_cpus()always empty if nohz=off?Good point
Quoted 20 lines
Do we need to initialize the
idle_cpus_spanwithsched_domain_span(sd)?Quoted 12 lines
kernel/sched/fair.c +1
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }kernel/sched/fair.c static void set_cpu_sd_state_idle(int cpu) · line 10216
Quoted 4 lines
kernel/sched/fair.c +1
-0@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared));This only works when entering/exiting tickless mode? :) Why not update
idle_cpus_spanduring tick_nohz_idle_enter()/exit()?
Quoted 8 lines
Quoted 6 lines
set_cpu_sd_state_busyis only called during a tick in order to limit the rate of the update to once per tick per cpu at most and prevents any kind of storm of update if short running tasks wake/sleep all the time. We don't want to update a cpumask at each and every enter/leave idle.Agree. But
set_cpu_sd_state_busyseems not being reached when nohz=off, which means it will not work for that case? :)
Yes set_cpu_sd_state_idle/busy are nohz function
Quoted 4 lines
Thx.
Regards,
Jiang
Re: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
Quoted 95 lines · Vincent Guittot
Quoted 94 lines · Jiang Biao
Hi, Vincent
Quoted 89 lines · Vincent Guittot
Quoted 52 lines · Jiang Biao
Hi, Aubrey
Quoted 48 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int tQuoted 5 lines
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 4 lines
kernel/sched/fair.c +1
-1time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);Is the
sds_idle_cpus()always empty if nohz=off?Good point
Quoted 16 lines
Do we need to initialize the
idle_cpus_spanwithsched_domain_span(sd)?Quoted 14 lines
kernel/sched/fair.c +1
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;Quoted 4 lines
kernel/sched/fair.c static void set_cpu_sd_state_idle(int cpu) · line 10216
Quoted 2 lines
kernel/sched/fair.c +1
-0atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared));This only works when entering/exiting tickless mode? :) Why not update
idle_cpus_spanduring tick_nohz_idle_enter()/exit()?
set_cpu_sd_state_busyis only called during a tick in order to limit the rate of the update to once per tick per cpu at most and prevents any kind of storm of update if short running tasks wake/sleep all the time. We don't want to update a cpumask at each and every enter/leave idle.
Quoted 4 lines
Quoted 2 lines
Agree. But
set_cpu_sd_state_busyseems not being reached when nohz=off, which means it will not work for that case? :)Yes set_cpu_sd_state_idle/busy are nohz function
Thanks Biao to point this out.
If the shared idle cpumask is initialized with sched_domain_span(sd), then nohz=off case will remain the previous behavior.
Thanks,
-Aubrey
Vincent Guittot
maintainerRe: [RFC PATCH v1 1/1] sched/fair: select idle cpu from idle cpumask in sched domain
In reply to Aubrey LiQuoted 47 lines · Aubrey Li
Added idle cpumask to track idle cpus in sched domain. When a CPU enters idle, its corresponding bit in the idle cpumask will be set, and when the CPU exits idle, its bit will be cleared.
When a task wakes up to select an idle cpu, scanning idle cpumask has low cost than scanning all the cpus in last level cache domain, especially when the system is heavily loaded.
Signed-off-by Aubrey Li <[email protected]>--- include/linux/sched/topology.h | 13 +++++++++++++ kernel/sched/fair.c | 4 +++- kernel/sched/topology.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-)include/linux/sched/topology.h +13
-0@@ -65,8 +65,21 @@ struct sched_domain_shared {atomic_t ref;atomic_t nr_busy_cpus;int has_idle_cores; +/* +* Span of all idle CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +*/ +unsigned long idle_cpus_span[]; }; +static inline struct cpumask *sds_idle_cpus (struct sched_domain_shared *sds) +{ +return to_cpumask (sds->idle_cpus_span); +} +struct sched_domain {/* These fields must be setup */ struct sched_domain __rcu *parent;/* top domain must be null terminated */ kernel/sched/fair.c +0
-0
kernel/sched/fair.c static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · line 6139
Quoted 5 lines
kernel/sched/fair.c +1
-1@@ -6136,7 +6136,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time =cpu_clock (this); -cpumask_and (cpus,sched_domain_span (sd), p->cpus_ptr); +cpumask_and (cpus,sds_idle_cpus (sd->shared), p->cpus_ptr);
sched_domain_shared is set only at shared cache level so this works only because select_idle_cpu is called with sd_llc. It's worth adding a comment to describe this.
Quoted 35 lines
kernel/sched/fair.c +2
-0for_each_cpu_wrap (cpu, cpus, target) {if (!--nr)@@ -10182,6 +10182,7 @@ static void set_cpu_sd_state_busy(int cpu) sd->nohz_idle =0 ;atomic_inc (&sd->shared->nr_busy_cpus); +cpumask_clear_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }@@ -10212,6 +10213,7 @@ static void set_cpu_sd_state_idle(int cpu) sd->nohz_idle =1 ;atomic_dec (&sd->shared->nr_busy_cpus); +cpumask_set_cpu (cpu,sds_idle_cpus (sd->shared)); unlock:rcu_read_unlock (); }kernel/sched/topology.c +1
-2@@ -1769,7 +1769,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr (sdd->sd, j) = sd; - sds =kzalloc_node (sizeof (struct sched_domain_shared ), + sds =kzalloc_node (sizeof (struct sched_domain_shared ) +cpumask_size (),GFP_KERNEL ,cpu_to_node (j));if (!sds)return -ENOMEM ; --2.25.1