sched: Skip queued wakeups only when L2 is shared
Subsystems (2)
v3 cover sched: Skip queued wakeups only when L2 is shared
This series improves performance of scheduler wakeups on large systems by skipping queued wakeups only when CPUs share their L2 cache, rather than when they share their LLC.
The speedup mainly reproduces on workloads which have at least *some* idle time (because it significantly increases the number of migrations, and thus remote wakeups), *and* it needs to have a sufficient load to cause contention on the runqueue locks.
Feedback is welcome,
Thanks,
Mathieu
Mathieu Desnoyers (3):
sched: Rename cpus_share_cache to cpus_share_llc
sched: Introduce cpus_share_l2c (v3)
sched: ttwu_queue_cond: skip queued wakeups across different l2 caches
block/blk-mq.c | 2 +- include/linux/sched/topology.h | 10 ++++++++-- kernel/sched/core.c | 14 +++++++++++--- kernel/sched/fair.c | 8 ++++---- kernel/sched/sched.h | 2 ++ kernel/sched/topology.c | 32 +++++++++++++++++++++++++++++--- 6 files changed, 55 insertions(+), 13 deletions(-)
Signature · 1 line
2.39.2
RFC PATCH v3 1/3 sched: Rename cpus_share_cache to cpus_share_llc
In preparation for introducing cpus_share_l2c, rename cpus_share_cache to cpus_share_llc, to make it clear that it specifically groups CPUs by LLC.
--- block/blk-mq.c | 2 +- include/linux/sched/topology.h | 4 ++-- kernel/sched/core.c | 4 ++-- kernel/sched/fair.c | 8 ++++---- kernel/sched/topology.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-)
block/blk-mq.c +1 -1
@@ -1136,7 +1136,7 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq)/* same CPU or cache domain? Complete locally */ if (cpu == rq->mq_ctx->cpu || (!test_bit (QUEUE_FLAG_SAME_FORCE , &rq->q->queue_flags) && -cpus_share_cache (cpu, rq->mq_ctx->cpu))) +cpus_share_llc (cpu, rq->mq_ctx->cpu)))return false;/* don't try to IPI to an offline CPU */
include/linux/sched/topology.h +2 -2
@@ -178,7 +178,7 @@ extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],cpumask_var_t *alloc_sched_domains (unsigned int ndoms);void free_sched_domains (cpumask_var_t doms[],unsigned int ndoms); -bool cpus_share_cache (int this_cpu,int that_cpu); +bool cpus_share_llc (int this_cpu,int that_cpu);typedef const struct cpumask *(*sched_domain_mask_f )(int cpu);typedef int (*sched_domain_flags_f )(void );
@@ -227,7 +227,7 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], { } -static inline bool cpus_share_cache (int this_cpu,int that_cpu) +static inline bool cpus_share_llc (int this_cpu,int that_cpu) {return true; }
kernel/sched/core.c +2 -2
@@ -3904,7 +3904,7 @@ void wake_up_if_idle(int cpu)rcu_read_unlock (); } -bool cpus_share_cache (int this_cpu,int that_cpu) +bool cpus_share_llc (int this_cpu,int that_cpu) {if (this_cpu == that_cpu)return true;
@@ -3929,7 +3929,7 @@ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)* If the CPU does not share cache, then queue the task on the * remote rqs wakelist to avoid accessing remote data. */ -if (!cpus_share_cache (smp_processor_id (), cpu)) +if (!cpus_share_llc (smp_processor_id (), cpu))return true;if (cpu ==smp_processor_id ())
kernel/sched/fair.c +4 -4
@@ -6626,7 +6626,7 @@ wake_affine_idle(int this_cpu, int prev_cpu, int sync)* a cpufreq perspective, it's better to have higher utilisation * on one CPU. */ -if (available_idle_cpu (this_cpu) &&cpus_share_cache (this_cpu, prev_cpu)) +if (available_idle_cpu (this_cpu) &&cpus_share_llc (this_cpu, prev_cpu))return available_idle_cpu (prev_cpu) ? prev_cpu : this_cpu;if (sync &&cpu_rq (this_cpu)->nr_running ==1 )
@@ -7146,7 +7146,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)/* * If the previous CPU is cache affine and idle, don't be stupid: */ -if (prev != target &&cpus_share_cache (prev, target) && +if (prev != target &&cpus_share_llc (prev, target) && (available_idle_cpu (prev) ||sched_idle_cpu (prev)) &&asym_fits_cpu (task_util, util_min, util_max, prev))return prev;
@@ -7172,7 +7172,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) p->recent_used_cpu = prev;if (recent_used_cpu != prev && recent_used_cpu != target && -cpus_share_cache (recent_used_cpu, target) && +cpus_share_llc (recent_used_cpu, target) && (available_idle_cpu (recent_used_cpu) ||sched_idle_cpu (recent_used_cpu)) &&cpumask_test_cpu (p->recent_used_cpu, p->cpus_ptr) &&asym_fits_cpu (task_util, util_min, util_max, recent_used_cpu)) {
@@ -7206,7 +7206,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)if (sched_smt_active ()) { has_idle_core =test_idle_cores (target); -if (!has_idle_core &&cpus_share_cache (prev, target)) { +if (!has_idle_core &&cpus_share_llc (prev, target)) { i =select_idle_smt (p, prev);if ((unsigned int )i < nr_cpumask_bits)return i;
kernel/sched/topology.c +1 -1
@@ -661,7 +661,7 @@ static void destroy_sched_domains(struct sched_domain *sd)* * Also keep a unique ID per domain (we use the first CPU number in * the cpumask of the domain), this allows us to quickly tell if -* two CPUs are in the same cache domain, see cpus_share_cache(). +* two CPUs are in the same cache domain, see cpus_share_llc(). */ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);DEFINE_PER_CPU (int, sd_llc_size);
Signature · 1 line
2.39.2
RFC PATCH v3 2/3 sched: Introduce cpus_share_l2c
In reply to Mathieu DesnoyersIntroduce cpus_share_l2c to allow querying whether two logical CPUs share a common L2 cache.
Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
---
Changes since v1:
- Fix l2c id for configurations where L2 have a single logical CPU:
use TOPOLOGY_CLUSTER_SYSFS to find out whether topology cluster is implemented or if LLC should be used as fallback.
Changes since v2:
- Reverse order of cpu_get_l2c_info() l2c_id and l2c_size output arguments to match the caller.
--- include/linux/sched/topology.h | 6 ++++++ kernel/sched/core.c | 8 ++++++++ kernel/sched/sched.h | 2 ++ kernel/sched/topology.c | 32 +++++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 3 deletions(-)
include/linux/sched/topology.h +6 -0
@@ -178,6 +178,7 @@ extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],cpumask_var_t *alloc_sched_domains (unsigned int ndoms);void free_sched_domains (cpumask_var_t doms[],unsigned int ndoms); +bool cpus_share_l2c (int this_cpu,int that_cpu);bool cpus_share_llc (int this_cpu,int that_cpu);typedef const struct cpumask *(*sched_domain_mask_f )(int cpu);
@@ -227,6 +228,11 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], { } +static inline bool cpus_share_l2c (int this_cpu,int that_cpu) +{ +return true; +} +static inline bool cpus_share_llc (int this_cpu,int that_cpu) {return true;
kernel/sched/core.c +8 -0
@@ -3904,6 +3904,14 @@ void wake_up_if_idle(int cpu)rcu_read_unlock (); } +bool cpus_share_l2c (int this_cpu,int that_cpu) +{ +if (this_cpu == that_cpu) +return true; + +return per_cpu (sd_l2c_id, this_cpu) ==per_cpu (sd_l2c_id, that_cpu); +} +bool cpus_share_llc (int this_cpu,int that_cpu) {if (this_cpu == that_cpu)
kernel/sched/sched.h +2 -0
@@ -1828,6 +1828,8 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)return sd; } +DECLARE_PER_CPU (int, sd_l2c_size); +DECLARE_PER_CPU (int, sd_l2c_id); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);DECLARE_PER_CPU (int, sd_llc_size);DECLARE_PER_CPU (int, sd_llc_id);
kernel/sched/topology.c +29 -3
@@ -661,8 +661,11 @@ static void destroy_sched_domains(struct sched_domain *sd)* * Also keep a unique ID per domain (we use the first CPU number in * the cpumask of the domain), this allows us to quickly tell if -* two CPUs are in the same cache domain, see cpus_share_llc(). +* two CPUs are in the same cache domain, see cpus_share_l2c() and +* cpus_share_llc(). */ +DEFINE_PER_CPU (int, sd_l2c_size); +DEFINE_PER_CPU (int, sd_l2c_id); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);DEFINE_PER_CPU (int, sd_llc_size);DEFINE_PER_CPU (int, sd_llc_id);
@@ -672,12 +675,27 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);DEFINE_STATIC_KEY_FALSE (sched_asym_cpucapacity); +#ifdef TOPOLOGY_CLUSTER_SYSFS +static int cpu_get_l2c_info (int cpu,int *l2c_size,int *l2c_id) +{ +const struct cpumask *cluster_mask =topology_cluster_cpumask (cpu); + + *l2c_size =cpumask_weight (cluster_mask); + *l2c_id =cpumask_first (cluster_mask); +return 0 ; +} +#else +static int cpu_get_l2c_info (int cpu,int *l2c_size,int *l2c_id) +{ +return -1 ; +} +#endif +static void update_top_cache_domain (int cpu) {struct sched_domain_shared *sds =NULL ;struct sched_domain *sd; -int id = cpu; -int size =1 ; +int id = cpu, size =1 , l2c_id, l2c_size; sd =highest_flag_domain (cpu,SD_SHARE_PKG_RESOURCES );if (sd) {
@@ -686,6 +704,14 @@ static void update_top_cache_domain(int cpu) sds = sd->shared; } +if (cpu_get_l2c_info (cpu, &l2c_size, &l2c_id)) { +/* Fallback on using LLC. */ + l2c_size = size; + l2c_id = id; + } +per_cpu (sd_l2c_size, cpu) = l2c_size; +per_cpu (sd_l2c_id, cpu) = l2c_id; +rcu_assign_pointer (per_cpu (sd_llc, cpu), sd);per_cpu (sd_llc_size, cpu) = size;per_cpu (sd_llc_id, cpu) = id;
Signature · 1 line
2.39.2
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 12 lines · Mathieu Desnoyers
Introduce
cpus_share_l2cto allow querying whether two logical CPUs share a common L2 cache.Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
So I did some more benchmarking to figure out whether the reason for this speedup is the latency delta between L2 and L3, or is due to the number of hw threads contending on the rq locks.
I tried to force grouping of those "skip ttwu queue" groups by a subset of the LLC id, basically by taking the LLC id and adding the cpu number modulo N, where N is chosen based on my machine topology.
The end result is that I have similar numbers for groups of 1, 2, 4 HW threads (which use rq locks and skip queued ttwu within the group). Starting with group of size 8, the performance starts to degrade.
So I wonder: do machines with more than 4 HW threads per L2 cache exist? If it's the case, there we should think about grouping not only by L2 cache, but also sub-divide this group so the number of hw threads per group is at most 4.
Here are my results with the hackbench test-case:
Group cpus by 16 hw threads:
Time: 49s
- group cpus by 8 hw threads: (llc_id + cpu modulo 2)
Time: 39s
- group cpus by 4 hw threads: (llc_id + cpu modulo 4)
Time: 34s
- group cpus by 2 hw threads: (llc_id + cpu modulo 8) (expect same as L2 grouping on this machine)
Time: 34s
- group cpus by 1 hw threads: (cpu)
Time: 33s
Thanks,
Mathieu
Quoted 151 lines
Signed-off-by Mathieu Desnoyers <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Valentin Schneider <[email protected]>Cc Steven Rostedt <[email protected]>Cc Ben Segall <[email protected]>Cc Mel Gorman <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Cc Vincent Guittot <[email protected]>Cc Juri Lelli <[email protected]>Cc Swapnil Sapkal <[email protected]>Cc Aaron Lu <[email protected]>Cc Julien Desfossez <[email protected]>---
Changes since v1:
- Fix l2c id for configurations where L2 have a single logical CPU:
useTOPOLOGY_CLUSTER_SYSFSto find out whether topology cluster is implemented or if LLC should be used as fallback.Changes since v2:
- Reverse order ofcpu_get_l2c_info()l2c_idandl2c_sizeoutput arguments to match the caller.--- include/linux/sched/topology.h | 6 ++++++ kernel/sched/core.c | 8 ++++++++ kernel/sched/sched.h | 2 ++ kernel/sched/topology.c | 32 +++++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 3 deletions(-)include/linux/sched/topology.h +6
-0@@ -178,6 +178,7 @@ extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],cpumask_var_t *alloc_sched_domains (unsigned int ndoms);void free_sched_domains (cpumask_var_t doms[],unsigned int ndoms); +bool cpus_share_l2c (int this_cpu,int that_cpu);bool cpus_share_llc (int this_cpu,int that_cpu);typedef const struct cpumask *(*sched_domain_mask_f )(int cpu);@@ -227,6 +228,11 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], { } +static inline bool cpus_share_l2c (int this_cpu,int that_cpu) +{ +return true; +} +static inline bool cpus_share_llc (int this_cpu,int that_cpu) {return true;kernel/sched/core.c +8
-0@@ -3904,6 +3904,14 @@ void wake_up_if_idle(int cpu)rcu_read_unlock (); } +bool cpus_share_l2c (int this_cpu,int that_cpu) +{ +if (this_cpu == that_cpu) +return true; + +return per_cpu (sd_l2c_id, this_cpu) ==per_cpu (sd_l2c_id, that_cpu); +} +bool cpus_share_llc (int this_cpu,int that_cpu) {if (this_cpu == that_cpu)kernel/sched/sched.h +2
-0@@ -1828,6 +1828,8 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)return sd; } +DECLARE_PER_CPU (int, sd_l2c_size); +DECLARE_PER_CPU (int, sd_l2c_id); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);DECLARE_PER_CPU (int, sd_llc_size);DECLARE_PER_CPU (int, sd_llc_id);kernel/sched/topology.c +29
-3@@ -661,8 +661,11 @@ static void destroy_sched_domains(struct sched_domain *sd)* * Also keep a unique ID per domain (we use the first CPU number in * the cpumask of the domain), this allows us to quickly tell if -* two CPUs are in the same cache domain, see cpus_share_llc(). +* two CPUs are in the same cache domain, see cpus_share_l2c() and +* cpus_share_llc(). */ +DEFINE_PER_CPU (int, sd_l2c_size); +DEFINE_PER_CPU (int, sd_l2c_id); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);DEFINE_PER_CPU (int, sd_llc_size);DEFINE_PER_CPU (int, sd_llc_id);@@ -672,12 +675,27 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);DEFINE_STATIC_KEY_FALSE (sched_asym_cpucapacity); +#ifdef TOPOLOGY_CLUSTER_SYSFS +static int cpu_get_l2c_info (int cpu,int *l2c_size,int *l2c_id) +{ +const struct cpumask *cluster_mask =topology_cluster_cpumask (cpu); + + *l2c_size =cpumask_weight (cluster_mask); + *l2c_id =cpumask_first (cluster_mask); +return 0 ; +} +#else +static int cpu_get_l2c_info (int cpu,int *l2c_size,int *l2c_id) +{ +return -1 ; +} +#endif +static void update_top_cache_domain (int cpu) {struct sched_domain_shared *sds =NULL ;struct sched_domain *sd; -int id = cpu; -int size =1 ; +int id = cpu, size =1 , l2c_id, l2c_size; sd =highest_flag_domain (cpu,SD_SHARE_PKG_RESOURCES );if (sd) {@@ -686,6 +704,14 @@ static void update_top_cache_domain(int cpu) sds = sd->shared; } +if (cpu_get_l2c_info (cpu, &l2c_size, &l2c_id)) { +/* Fallback on using LLC. */ + l2c_size = size; + l2c_id = id; + } +per_cpu (sd_l2c_size, cpu) = l2c_size; +per_cpu (sd_l2c_id, cpu) = l2c_id; +rcu_assign_pointer (per_cpu (sd_llc, cpu), sd);per_cpu (sd_llc_size, cpu) = size;per_cpu (sd_llc_id, cpu) = id;
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 36 lines · Mathieu Desnoyers
Quoted 12 lines · Mathieu Desnoyers
Introduce
cpus_share_l2cto allow querying whether two logical CPUs share a common L2 cache.Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
So I did some more benchmarking to figure out whether the reason for this speedup is the latency delta between L2 and L3, or is due to the number of hw threads contending on the rq locks.
I tried to force grouping of those "skip ttwu queue" groups by a subset of the LLC id, basically by taking the LLC id and adding the cpu number modulo N, where N is chosen based on my machine topology.
The end result is that I have similar numbers for groups of 1, 2, 4 HW threads (which use rq locks and skip queued ttwu within the group). Starting with group of size 8, the performance starts to degrade.
So I wonder: do machines with more than 4 HW threads per L2 cache exist? If it's the case, there we should think about grouping not only by L2 cache, but also sub-divide this group so the number of hw threads per group is at most 4.
Here are my results with the hackbench test-case:
Group cpus by 16 hw threads:
Time: 49s
- group cpus by 8 hw threads: (
llc_id+ cpu modulo 2)Time: 39s
- group cpus by 4 hw threads: (
llc_id+ cpu modulo 4)Time: 34s
Quoted 8 lines
- group cpus by 2 hw threads: (
llc_id+ cpu modulo 8) (expect same as L2 grouping on this machine)Time: 34s
- group cpus by 1 hw threads: (cpu)
Time: 33s
One more interesting data point: I tried modifying the grouping so that I would explicitly group by hw threads which sit in different L3, and even on different NUMA nodes for some
(group id = cpu_id % 192). This is expected to generate really _bad_ cache locality for the runqueue locks within a group.
The result for these groups of 3 HW threads is about 33s with the
hackbench benchmark, which seems to confirm that the cause of the
speedup is reduction of the contention on the rq locks by making the
groups smaller, and therefore reducing the likelihood of contention for the rq locks, rather than by improving cache locality from L3 to L2.
So grouping by shared L2 only happens to make the group size OK, but this benchmark does not significantly benefit from having all runqueue locks on the same L2.
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 61 lines · Mathieu Desnoyers
Quoted 45 lines · Mathieu Desnoyers
Quoted 12 lines · Mathieu Desnoyers
Introduce
cpus_share_l2cto allow querying whether two logical CPUs share a common L2 cache.Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
So I did some more benchmarking to figure out whether the reason for this speedup is the latency delta between L2 and L3, or is due to the number of hw threads contending on the rq locks.
I tried to force grouping of those "skip ttwu queue" groups by a subset of the LLC id, basically by taking the LLC id and adding the cpu number modulo N, where N is chosen based on my machine topology.
The end result is that I have similar numbers for groups of 1, 2, 4 HW threads (which use rq locks and skip queued ttwu within the group). Starting with group of size 8, the performance starts to degrade.
So I wonder: do machines with more than 4 HW threads per L2 cache exist? If it's the case, there we should think about grouping not only by L2 cache, but also sub-divide this group so the number of hw threads per group is at most 4.
Here are my results with the hackbench test-case:
Group cpus by 16 hw threads:
Time: 49s
- group cpus by 8 hw threads: (
llc_id+ cpu modulo 2)Time: 39s
- group cpus by 4 hw threads: (
llc_id+ cpu modulo 4)Time: 34s
Quoted 8 lines
- group cpus by 2 hw threads: (
llc_id+ cpu modulo 8) (expect same as L2 grouping on this machine)Time: 34s
- group cpus by 1 hw threads: (cpu)
Time: 33s
One more interesting data point: I tried modifying the grouping so that I would explicitly group by hw threads which sit in different L3, and even on different NUMA nodes for some
(group id =cpu_id% 192). This is expected to generate really _bad_ cache locality for the runqueue locks within a group.
Quoted 5 lines
The result for these groups of 3 HW threads is about 33s with the hackbench benchmark, which seems to confirm that the cause of the speedup is reduction of the contention on the rq locks by making the groups smaller, and therefore reducing the likelihood of contention for the rq locks, rather than by improving cache locality from L3 to L2.
In addition to reduced rq lock contention, I think another reason this improves performance is because it reduced task migration. Not sure if it is the case on your test system, but on my test machine(Intel SPR), task migration number dropped.
Hackbench on Intel SPR(2sockets/112cores/224threads) test summary:
- performance improved for all three cases; the more tasks(groups), the more performance gain;
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to tg->load_avg and thus, fewer task migrations. This is contradictory and I can not explain yet;
- rq lock contention dropped for all three cases and it dropped the most under more overloaded case: nr_group=32.
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's ttwu_pending is set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq's nr_running or set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.
Below are detailed test data.
Base: 6.5-rc1.
rq_spin%: The percent of raw_spin_rq_lock_nested() as reported by
perf/events=cycles:pp
migration: cpu-migrations reported by "perf stat -a -- sleep 5"
The cmdline used is:
hackbench -g $nr_group -f 20 --pipe --threads -l 480000 -s 100
nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migration
base 46s 1.32% 20.06% 10.78% 10.227 K/sec this_series 37s 0.57% 15.08% 7.05% 10.722 K/sec
nr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migration
base 69s 2.57% 19.68% 10.74% 12.098 K/sec this_series 41s 0.62% 12.11% 5.78% 8.617 K/sec
nr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migration
base 192s±25% 15.12% 25.83% 9.33% 12.141 K/sec this_series 71s 0.47% 10.98% 4.58% 8.198 K/sec
I also tested applying my "ratelimit update to tg->load_avg" patch and the test summary is:
- performance improved noticeably for nr_group=20 and slightly for nr_group=10 case; nr_group=32's performance is roughly the same.
- task migrations dropped for all three cases; nr_group=20 saw the biggest drop.
- rq lock contention dropped for all three cases and again, nr_group=32 saw the biggest drop.
Below are detailed data.
Base: peter's sched/core branch with my "ratelimit" patch.this_series: apply this patchset on top of base.
nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migration
base 36s 0.55% 0.46% 1.43% 15.034 K/sec this_series 35s 0.56% 0.52% 1.53% 13.751 K/sec
nr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migration
base 47s 1.28% 0.73% 2.33% 21.217 K/sec this_series 42s 0.60% 0.69% 1.69% 14.130 K/sec
nr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migration
base 70s 2.38% 0.60% 2.19% 17.855 K/sec this_series 70s 0.58% 0.63% 1.77% 12.331 K/sec
Thanks,
Aaron
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 68 lines · Aaron Lu
Quoted 61 lines · Mathieu Desnoyers
Quoted 45 lines · Mathieu Desnoyers
Quoted 12 lines · Mathieu Desnoyers
Introduce
cpus_share_l2cto allow querying whether two logical CPUs share a common L2 cache.Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
So I did some more benchmarking to figure out whether the reason for this speedup is the latency delta between L2 and L3, or is due to the number of hw threads contending on the rq locks.
I tried to force grouping of those "skip ttwu queue" groups by a subset of the LLC id, basically by taking the LLC id and adding the cpu number modulo N, where N is chosen based on my machine topology.
The end result is that I have similar numbers for groups of 1, 2, 4 HW threads (which use rq locks and skip queued ttwu within the group). Starting with group of size 8, the performance starts to degrade.
So I wonder: do machines with more than 4 HW threads per L2 cache exist? If it's the case, there we should think about grouping not only by L2 cache, but also sub-divide this group so the number of hw threads per group is at most 4.
Here are my results with the hackbench test-case:
Group cpus by 16 hw threads:
Time: 49s
- group cpus by 8 hw threads: (
llc_id+ cpu modulo 2)Time: 39s
- group cpus by 4 hw threads: (
llc_id+ cpu modulo 4)Time: 34s
Quoted 8 lines
- group cpus by 2 hw threads: (
llc_id+ cpu modulo 8) (expect same as L2 grouping on this machine)Time: 34s
- group cpus by 1 hw threads: (cpu)
Time: 33s
One more interesting data point: I tried modifying the grouping so that I would explicitly group by hw threads which sit in different L3, and even on different NUMA nodes for some
(group id =cpu_id% 192). This is expected to generate really _bad_ cache locality for the runqueue locks within a group.Quoted 5 lines
The result for these groups of 3 HW threads is about 33s with the hackbench benchmark, which seems to confirm that the cause of the speedup is reduction of the contention on the rq locks by making the groups smaller, and therefore reducing the likelihood of contention for the rq locks, rather than by improving cache locality from L3 to L2.
Quoted 4 lines
In addition to reduced rq lock contention, I think another reason this improves performance is because it reduced task migration. Not sure if it is the case on your test system, but on my test machine(Intel SPR), task migration number dropped.
Yes, it's indeed the case on my system as well. It cuts migrations by half (9.2K/sec down to 5.0K/sec).
Quoted 3 lines
Hackbench on Intel SPR(2sockets/112cores/224threads) test summary:
- performance improved for all three cases; the more tasks(groups), the more performance gain;
Interesting!
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;
Neither can I.
Quoted 2 lines
- rq lock contention dropped for all three cases and it dropped the most under more overloaded case: nr_group=32.
The fact that you observed rq lock contention dropping is a good sign that doing more queued wakeups is a good thing compared to allowing non-queued wakeups across cpus sharing a whole LLC.
At this point I'm not sure if the reduction on rq lock contention is mostly due to using queued wakeups rather than grabbing remote rq locks, or by a side-effet of doing a queued wakeup rather than immediately doing the wakeup, which would open a window where the target rq is still considered idle by the various code paths within select_task_rq_fair which don't care about rq->ttwu_pending.
Quoted 10 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.
Many code paths in select_task_rq_fair don't seem to care about rq->ttwu_pending.
In wake_affine_idle, for sync wakeups, if nr_running is 1 on the waker, we choose the waker cpu as target.
In wake_affine_idle, if none of waker or prev wakee cpus are idle, then it uses wake_affine_weight to find out which of the waker/prev wakee cpus are targets based on their respective load.
If wake_affine_idle cannot find an idle waker/prev wakee cpu, and if wake_affine_weight finds that the prev wakee cpu had a lower load, then wake_affine returns the prev wakee cpu as target. This happens even if the prev wakee cpu is not idle.
This "target" cpu is then passed to select_idle_sibling. It expects the available_idle_cpu(target) to check again to see whether the target cpu is idle. However, it also uses "sched_idle_cpu(target)" which _only_ considers nr_running (not ttwu_pending flag). Likewise for the other similar idleness checks below in select_idle_sibling for prev and recent_used_cpu. The same happens for the case where a per-cpu kthread stacks with the wakee.
I've tried adding checks for rq->ttwu_pending in those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.
I'm starting to think may want to explore explicitly rate limiting task migrations as well.
For instance, we could do something like this:
Within a 1ms window, if a task is migrated more than once, the following wakeups would consider that the prev runqueue should be considered in priority (as if it was completely idle) as long as its load is below a given threshold.
So every 1ms tasks have a chance to be migrated to the idlest runqueues, but we would then eliminate those frequent migration patterns which end up being bad for cache locality.
Thoughts ?
Thanks,
Mathieu
Quoted 55 lines
Below are detailed test data.
Base: 6.5-rc1.
rq_spin%: The percent ofraw_spin_rq_lock_nested()as reported by
perf/events=cycles:pp
migration: cpu-migrations reported by "perf stat -a -- sleep 5"The cmdline used is:
hackbench -g $nr_group -f 20 --pipe --threads -l 480000 -s 100nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 46s 1.32% 20.06% 10.78% 10.227 K/sec this_series 37s 0.57% 15.08% 7.05% 10.722 K/secnr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 69s 2.57% 19.68% 10.74% 12.098 K/sec this_series 41s 0.62% 12.11% 5.78% 8.617 K/secnr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 192s±25% 15.12% 25.83% 9.33% 12.141 K/sec this_series 71s 0.47% 10.98% 4.58% 8.198 K/secI also tested applying my "ratelimit update to
tg->load_avg" patch and the test summary is:
- performance improved noticeably for nr_group=20 and slightly for nr_group=10 case; nr_group=32's performance is roughly the same.
- task migrations dropped for all three cases; nr_group=20 saw the biggest drop.
- rq lock contention dropped for all three cases and again, nr_group=32 saw the biggest drop.Below are detailed data.
Base: peter's sched/core branch with my "ratelimit" patch.this_series: apply this patchset on top of base.nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 36s 0.55% 0.46% 1.43% 15.034 K/sec this_series 35s 0.56% 0.52% 1.53% 13.751 K/secnr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 47s 1.28% 0.73% 2.33% 21.217 K/sec this_series 42s 0.60% 0.69% 1.69% 14.130 K/secnr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 70s 2.38% 0.60% 2.19% 17.855 K/sec this_series 70s 0.58% 0.63% 1.77% 12.331 K/secThanks,
Aaron
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 131 lines · Mathieu Desnoyers
Quoted 68 lines · Aaron Lu
Quoted 61 lines · Mathieu Desnoyers
Quoted 45 lines · Mathieu Desnoyers
Quoted 12 lines · Mathieu Desnoyers
Introduce
cpus_share_l2cto allow querying whether two logical CPUs share a common L2 cache.Considering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
So I did some more benchmarking to figure out whether the reason for this speedup is the latency delta between L2 and L3, or is due to the number of hw threads contending on the rq locks.
I tried to force grouping of those "skip ttwu queue" groups by a subset of the LLC id, basically by taking the LLC id and adding the cpu number modulo N, where N is chosen based on my machine topology.
The end result is that I have similar numbers for groups of 1, 2, 4 HW threads (which use rq locks and skip queued ttwu within the group). Starting with group of size 8, the performance starts to degrade.
So I wonder: do machines with more than 4 HW threads per L2 cache exist? If it's the case, there we should think about grouping not only by L2 cache, but also sub-divide this group so the number of hw threads per group is at most 4.
Here are my results with the hackbench test-case:
Group cpus by 16 hw threads:
Time: 49s
- group cpus by 8 hw threads: (
llc_id+ cpu modulo 2)Time: 39s
- group cpus by 4 hw threads: (
llc_id+ cpu modulo 4)Time: 34s
Quoted 8 lines
- group cpus by 2 hw threads: (
llc_id+ cpu modulo 8) (expect same as L2 grouping on this machine)Time: 34s
- group cpus by 1 hw threads: (cpu)
Time: 33s
One more interesting data point: I tried modifying the grouping so that I would explicitly group by hw threads which sit in different L3, and even on different NUMA nodes for some
(group id =cpu_id% 192). This is expected to generate really _bad_ cache locality for the runqueue locks within a group.Quoted 5 lines
The result for these groups of 3 HW threads is about 33s with the hackbench benchmark, which seems to confirm that the cause of the speedup is reduction of the contention on the rq locks by making the groups smaller, and therefore reducing the likelihood of contention for the rq locks, rather than by improving cache locality from L3 to L2.
Quoted 4 lines
In addition to reduced rq lock contention, I think another reason this improves performance is because it reduced task migration. Not sure if it is the case on your test system, but on my test machine(Intel SPR), task migration number dropped.
Yes, it's indeed the case on my system as well. It cuts migrations by half (9.2K/sec down to 5.0K/sec).
Quoted 3 lines
Hackbench on Intel SPR(2sockets/112cores/224threads) test summary:
- performance improved for all three cases; the more tasks(groups), the more performance gain;Interesting!
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
Quoted 2 lines
- rq lock contention dropped for all three cases and it dropped the most under more overloaded case: nr_group=32.
The fact that you observed rq lock contention dropping is a good sign that doing more queued wakeups is a good thing compared to allowing non-queued wakeups across cpus sharing a whole LLC.
At this point I'm not sure if the reduction on rq lock contention is mostly due to using queued wakeups rather than grabbing remote rq locks, or by a side-effet of doing a queued wakeup rather than immediately doing the wakeup, which would open a window where the target rq is still considered idle by the various code paths within
select_task_rq_fairwhich don't care aboutrq->ttwu_pending.Quoted 10 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.Many code paths in
select_task_rq_fairdon't seem to care aboutrq->ttwu_pending.In
wake_affine_idle, for sync wakeups, ifnr_runningis 1 on the waker, we choose the waker cpu as target.In
wake_affine_idle, if none of waker or prev wakee cpus are idle, then it useswake_affine_weightto find out which of the waker/prev wakee cpus are targets based on their respective load.If
wake_affine_idlecannot find an idle waker/prev wakee cpu, and ifwake_affine_weightfinds that the prev wakee cpu had a lower load, thenwake_affinereturns the prev wakee cpu as target. This happens even if the prev wakee cpu is not idle.
Quoted 7 lines
This "target" cpu is then passed to
select_idle_sibling. It expects theavailable_idle_cpu(target)to check again to see whether the target cpu is idle. However, it also uses "sched_idle_cpu(target)" which _only_ considersnr_running(notttwu_pendingflag). Likewise for the other similar idleness checks below inselect_idle_siblingfor prev andrecent_used_cpu. The same happens for the case where a per-cpu kthread
stacks with the wakee.
sched_idle_cpu() mainly concerns with idle policy tasks and if the rq does not have any idle policy tasks, it will not return true. Since our tests do not use idle policy tasks, it should never return true.
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.
An interesting puzzle.
Quoted 10 lines
I'm starting to think may want to explore explicitly rate limiting task migrations as well.
For instance, we could do something like this:
Within a 1ms window, if a task is migrated more than once, the following wakeups would consider that the prev runqueue should be considered in priority (as if it was completely idle) as long as its load is below a given threshold.
Quoted 5 lines
So every 1ms tasks have a chance to be migrated to the idlest runqueues, but we would then eliminate those frequent migration patterns which end up being bad for cache locality.
Thoughts ?
Not sure if this is a good idea. I had a feeling it could hurt latency..
Thanks,
Aaron
Quoted 55 lines
Quoted 55 lines
Below are detailed test data.
Base: 6.5-rc1.
rq_spin%: The percent ofraw_spin_rq_lock_nested()as reported by
perf/events=cycles:pp
migration: cpu-migrations reported by "perf stat -a -- sleep 5"The cmdline used is:
hackbench -g $nr_group -f 20 --pipe --threads -l 480000 -s 100nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 46s 1.32% 20.06% 10.78% 10.227 K/sec this_series 37s 0.57% 15.08% 7.05% 10.722 K/secnr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 69s 2.57% 19.68% 10.74% 12.098 K/sec this_series 41s 0.62% 12.11% 5.78% 8.617 K/secnr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 192s±25% 15.12% 25.83% 9.33% 12.141 K/sec this_series 71s 0.47% 10.98% 4.58% 8.198 K/secI also tested applying my "ratelimit update to
tg->load_avg" patch and the test summary is:
- performance improved noticeably for nr_group=20 and slightly for nr_group=10 case; nr_group=32's performance is roughly the same.
- task migrations dropped for all three cases; nr_group=20 saw the biggest drop.
- rq lock contention dropped for all three cases and again, nr_group=32 saw the biggest drop.Below are detailed data.
Base: peter's sched/core branch with my "ratelimit" patch.this_series: apply this patchset on top of base.nr_group=10:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 36s 0.55% 0.46% 1.43% 15.034 K/sec this_series 35s 0.56% 0.52% 1.53% 13.751 K/secnr_group=20:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 47s 1.28% 0.73% 2.33% 21.217 K/sec this_series 42s 0.60% 0.69% 1.69% 14.130 K/secnr_group=32:
time rq_spin% update_cfs_group% update_load_avg% migrationbase 70s 2.38% 0.60% 2.19% 17.855 K/sec this_series 70s 0.58% 0.63% 1.77% 12.331 K/secThanks,
Aaron
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 1 line · Aaron Lu
On Thu, Aug 24, 2023 at 10:40:45AM -0400, Mathieu Desnoyers wrote:
[...]
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 11 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.
[...]
Quoted 6 lines
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.An interesting puzzle.
One metric that can help understand the impact of my patch: comparing hackbench from a baseline where only your load_avg patch is applied to a kernel with my l2c patch applied, I notice that the goidle schedstat is cut in half. For a given CPU (they are pretty much alike), it goes from 650456 to 353487.
So could it be that by doing queued wakeups, we end up batching execution of the woken up tasks for a given CPU, rather than going back and forth between idle and non-idle ? One important thing that this changes is to reduce the number of newidle balance triggered.
Thoughts ?
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 41 lines · Mathieu Desnoyers
Quoted 1 line · Aaron Lu
On Thu, Aug 24, 2023 at 10:40:45AM -0400, Mathieu Desnoyers wrote:
[...]
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 11 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.[...]
Quoted 6 lines
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.An interesting puzzle.
One metric that can help understand the impact of my patch: comparing hackbench from a baseline where only your
load_avgpatch is applied to a kernel with my l2c patch applied, I notice that the goidle schedstat is cut in half. For a given CPU (they are pretty much alike), it goes from 650456 to 353487.
Quoted 4 lines
So could it be that by doing queued wakeups, we end up batching execution of the woken up tasks for a given CPU, rather than going back and forth between idle and non-idle ? One important thing that this changes is to reduce the number of newidle balance triggered.
I noticed the majority(>99%) migrations are from wakeup path on this Intel SPR when running hackbench: ttwu() -> set_task_cpu() -> migrate_task_rq_fair(), so while I think it's a good finding that newidle balance dropped, it's probably not the reason why migration number dropped...
Thanks,
Aaron
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 47 lines · Aaron Lu
Quoted 41 lines · Mathieu Desnoyers
Quoted 1 line · Aaron Lu
On Thu, Aug 24, 2023 at 10:40:45AM -0400, Mathieu Desnoyers wrote:
[...]
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 11 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.[...]
Quoted 6 lines
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.An interesting puzzle.
One metric that can help understand the impact of my patch: comparing hackbench from a baseline where only your
load_avgpatch is applied to a kernel with my l2c patch applied, I notice that the goidle schedstat is cut in half. For a given CPU (they are pretty much alike), it goes from 650456 to 353487.Quoted 4 lines
So could it be that by doing queued wakeups, we end up batching execution of the woken up tasks for a given CPU, rather than going back and forth between idle and non-idle ? One important thing that this changes is to reduce the number of newidle balance triggered.
Quoted 5 lines
I noticed the majority(>99%) migrations are from wakeup path on this Intel SPR when running hackbench:
ttwu()->set_task_cpu()->migrate_task_rq_fair(), so while I think it's a good finding that newidle balance dropped, it's probably not the reason why migration number dropped...
I profiled select_idle_sibling() and found that with this series, select_idle_cpu() tends to fail more and select_idle_sibling() fallbacks to use target in the end, which equals to prev_cpu very often.
Initially I think the reason why select_idle_cpu() failed more with this series is because "wake_list style enqueue" can make the target cpu appear as busy earlier and thus, it will be harder for select_idle_cpu() to find an idle cpu overall. But I also suspect SIS_UTIL makes a difference
here: in vanilla kernel, the idle% is 8% and with this series, the idle% is only 2% and SIS_UTIL may simply skip doing any search for idle cpu. Anyway, I think I'll also need to profile select_idle_cpu() to see what's going on there too.
The above profile was done with below workload on a 2 sockets Intel SPR:
hackbench -g 20 -f 20 --pipe --threads -l 480000 -s 100
Thanks,
Aaron
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 58 lines · Aaron Lu
Quoted 47 lines · Aaron Lu
Quoted 41 lines · Mathieu Desnoyers
Quoted 1 line · Aaron Lu
On Thu, Aug 24, 2023 at 10:40:45AM -0400, Mathieu Desnoyers wrote:
[...]
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 11 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.[...]
Quoted 6 lines
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.An interesting puzzle.
One metric that can help understand the impact of my patch: comparing hackbench from a baseline where only your
load_avgpatch is applied to a kernel with my l2c patch applied, I notice that the goidle schedstat is cut in half. For a given CPU (they are pretty much alike), it goes from 650456 to 353487.Quoted 4 lines
So could it be that by doing queued wakeups, we end up batching execution of the woken up tasks for a given CPU, rather than going back and forth between idle and non-idle ? One important thing that this changes is to reduce the number of newidle balance triggered.
Quoted 5 lines
I noticed the majority(>99%) migrations are from wakeup path on this Intel SPR when running hackbench:
ttwu()->set_task_cpu()->migrate_task_rq_fair(), so while I think it's a good finding that newidle balance dropped, it's probably not the reason why migration number dropped...I profiled
select_idle_sibling()and found that with this series,select_idle_cpu()tends to fail more andselect_idle_sibling()fallbacks to use target in the end, which equals toprev_cpuvery often.
Quoted 8 lines
Initially I think the reason why
select_idle_cpu()failed more with this series is because "wake_liststyle enqueue" can make the target cpu appear as busy earlier and thus, it will be harder forselect_idle_cpu()to find an idle cpu overall. But I also suspectSIS_UTILmakes a difference
here: in vanilla kernel, the idle% is 8% and with this series, the idle% is only 2% andSIS_UTILmay simply skip doing any search for idle cpu. Anyway, I think I'll also need to profileselect_idle_cpu()to see what's going on there too.
Looks like the reduction in task migration is due to SIS_UTIL, i.e. select_idle_cpu() aborts a lot more after applying this series because system utilization increased.
Here are some numbers:
@sis @sic @migrate_idle_cpu @abort
vanilla: 24640640 15883958 11913588 4148649 this_series: 22345434 18597564 4294995 14319284
note:
- @sis: number of times select_idle_sibling() called;
- @sic: number of times select_idle_cpu() called;
- @migrate_idle_cpu: number of times task migrated due to select_idle_cpu() found an idle cpu that is different from prev_cpu;
- @abort: number of times select_idle_cpu() aborts the search due to SIS_UTIL.
All numbers are captured during a 5s window while running the below workload on a 2 sockets Intel SPR(56 cores, 112 threads per socket):
hackbench -g 20 -f 20 --pipe --threads -l 480000 -s 100
So for this workload, I think this series is doing something good: it increased system utilization and due to SIS_UTIL, it also reduced task migration where task migration isn't very useful since system is already overloaded.
Thanks,
Aaron
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 85 lines · Aaron Lu
Quoted 58 lines · Aaron Lu
Quoted 47 lines · Aaron Lu
Quoted 41 lines · Mathieu Desnoyers
Quoted 1 line · Aaron Lu
On Thu, Aug 24, 2023 at 10:40:45AM -0400, Mathieu Desnoyers wrote:
[...]
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
- task migrations dropped with this series for nr_group=20 and 32 according to 'perf stat'. migration number didn't drop for nr_group=10 but the two update functions' cost dropped which means fewer access to
tg->load_avgand thus, fewer task migrations. This is contradictory and I can not explain yet;Neither can I.
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 11 lines
It's not clear to me why this series can reduce task migrations. I doubt it has something to do with more wakelist style wakeup becasue for this test machine, only a single core with two SMT threads share L2 so more wakeups are through wakelist. In wakelist style wakeup, the target rq's
ttwu_pendingis set and that will make the target cpu as !idle_cpu(); This is faster than grabbing the target rq's lock and then increase target rq'snr_runningor set target rq's curr to something else than idle. So wakelist style wakeup can make target cpu appear as non idle faster, but I can't connect this with reduced migration yet, I just feel this might be the reason why task migration reduced.[...]
Quoted 6 lines
Quoted 4 lines
I've tried adding checks for
rq->ttwu_pendingin those code paths on top of my patch and I'm still observing the reduction in number of migrations, so it's unclear to me how doing more queued wakeups can reduce migrations the way it does.An interesting puzzle.
One metric that can help understand the impact of my patch: comparing hackbench from a baseline where only your
load_avgpatch is applied to a kernel with my l2c patch applied, I notice that the goidle schedstat is cut in half. For a given CPU (they are pretty much alike), it goes from 650456 to 353487.Quoted 4 lines
So could it be that by doing queued wakeups, we end up batching execution of the woken up tasks for a given CPU, rather than going back and forth between idle and non-idle ? One important thing that this changes is to reduce the number of newidle balance triggered.
Quoted 5 lines
I noticed the majority(>99%) migrations are from wakeup path on this Intel SPR when running hackbench:
ttwu()->set_task_cpu()->migrate_task_rq_fair(), so while I think it's a good finding that newidle balance dropped, it's probably not the reason why migration number dropped...I profiled
select_idle_sibling()and found that with this series,select_idle_cpu()tends to fail more andselect_idle_sibling()fallbacks to use target in the end, which equals toprev_cpuvery often.Quoted 8 lines
Initially I think the reason why
select_idle_cpu()failed more with this series is because "wake_liststyle enqueue" can make the target cpu appear as busy earlier and thus, it will be harder forselect_idle_cpu()to find an idle cpu overall. But I also suspectSIS_UTILmakes a difference
here: in vanilla kernel, the idle% is 8% and with this series, the idle% is only 2% andSIS_UTILmay simply skip doing any search for idle cpu. Anyway, I think I'll also need to profileselect_idle_cpu()to see what's going on there too.Looks like the reduction in task migration is due to
SIS_UTIL, i.e.select_idle_cpu()aborts a lot more after applying this series because system utilization increased.Here are some numbers:
@sis @sic @migrate_idle_cpu @abortvanilla: 24640640 15883958 11913588 4148649 this_series: 22345434 18597564 4294995 14319284note:
- @sis: number of timesselect_idle_sibling()called;
- @sic: number of timesselect_idle_cpu()called;
- @migrate_idle_cpu: number of times task migrated due toselect_idle_cpu()found an idle cpu that is different fromprev_cpu;
- @abort: number of timesselect_idle_cpu()aborts the search due toSIS_UTIL.
Quoted 8 lines
All numbers are captured during a 5s window while running the below workload on a 2 sockets Intel
SPR(56 cores, 112 threads per socket):
hackbench -g 20 -f 20 --pipe --threads -l 480000 -s 100So for this workload, I think this series is doing something good: it increased system utilization and due to
SIS_UTIL, it also reduced task migration where task migration isn't very useful since system is already overloaded.
This is interesting. Did you also profile the impact of the patches on wake_affine(), especially wake_affine_idle() ? Its behavior did change very significantly in my tests, and this impacts the target cpu number received by select_idle_sibling(). But independently of what wake_affine() returns as target (waker cpu or prev_cpu), if select_idle_cpu() is trigger-happy and finds idle cores near that target, this will cause lots of migrations.
Based on your metrics, the ttwu-queued-l2 approach (in addition to reduce lock contention) appear to decrease the SIS_UTIL idleless level of the cpus enough to completely change the runqueue selection and migration behavior.
I fear that we hide a bad scheduler behavior under the rug by changing the idleless level of a specific workload pattern, while leaving the underlying root cause unfixed.
I'm currently working on a different approach: rate limit migrations. Basically, the idea is to detect when a task is migrated too often for its own good, and prevent the scheduler from migrating it for a short while. I get about 30% performance improvement with this approach as well (limit migration to 1 per 2ms window per task). I'll finish polishing my commit messages and send a series as RFC soon.
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Re: [RFC PATCH v3 2/3] sched: Introduce cpus_share_l2c
Quoted 22 lines · Mathieu Desnoyers
Quoted 21 lines · Aaron Lu
Looks like the reduction in task migration is due to
SIS_UTIL, i.e.select_idle_cpu()aborts a lot more after applying this series because system utilization increased.Here are some numbers:
@sis @sic @migrate_idle_cpu @abortvanilla: 24640640 15883958 11913588 4148649 this_series: 22345434 18597564 4294995 14319284note:
- @sis: number of timesselect_idle_sibling()called;
- @sic: number of timesselect_idle_cpu()called;
- @migrate_idle_cpu: number of times task migrated due toselect_idle_cpu()found an idle cpu that is different fromprev_cpu;
- @abort: number of timesselect_idle_cpu()aborts the search due toSIS_UTIL.All numbers are captured during a 5s window while running the below workload on a 2 sockets Intel
SPR(56 cores, 112 threads per socket):
hackbench -g 20 -f 20 --pipe --threads -l 480000 -s 100
Quoted 7 lines
Quoted 4 lines
So for this workload, I think this series is doing something good: it increased system utilization and due to
SIS_UTIL, it also reduced task migration where task migration isn't very useful since system is already overloaded.This is interesting. Did you also profile the impact of the patches on
wake_affine(), especiallywake_affine_idle()? Its behavior did change very
For group=20 case, wake_affine() and wake_affine_idle() don't appear to change much on this Intel machine, in that target received by sis() is mostly prev_cpu instead of waker(this) cpu for both kernels.
But I do notice for group=32 case, in vanilla kernel, the chance of target as received by sis() becoming to waker cpu increased a lot while with this series, targer remains mostly prev_cpu and that is the reason why migration dropped with this series for group=32 case becasue when sis() fallback to use target, this series has a higher chance of not mirgating the task. And my profile shows for vanilla kernel, when it choose target as waker cpu, it's mostly due to wake_affine_weight(), not wake_affine_idle().
Thanks,
Aaron
Quoted 29 lines
significantly in my tests, and this impacts the target cpu number received by
select_idle_sibling(). But independently of whatwake_affine()returns as target (waker cpu orprev_cpu), ifselect_idle_cpu()is trigger-happy and finds idle cores near that target, this will cause lots of migrations.Based on your metrics, the ttwu-queued-l2 approach (in addition to reduce lock contention) appear to decrease the
SIS_UTILidleless level of the cpus enough to completely change the runqueue selection and migration behavior.I fear that we hide a bad scheduler behavior under the rug by changing the idleless level of a specific workload pattern, while leaving the underlying root cause unfixed.
I'm currently working on a different approach: rate limit migrations. Basically, the idea is to detect when a task is migrated too often for its own good, and prevent the scheduler from migrating it for a short while. I get about 30% performance improvement with this approach as well (limit migration to 1 per 2ms window per task). I'll finish polishing my commit messages and send a series as RFC soon.
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
RFC PATCH v3 3/3 sched: ttwu_queue_cond: skip queued wakeups across different l2 caches
In reply to Mathieu DesnoyersConsidering a system like the AMD EPYC 9654 96-Core Processor, the L1 cache has a latency of 4-5 cycles, the L2 cache has a latency of at least 14ns, whereas the L3 cache has a latency of 50ns [1]. Compared to this, I measured the RAM accesses to a latency around 120ns on my system [2]. So L3 really is only 2.4x faster than RAM accesses. Therefore, with this relatively slow access speed compared to L2, the scheduler will benefit from only considering CPUs sharing an L2 cache for the purpose of using remote runqueue locking rather than queued wakeups.
Skipping queued wakeups for all logical CPUs sharing an LLC means that on a 192 cores AMD EPYC 9654 96-Core Processor (over 2 sockets), groups of 8 cores (16 hardware threads) end up grabbing runqueue locks of other runqueues within the same group for each wakeup, causing contention on the runqueue locks.
Improve this by only considering logical cpus sharing an L2 cache as candidates for skipping use of the queued wakeups.
This results in the following benchmark improvements:
hackbench -g 32 -f 20 --threads --pipe -l 480000 -s 100
from 49s to 34s. (30% speedup)
And similarly with perf bench:
perf bench sched messaging -g 32 -p -t -l 100000
from 10.9s to 7.4s (32% speedup)
I have noticed that in order to observe the speedup, the workload needs to keep the CPUs sufficiently busy to cause runqueue lock contention, but not so busy that they don't go idle. This can be explained by the fact that idle CPUs are a preferred target for task wakeup runqueue selection, and therefore having idle cpus causes more migrations, which triggers more remote wakeups. For both the hackbench and the perf bench sched messaging benchmarks, the scale of the workload can be tweaked by changing the number groups.
This was developed as part of the investigation into a weird regression reported by AMD where adding a raw spinlock in the scheduler context switch accelerated hackbench. It turned out that changing this raw spinlock for a loop of 10000x cpu_relax within do_idle() had similar benefits.
This patch achieves a similar effect without busy waiting nor changing anything about runqueue selection on wakeup. It considers that only hardware threads sharing an L2 cache should skip the queued try-to-wakeup and directly grab the target runqueue lock, rather than allowing all hardware threads sharing an LLC to do so.
I would be interested to hear feedback about performance impact of this patch (improvement or regression) on other workloads and hardware, especially for Intel CPUs. One thing that we might want to empirically figure out from the topology is whether there is a maximum number of hardware threads within an LLC below which it would make sense to use the LLC rather than L2 as group within which queued wakeups can be skipped.
--- kernel/sched/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/sched/core.c +2 -2
@@ -3934,10 +3934,10 @@ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)return false;/* -* If the CPU does not share cache, then queue the task on the +* If the CPU does not share L2 cache, then queue the task on the * remote rqs wakelist to avoid accessing remote data. */ -if (!cpus_share_llc (smp_processor_id (), cpu)) +if (!cpus_share_l2c (smp_processor_id (), cpu))return true;if (cpu ==smp_processor_id ())
Signature · 1 line
2.39.2
Hello Mathieu,
Quoted 10 lines · Mathieu Desnoyers
This series improves performance of scheduler wakeups on large systems by skipping queued wakeups only when CPUs share their L2 cache, rather than when they share their LLC.
The speedup mainly reproduces on workloads which have at least *some* idle time (because it significantly increases the number of migrations, and thus remote wakeups), *and* it needs to have a sufficient load to cause contention on the runqueue locks.
Feedback is welcome,
I ran some micro-benchmarks as part of testing this series. Here are the observations:
- Hackbench shows improvement with this patch and Aaron's patch with 6.5-rc1 kernel as the baseline.
- tbench and netperf shows shows some dip in performance with highly overloaded case.
- Other micro-benchmarks shows more or less similar performance with these patches.
o System Details
- 4th Generation EPYC System
- 2 x 128C/256T
- NPS1 mode
o Kernels
base: 6.5.0-rc1
base + mathieu-queued-wakeup: 6.5.0-rc1 + Mathieu's patches [1]
base + aaron-tg-load-avg: 6.5.0-rc1 + Aaron's patch [2]
base + queued-wakeup + tg-load-avg: 6.5.0-rc1 + Mathieu's patches [1] + Aaron's patch [2]
[References]
[1] "sched: Skip queued wakeups only when L2 is shared"
(https://lore.kernel.org/all/[email protected]/) [2] "Reduce cost of accessing tg->load_avg" (https://lore.kernel.org/lkml/[email protected]/)
==================================================================
Test : hackbench
Units : Time in seconds
Interpretation: Lower is better
Statistic : AMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg 1-groups: 22.15 (0.00 pct) 22.46 (-1.39 pct) 22.35 (-0.90 pct) 21.20 (4.28 pct) 2-groups: 22.76 (0.00 pct) 21.78 (4.30 pct) 22.60 (0.70 pct) 21.90 (3.77 pct) 4-groups: 22.12 (0.00 pct) 22.02 (0.45 pct) 22.22 (-0.45 pct) 21.94 (0.81 pct) 8-groups: 24.80 (0.00 pct) 22.36 (9.83 pct) 22.99 (7.29 pct) 22.00 (11.29 pct) 16-groups: 31.09 (0.00 pct) 21.56 (30.65 pct) 22.13 (28.81 pct) 20.60 (33.74 pct)
==================================================================
Test : tbench
Units : Throughput
Interpretation: Higher is better
Statistic : AMean
==================================================================
Clients: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1 261.49 (0.00 pct) 261.18 (-0.11 pct) 262.29 (0.30 pct) 257.80 (-1.41 pct)
2 514.08 (0.00 pct) 521.30 (1.40 pct) 517.66 (0.69 pct) 510.96 (-0.60 pct)
4 1002.51 (0.00 pct) 988.81 (-1.36 pct) 995.04 (-0.74 pct) 987.74 (-1.47 pct)
8 1978.74 (0.00 pct) 1966.60 (-0.61 pct) 1991.85 (0.66 pct) 1941.39 (-1.88 pct)
16 3864.14 (0.00 pct) 3952.03 (2.27 pct) 3914.80 (1.31 pct) 3873.88 (0.25 pct)
32 7473.19 (0.00 pct) 7602.38 (1.72 pct) 7585.94 (1.50 pct) 7423.44 (-0.66 pct)
64 14335.10 (0.00 pct) 14313.17 (-0.15 pct) 14474.67 (0.97 pct) 14030.63 (-2.12 pct)
128 27275.73 (0.00 pct) 25176.80 (-7.69 pct) 28066.53 (2.89 pct) 25045.53 (-8.17 pct)
256 41688.17 (0.00 pct) 44373.40 (6.44 pct) 43779.37 (5.01 pct) 41427.00 (-0.62 pct)
512 137481.33 (0.00 pct) 136466.67 (-0.73 pct) 134824.00 (-1.93 pct) 141280.00 (2.76 pct)
1024 140534.00 (0.00 pct) 141916.33 (0.98 pct) 137008.33 (-2.50 pct) 126319.33 (-10.11 pct)
2048 145378.00 (0.00 pct) 145479.33 (0.06 pct) 138763.67 (-4.54 pct) 124471.00 (-14.38 pct) ==================================================================
Test : netperf
Units : Througput
Interpretation: Higher is better
Statistic : AMean
==================================================================
6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1-clients: 59642.88 (0.00 pct) 61647.37 (3.36 pct) 61186.24 (2.58 pct) 59099.11 (-0.91 pct) 2-clients: 59349.65 (0.00 pct) 60896.01 (2.60 pct) 60582.49 (2.07 pct) 62738.47 (5.70 pct) 4-clients: 59197.37 (0.00 pct) 60457.29 (2.12 pct) 63042.52 (6.49 pct) 60879.58 (2.84 pct) 8-clients: 61977.66 (0.00 pct) 60389.92 (-2.56 pct) 62078.15 (0.16 pct) 60314.65 (-2.68 pct) 16-clients: 61518.83 (0.00 pct) 61143.51 (-0.61 pct) 60946.08 (-0.93 pct) 59388.78 (-3.46 pct) 32-clients: 58230.81 (0.00 pct) 58653.20 (0.72 pct) 58594.14 (0.62 pct) 58188.52 (-0.07 pct) 64-clients: 58050.92 (0.00 pct) 57834.55 (-0.37 pct) 58183.51 (0.22 pct) 57565.75 (-0.83 pct) 128-clients: 54324.55 (0.00 pct) 54385.60 (0.11 pct) 54913.43 (1.08 pct) 53917.11 (-0.75 pct) 256-clients: 70155.29 (0.00 pct) 69390.68 (-1.08 pct) 70097.50 (-0.08 pct) 64410.66 (-8.18 pct) 512-clients: 61511.77 (0.00 pct) 61480.99 (-0.05 pct) 54493.82 (-11.40 pct) 46227.05 (-24.84 pct)
==================================================================
Test : stream-10
Units : Bandwidth, MB/s
Interpretation: Higher is better
Statistic : HMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
Copy: 353336.76 (0.00 pct) 352956.36 (-0.10 pct) 349583.67 (-1.06 pct) 351152.80 (-0.61 pct) Scale: 353474.88 (0.00 pct) 354582.35 (0.31 pct) 350543.75 (-0.82 pct) 353275.74 (-0.05 pct) Add: 371984.24 (0.00 pct) 372824.87 (0.22 pct) 369173.72 (-0.75 pct) 370483.63 (-0.40 pct) Triad: 372625.41 (0.00 pct) 278389.62 (-25.28 pct) 369504.06 (-0.83 pct) 369070.11 (-0.95 pct)
==================================================================
Test : stream-100
Units : Bandwidth, MB/s
Interpretation: Higher is better
Statistic : HMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
Copy: 353476.35 (0.00 pct) 354954.50 (0.41 pct) 354614.56 (0.32 pct) 353512.71 (0.01 pct) Scale: 353214.73 (0.00 pct) 354884.12 (0.47 pct) 355841.17 (0.74 pct) 353220.53 (0.00 pct) Add: 370755.48 (0.00 pct) 372292.72 (0.41 pct) 375307.35 (1.22 pct) 369917.77 (-0.22 pct) Triad: 370652.02 (0.00 pct) 372732.11 (0.56 pct) 375718.85 (1.36 pct) 369926.26 (-0.19 pct)
==================================================================
Test : schbench (old)
Units : 99th percentile latency in us
Interpretation: Lower is better
Statistic : Median
==================================================================
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 56.00 (0.00 pct) 58.00 (-3.57 pct) 60.00 (-7.14 pct) 60.00 (-7.14 pct) 2: 61.00 (0.00 pct) 56.00 (8.19 pct) 59.00 (3.27 pct) 60.00 (1.63 pct) 4: 64.00 (0.00 pct) 62.00 (3.12 pct) 66.00 (-3.12 pct) 64.00 (0.00 pct) 8: 96.00 (0.00 pct) 78.00 (18.75 pct) 76.00 (20.83 pct) 93.00 (3.12 pct) 16: 98.00 (0.00 pct) 95.00 (3.06 pct) 98.00 (0.00 pct) 95.00 (3.06 pct) 32: 137.00 (0.00 pct) 144.00 (-5.10 pct) 133.00 (2.91 pct) 130.00 (5.10 pct) 64: 206.00 (0.00 pct) 210.00 (-1.94 pct) 200.00 (2.91 pct) 217.00 (-5.33 pct) 128: 348.00 (0.00 pct) 347.00 (0.28 pct) 413.00 (-18.67 pct) 366.00 (-5.17 pct) 256: 679.00 (0.00 pct) 669.00 (1.47 pct) 669.00 (1.47 pct) 675.00 (0.58 pct) 512: 1366.00 (0.00 pct) 1366.00 (0.00 pct) 1442.00 (-5.56 pct) 1430.00 (-4.68 pct)
==================================================================
Test : schbench (new)
Units : 99th percentile latency in us
Interpretation: Lower is better
Statistic : Median
==================================================================
Metric: wakeup_lat_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 15.00 (0.00 pct) 15.00 (0.00 pct) 16.00 (-6.66 pct) 17.00 (-13.33 pct) 2: 16.00 (0.00 pct) 16.00 (0.00 pct) 17.00 (-6.25 pct) 17.00 (-6.25 pct) 4: 17.00 (0.00 pct) 17.00 (0.00 pct) 15.00 (11.76 pct) 17.00 (0.00 pct) 8: 11.00 (0.00 pct) 13.00 (-18.18 pct) 11.00 (0.00 pct) 11.00 (0.00 pct) 16: 11.00 (0.00 pct) 11.00 (0.00 pct) 10.00 (9.09 pct) 9.00 (18.18 pct) 32: 11.00 (0.00 pct) 11.00 (0.00 pct) 11.00 (0.00 pct) 11.00 (0.00 pct) 64: 10.00 (0.00 pct) 11.00 (-10.00 pct) 10.00 (0.00 pct) 10.00 (0.00 pct) 128: 11.00 (0.00 pct) 12.00 (-9.09 pct) 12.00 (-9.09 pct) 11.00 (0.00 pct) 256: 117.00 (0.00 pct) 162.00 (-38.46 pct) 90.00 (23.07 pct) 103.00 (11.96 pct)
512: 22496.00 (0.00 pct) 21664.00 (3.69 pct) 22368.00 (0.56 pct) 21408.00 (4.83 pct)
Metric: request_lat_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 6872.00 (0.00 pct) 6872.00 (0.00 pct) 6792.00 (1.16 pct) 6856.00 (0.23 pct) 2: 6824.00 (0.00 pct) 6824.00 (0.00 pct) 6872.00 (-0.70 pct) 6856.00 (-0.46 pct) 4: 6824.00 (0.00 pct) 6808.00 (0.23 pct) 6872.00 (-0.70 pct) 6824.00 (0.00 pct) 8: 6824.00 (0.00 pct) 6824.00 (0.00 pct) 6872.00 (-0.70 pct) 6824.00 (0.00 pct) 16: 6824.00 (0.00 pct) 6840.00 (-0.23 pct) 6872.00 (-0.70 pct) 6840.00 (-0.23 pct) 32: 6840.00 (0.00 pct) 6840.00 (0.00 pct) 6888.00 (-0.70 pct) 6856.00 (-0.23 pct) 64: 6840.00 (0.00 pct) 6872.00 (-0.46 pct) 6888.00 (-0.70 pct) 6872.00 (-0.46 pct) 128: 12272.00 (0.00 pct) 12784.00 (-4.17 pct) 13200.00 (-7.56 pct) 12016.00 (2.08 pct) 256: 13328.00 (0.00 pct) 13392.00 (-0.48 pct) 13712.00 (-2.88 pct) 13552.00 (-1.68 pct) 512: 88832.00 (0.00 pct) 86400.00 (2.73 pct) 88192.00 (0.72 pct) 85632.00 (3.60 pct)
Metric: rps_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 297.00 (0.00 pct) 297.00 (0.00 pct) 297.00 (0.00 pct) 299.00 (-0.67 pct) 2: 601.00 (0.00 pct) 603.00 (-0.33 pct) 595.00 (0.99 pct) 601.00 (0.00 pct) 4: 1206.00 (0.00 pct) 1206.00 (0.00 pct) 1190.00 (1.32 pct) 1206.00 (0.00 pct) 8: 2412.00 (0.00 pct) 2412.00 (0.00 pct) 2396.00 (0.66 pct) 2420.00 (-0.33 pct) 16: 4840.00 (0.00 pct) 4824.00 (0.33 pct) 4792.00 (0.99 pct) 4840.00 (0.00 pct) 32: 9648.00 (0.00 pct) 9648.00 (0.00 pct) 9584.00 (0.66 pct) 9680.00 (-0.33 pct) 64: 19360.00 (0.00 pct) 19296.00 (0.33 pct) 19168.00 (0.99 pct) 19296.00 (0.33 pct) 128: 37952.00 (0.00 pct) 35264.00 (7.08 pct) 36672.00 (3.37 pct) 38080.00 (-0.33 pct) 256: 41408.00 (0.00 pct) 41536.00 (-0.30 pct) 39744.00 (4.01 pct) 40896.00 (1.23 pct) 512: 36288.00 (0.00 pct) 36800.00 (-1.41 pct) 35264.00 (2.82 pct) 35776.00 (1.41 pct)
Quoted 24 lines
Thanks,
Mathieu
Mathieu Desnoyers (3):
sched: Renamecpus_share_cachetocpus_share_llc
sched: Introducecpus_share_l2c(v3)
sched:ttwu_queue_cond: skip queued wakeups across different l2 cachesCc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Valentin Schneider <[email protected]>Cc Steven Rostedt <[email protected]>Cc Ben Segall <[email protected]>Cc Mel Gorman <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Cc Vincent Guittot <[email protected]>Cc Juri Lelli <[email protected]>Cc Swapnil Sapkal <[email protected]>Cc Aaron Lu <[email protected]>Cc Julien Desfossez <[email protected]>
Quoted 8 lines
block/blk-mq.c | 2 +- include/linux/sched/topology.h | 10 ++++++++-- kernel/sched/core.c | 14 +++++++++++--- kernel/sched/fair.c | 8 ++++---- kernel/sched/sched.h | 2 ++ kernel/sched/topology.c | 32 +++++++++++++++++++++++++++++--- 6 files changed, 55 insertions(+), 13 deletions(-)
--
Thanks and Regards,
Swapnil
Quoted 17 lines · Swapnil Sapkal
Hello Mathieu,
Quoted 10 lines · Mathieu Desnoyers
This series improves performance of scheduler wakeups on large systems by skipping queued wakeups only when CPUs share their L2 cache, rather than when they share their LLC.
The speedup mainly reproduces on workloads which have at least *some* idle time (because it significantly increases the number of migrations, and thus remote wakeups), *and* it needs to have a sufficient load to cause contention on the runqueue locks.
Feedback is welcome,
I ran some micro-benchmarks as part of testing this series. Here are the observations:
Quoted 8 lines
- Hackbench shows improvement with this patch and Aaron's patch with 6.5-rc1 kernel as the baseline.
- tbench and netperf shows shows some dip in performance with highly overloaded case.
- Other micro-benchmarks shows more or less similar performance with these patches.
Those results look promising! Thanks for testing!
Mathieu
Quoted 264 lines
o System Details
- 4th Generation EPYC System
- 2 x 128C/256T
- NPS1 modeo Kernels
base: 6.5.0-rc1
base + mathieu-queued-wakeup: 6.5.0-rc1 + Mathieu's patches [1]
base + aaron-tg-load-avg: 6.5.0-rc1 + Aaron's patch [2]
base + queued-wakeup + tg-load-avg: 6.5.0-rc1 + Mathieu's patches [1] + Aaron's patch [2][References]
[1] "sched: Skip queued wakeups only when L2 is shared"
(https://lore.kernel.org/all/[email protected]/)
[2] "Reduce cost of accessingtg->load_avg"(https://lore.kernel.org/lkml/[email protected]/)
==================================================================
Test : hackbench
Units : Time in seconds
Interpretation: Lower is better
Statistic : AMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg 1-groups: 22.15 (0.00 pct) 22.46 (-1.39 pct)
22.35 (-0.90 pct) 21.20 (4.28 pct)
2-groups: 22.76 (0.00 pct) 21.78 (4.30 pct)
22.60 (0.70 pct) 21.90 (3.77 pct)
4-groups: 22.12 (0.00 pct) 22.02 (0.45 pct)
22.22 (-0.45 pct) 21.94 (0.81 pct)
8-groups: 24.80 (0.00 pct) 22.36 (9.83 pct)
22.99 (7.29 pct) 22.00 (11.29 pct)
16-groups: 31.09 (0.00 pct) 21.56 (30.65 pct) 22.13 (28.81 pct) 20.60 (33.74 pct)==================================================================
Test : tbench
Units : Throughput
Interpretation: Higher is better
Statistic : AMean
==================================================================
Clients: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg 1 261.49 (0.00 pct) 261.18 (-0.11 pct)
262.29 (0.30 pct) 257.80 (-1.41 pct)
2 514.08 (0.00 pct) 521.30 (1.40 pct)
517.66 (0.69 pct) 510.96 (-0.60 pct)
4 1002.51 (0.00 pct) 988.81 (-1.36 pct)
995.04 (-0.74 pct) 987.74 (-1.47 pct)
8 1978.74 (0.00 pct) 1966.60 (-0.61 pct)
1991.85 (0.66 pct) 1941.39 (-1.88 pct)
16 3864.14 (0.00 pct) 3952.03 (2.27 pct)
3914.80 (1.31 pct) 3873.88 (0.25 pct)
32 7473.19 (0.00 pct) 7602.38 (1.72 pct)
7585.94 (1.50 pct) 7423.44 (-0.66 pct)
64 14335.10 (0.00 pct) 14313.17 (-0.15 pct)
14474.67 (0.97 pct) 14030.63 (-2.12 pct)
128 27275.73 (0.00 pct) 25176.80 (-7.69 pct)
28066.53 (2.89 pct) 25045.53 (-8.17 pct)
256 41688.17 (0.00 pct) 44373.40 (6.44 pct)
43779.37 (5.01 pct) 41427.00 (-0.62 pct)
512 137481.33 (0.00 pct) 136466.67 (-0.73 pct)
134824.00 (-1.93 pct) 141280.00 (2.76 pct)
1024 140534.00 (0.00 pct) 141916.33 (0.98 pct)
137008.33 (-2.50 pct) 126319.33 (-10.11 pct)
2048 145378.00 (0.00 pct) 145479.33 (0.06 pct)
138763.67 (-4.54 pct) 124471.00 (-14.38 pct)==================================================================
Test : netperf
Units : Througput
Interpretation: Higher is better
Statistic : AMean
==================================================================
6.5.0-rc1 (base) base + mathieu-queued-wakeup
base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1-clients: 59642.88 (0.00 pct) 61647.37 (3.36
pct) 61186.24 (2.58 pct) 59099.11 (-0.91 pct)
2-clients: 59349.65 (0.00 pct) 60896.01 (2.60
pct) 60582.49 (2.07 pct) 62738.47 (5.70 pct)
4-clients: 59197.37 (0.00 pct) 60457.29 (2.12
pct) 63042.52 (6.49 pct) 60879.58 (2.84 pct)
8-clients: 61977.66 (0.00 pct) 60389.92 (-2.56
pct) 62078.15 (0.16 pct) 60314.65 (-2.68 pct)
16-clients: 61518.83 (0.00 pct) 61143.51 (-0.61
pct) 60946.08 (-0.93 pct) 59388.78 (-3.46 pct)
32-clients: 58230.81 (0.00 pct) 58653.20 (0.72
pct) 58594.14 (0.62 pct) 58188.52 (-0.07 pct)
64-clients: 58050.92 (0.00 pct) 57834.55 (-0.37
pct) 58183.51 (0.22 pct) 57565.75 (-0.83 pct)
128-clients: 54324.55 (0.00 pct) 54385.60 (0.11
pct) 54913.43 (1.08 pct) 53917.11 (-0.75 pct)
256-clients: 70155.29 (0.00 pct) 69390.68 (-1.08
pct) 70097.50 (-0.08 pct) 64410.66 (-8.18 pct)
512-clients: 61511.77 (0.00 pct) 61480.99 (-0.05
pct) 54493.82 (-11.40 pct) 46227.05 (-24.84 pct)==================================================================
Test : stream-10
Units : Bandwidth, MB/s
Interpretation: Higher is better
Statistic : HMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
Copy: 353336.76 (0.00 pct) 352956.36 (-0.10 pct)
349583.67 (-1.06 pct) 351152.80 (-0.61 pct)
Scale: 353474.88 (0.00 pct) 354582.35 (0.31 pct) 350543.75 (-0.82 pct) 353275.74 (-0.05 pct)
Add: 371984.24 (0.00 pct) 372824.87 (0.22 pct)
369173.72 (-0.75 pct) 370483.63 (-0.40 pct)
Triad: 372625.41 (0.00 pct) 278389.62 (-25.28 pct) 369504.06 (-0.83 pct) 369070.11 (-0.95 pct)==================================================================
Test : stream-100
Units : Bandwidth, MB/s
Interpretation: Higher is better
Statistic : HMean
==================================================================
Test: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base + aaron-tg-load-avg base + queued-wakeup + tg-load-avg
Copy: 353476.35 (0.00 pct) 354954.50 (0.41 pct)
354614.56 (0.32 pct) 353512.71 (0.01 pct)
Scale: 353214.73 (0.00 pct) 354884.12 (0.47 pct) 355841.17 (0.74 pct) 353220.53 (0.00 pct)
Add: 370755.48 (0.00 pct) 372292.72 (0.41 pct)
375307.35 (1.22 pct) 369917.77 (-0.22 pct)
Triad: 370652.02 (0.00 pct) 372732.11 (0.56 pct) 375718.85 (1.36 pct) 369926.26 (-0.19 pct)==================================================================
Test : schbench (old)
Units : 99th percentile latency in us
Interpretation: Lower is better
Statistic : Median
==================================================================
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 56.00 (0.00 pct) 58.00 (-3.57
pct) 60.00 (-7.14 pct) 60.00 (-7.14 pct)
2: 61.00 (0.00 pct) 56.00 (8.19
pct) 59.00 (3.27 pct) 60.00 (1.63 pct)
4: 64.00 (0.00 pct) 62.00 (3.12
pct) 66.00 (-3.12 pct) 64.00 (0.00 pct)
8: 96.00 (0.00 pct) 78.00 (18.75
pct) 76.00 (20.83 pct) 93.00 (3.12 pct)
16: 98.00 (0.00 pct) 95.00 (3.06
pct) 98.00 (0.00 pct) 95.00 (3.06 pct)
32: 137.00 (0.00 pct) 144.00 (-5.10 pct)
133.00 (2.91 pct) 130.00 (5.10 pct)
64: 206.00 (0.00 pct) 210.00 (-1.94 pct)
200.00 (2.91 pct) 217.00 (-5.33 pct)
128: 348.00 (0.00 pct) 347.00 (0.28 pct) 413.00 (-18.67 pct) 366.00 (-5.17 pct)
256: 679.00 (0.00 pct) 669.00 (1.47 pct) 669.00 (1.47 pct) 675.00 (0.58 pct)
512: 1366.00 (0.00 pct) 1366.00 (0.00 pct) 1442.00 (-5.56 pct) 1430.00 (-4.68 pct)==================================================================
Test : schbench (new)
Units : 99th percentile latency in us
Interpretation: Lower is better
Statistic : Median
==================================================================
Metric:wakeup_lat_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg
1: 15.00 (0.00 pct) 15.00 (0.00
pct) 16.00 (-6.66 pct) 17.00 (-13.33 pct)
2: 16.00 (0.00 pct) 16.00 (0.00
pct) 17.00 (-6.25 pct) 17.00 (-6.25 pct)
4: 17.00 (0.00 pct) 17.00 (0.00
pct) 15.00 (11.76 pct) 17.00 (0.00 pct)
8: 11.00 (0.00 pct) 13.00 (-18.18
pct) 11.00 (0.00 pct) 11.00 (0.00 pct)
16: 11.00 (0.00 pct) 11.00 (0.00
pct) 10.00 (9.09 pct) 9.00 (18.18 pct)
32: 11.00 (0.00 pct) 11.00 (0.00
pct) 11.00 (0.00 pct) 11.00 (0.00 pct)
64: 10.00 (0.00 pct) 11.00 (-10.00
pct) 10.00 (0.00 pct) 10.00 (0.00 pct)
128: 11.00 (0.00 pct) 12.00 (-9.09 pct) 12.00 (-9.09 pct) 11.00 (0.00 pct)
256: 117.00 (0.00 pct) 162.00 (-38.46 pct) 90.00 (23.07 pct) 103.00 (11.96 pct)
512: 22496.00 (0.00 pct) 21664.00 (3.69 pct) 22368.00 (0.56 pct) 21408.00 (4.83 pct)Metric:
request_lat_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg 1: 6872.00 (0.00 pct) 6872.00 (0.00 pct)
6792.00 (1.16 pct) 6856.00 (0.23 pct)
2: 6824.00 (0.00 pct) 6824.00 (0.00 pct)
6872.00 (-0.70 pct) 6856.00 (-0.46 pct)
4: 6824.00 (0.00 pct) 6808.00 (0.23 pct)
6872.00 (-0.70 pct) 6824.00 (0.00 pct)
8: 6824.00 (0.00 pct) 6824.00 (0.00 pct)
6872.00 (-0.70 pct) 6824.00 (0.00 pct)
16: 6824.00 (0.00 pct) 6840.00 (-0.23 pct)
6872.00 (-0.70 pct) 6840.00 (-0.23 pct)
32: 6840.00 (0.00 pct) 6840.00 (0.00 pct)
6888.00 (-0.70 pct) 6856.00 (-0.23 pct)
64: 6840.00 (0.00 pct) 6872.00 (-0.46 pct)
6888.00 (-0.70 pct) 6872.00 (-0.46 pct)128: 12272.00 (0.00 pct) 12784.00 (-4.17 pct) 13200.00 (-7.56 pct) 12016.00 (2.08 pct) 256: 13328.00 (0.00 pct) 13392.00 (-0.48 pct) 13712.00 (-2.88 pct) 13552.00 (-1.68 pct) 512: 88832.00 (0.00 pct) 86400.00 (2.73 pct) 88192.00 (0.72 pct) 85632.00 (3.60 pct)Metric:
rps_summary
#workers: 6.5.0-rc1 (base) base + mathieu-queued-wakeup base
+ aaron-tg-load-avg base + queued-wakeup + tg-load-avg 1: 297.00 (0.00 pct) 297.00 (0.00 pct)
297.00 (0.00 pct) 299.00 (-0.67 pct)
2: 601.00 (0.00 pct) 603.00 (-0.33 pct)
595.00 (0.99 pct) 601.00 (0.00 pct)
4: 1206.00 (0.00 pct) 1206.00 (0.00 pct)
1190.00 (1.32 pct) 1206.00 (0.00 pct)
8: 2412.00 (0.00 pct) 2412.00 (0.00 pct)
2396.00 (0.66 pct) 2420.00 (-0.33 pct)
16: 4840.00 (0.00 pct) 4824.00 (0.33 pct)
4792.00 (0.99 pct) 4840.00 (0.00 pct)
32: 9648.00 (0.00 pct) 9648.00 (0.00 pct)
9584.00 (0.66 pct) 9680.00 (-0.33 pct)
64: 19360.00 (0.00 pct) 19296.00 (0.33 pct)
19168.00 (0.99 pct) 19296.00 (0.33 pct)128: 37952.00 (0.00 pct) 35264.00 (7.08 pct) 36672.00 (3.37 pct) 38080.00 (-0.33 pct) 256: 41408.00 (0.00 pct) 41536.00 (-0.30 pct) 39744.00 (4.01 pct) 40896.00 (1.23 pct) 512: 36288.00 (0.00 pct) 36800.00 (-1.41 pct) 35264.00 (2.82 pct) 35776.00 (1.41 pct)Tested-by Swapnil Sapkal <[email protected]>Quoted 32 lines
Thanks,
Mathieu
Mathieu Desnoyers (3):
sched: Renamecpus_share_cachetocpus_share_llc
sched: Introducecpus_share_l2c(v3)
sched:ttwu_queue_cond: skip queued wakeups across different l2 cachesCc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Valentin Schneider <[email protected]>Cc Steven Rostedt <[email protected]>Cc Ben Segall <[email protected]>Cc Mel Gorman <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Cc Vincent Guittot <[email protected]>Cc Juri Lelli <[email protected]>Cc Swapnil Sapkal <[email protected]>Cc Aaron Lu <[email protected]>Cc Julien Desfossez <[email protected]>block/blk-mq.c | 2 +-
include/linux/sched/topology.h| 10 ++++++++-- kernel/sched/core.c | 14 +++++++++++--- kernel/sched/fair.c | 8 ++++---- kernel/sched/sched.h | 2 ++ kernel/sched/topology.c | 32 +++++++++++++++++++++++++++++---6 files changed, 55 insertions(+), 13 deletions(-)Signature · 2 lines
Thanks and Regards, Swapnil
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com