Adjust CFS loadbalance to adapt QEMU CPU topology.
cover Adjust CFS loadbalance to adapt QEMU CPU topology.
From: "Kenan.Liu" <[email protected]>
Multithreading workloads in VM with Qemu may encounter an unexpected
phenomenon: one hyperthread of a physical core is busy while its sibling is idle. Such as:
%Cpu0 : 19.8 us, 3.8 sy, 0.0 ni, 76.5 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu1 : 6.4 us, 1.0 sy, 0.0 ni, 92.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu2 : 15.8 us, 4.5 sy, 0.0 ni, 79.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu3 : 4.7 us, 0.7 sy, 0.0 ni, 94.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu4 : 15.5 us, 4.5 sy, 0.0 ni, 80.1 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu5 : 4.7 us, 0.7 sy, 0.0 ni, 94.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu6 : 13.4 us, 3.4 sy, 0.0 ni, 83.2 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu7 : 2.7 us, 0.3 sy, 0.0 ni, 97.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu8 : 16.1 us, 4.8 sy, 0.0 ni, 79.1 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu9 : 2.0 us, 0.3 sy, 0.0 ni, 97.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu10 : 17.5 us, 5.2 sy, 0.0 ni, 77.3 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu11 : 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu12 : 17.6 us, 4.5 sy, 0.0 ni, 77.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu13 : 0.3 us, 0.0 sy, 0.0 ni, 99.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu14 : 16.1 us, 4.1 sy, 0.0 ni, 79.8 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st %Cpu15 : 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
The main reason is that hyperthread index is consecutive in qemu native x86 CPU model which is different from the physical topology. As the current kernel scheduler implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying.
This RFC targets to solve the problem by adjusting CFS loabalance policy:
1. Explore CPU topology and adjust CFS loadbalance policy when we found machine with qemu native CPU topology.
2. Export a procfs to control the traverse length when select idle cpu.
Kenan.Liu (2):
sched/fair: Adjust CFS loadbalance for machine with qemu native CPU
topology.
sched/fair: Export a param to control the traverse len when select
idle cpu.
kernel/sched/fair.c | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.3.1
RFC PATCH 1/2 sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology.
From: "Kenan.Liu" <[email protected]>
Multithreading workloads in VM with Qemu may encounter an unexpected
phenomenon: one hyperthread of a physical core is busy while its sibling is idle. The main reason is that hyperthread index is consecutive in qemu native x86 CPU model which is different from the physical topology. As the current kernel scheduler implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying. To solve the imbalance, when on a machine with multi core and hyperthread index is consecutive per core, change the result of select_idle_core() according to the hyperthread on which the task ran before.
--- kernel/sched/fair.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-)
kernel/sched/fair.c +36 -2
@@ -125,6 +125,9 @@static unsigned int normalized_sysctl_sched_wakeup_granularity =1000000UL ; const_debug unsigned int sysctl_sched_migration_cost = 500000UL; +static bool smt_neighbour_topo; +static bool core_smt_topo_detect; +static unsigned int smt_nr_cpu =2 ;int sched_thermal_decay_shift;static int __init setup_sched_thermal_decay_shift (char *str)
@@ -140,6 +143,26 @@ static int __init setup_sched_thermal_decay_shift(char *str)__setup ("sched_thermal_decay_shift=" , setup_sched_thermal_decay_shift);#ifdef CONFIG_SMP +static void explore_core_smp_topology (void ) +{ +int cpu =smp_processor_id (), sibling; +const struct cpumask *smt_mask =cpu_smt_mask (cpu); + +if (nr_cpu_ids <=2 ) +return ; + + smt_nr_cpu =cpumask_weight (smt_mask); +if (smt_nr_cpu <2 ) +return ; + +for_each_cpu (sibling,cpu_smt_mask (cpu)) { +if (cpu == sibling) +continue ; +if (abs (cpu - sibling) ==1 ) + smt_neighbour_topo = true; + } +} +/* * For asym packing, by default the lower numbered CPU has higher priority. */
@@ -6887,9 +6910,16 @@ void __update_idle_core(struct rq *rq)static int select_idle_core (struct task_struct *p,int core,struct cpumask *cpus,int *idle_cpu) {bool idle = true; -int cpu; +int cpu, sibling = core; + +if (!core_smt_topo_detect) { +explore_core_smp_topology (); + core_smt_topo_detect = true; + }for_each_cpu (cpu,cpu_smt_mask (core)) { +if (cpu != core) + sibling = cpu;if (!available_idle_cpu (cpu)) { idle = false;if (*idle_cpu ==-1 ) {
@@ -6905,8 +6935,12 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu *idle_cpu = cpu; } -if (idle) +if (idle) { +if (!smt_neighbour_topo ||unlikely (core % smt_nr_cpu)) +return core; + core =task_cpu (p) % smt_nr_cpu ? core : sibling;return core; + }cpumask_andnot (cpus, cpus,cpu_smt_mask (core));return -1 ;
Signature · 1 line
1.8.3.1
kernel test robot
build botRe: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology.
kernel test robot noticed the following build warnings:
Hi Kenan.Liu,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:
[auto build test WARNING on tip/sched/core]
[also build test WARNING on linus/master v6.5-rc2 next-20230720] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kenan-Liu/sched-fair-Adjust-CFS-loadbalance-for-machine-with-qemu-native-CPU-topology/20230720-170238 base: tip/sched/core patch link: https://lore.kernel.org/r/1689842053-5291-2-git-send-email-Kenan.Liu%40linux.alibaba.com patch subject: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology. config: alpha-allyesconfig (https://download.01.org/0day-ci/archive/20230720/[email protected]/config) compiler: alpha-linux-gcc (GCC) 12.3.0 reproduce: (https://download.01.org/0day-ci/archive/20230720/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
kernel/sched/fair.c: In function 'explore_core_smp_topology':
kernel/sched/fair.c:149:42: error: implicit declaration of function 'cpu_smt_mask'; did you mean 'cpu_cpu_mask'? [-Werror=implicit-function-declaration]
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^~~~~~~~~~~~
| cpu_cpu_mask
>> kernel/sched/fair.c:149:42: warning: initialization of 'const struct cpumask *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
In file included from include/linux/bitmap.h:9,
from include/linux/cpumask.h:12,
from include/linux/energy_model.h:4,
from kernel/sched/fair.c:23:
include/linux/cpumask.h:28:37: error: invalid type argument of '->' (have 'int')
28 | #define cpumask_bits(maskp) ((maskp)->bits)
| ^~
include/linux/find.h:559:48: note: in definition of macro 'for_each_set_bit'
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^~~~
include/linux/cpumask.h:282:31: note: in expansion of macro 'cpumask_bits'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
include/linux/find.h:559:69: warning: left-hand operand of comma expression has no effect [-Wunused-value]
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^
include/linux/cpumask.h:282:9: note: in expansion of macro 'for_each_set_bit'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
kernel/sched/fair.c: At top level:
kernel/sched/fair.c:146:13: warning: 'explore_core_smp_topology' defined but not used [-Wunused-function]
146 | static void explore_core_smp_topology(void)
| ^~~~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:129:13: warning: 'core_smt_topo_detect' defined but not used [-Wunused-variable]
129 | static bool core_smt_topo_detect;
| ^~~~~~~~~~~~~~~~~~~~cc1: some warnings being treated as errors
vim +149 kernel/sched/fair.c
144
145 #ifdef CONFIG_SMP
146 static void explore_core_smp_topology(void)
147 {
148 int cpu = smp_processor_id(), sibling;
> 149 const struct cpumask *smt_mask = cpu_smt_mask(cpu);
150
151 if (nr_cpu_ids <= 2)
152 return;
153
154 smt_nr_cpu = cpumask_weight(smt_mask);
155 if (smt_nr_cpu < 2)
156 return;
157
158 for_each_cpu(sibling, cpu_smt_mask(cpu)) {
159 if (cpu == sibling)
160 continue;
161 if (abs(cpu - sibling) == 1)
162 smt_neighbour_topo = true;
163 }
164 }
165
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology.
In reply to Kenan.Liukernel test robot noticed the following build warnings:
Hi Kenan.Liu,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:
[auto build test WARNING on tip/sched/core]
[also build test WARNING on linus/master v6.5-rc2 next-20230720] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kenan-Liu/sched-fair-Adjust-CFS-loadbalance-for-machine-with-qemu-native-CPU-topology/20230720-170238 base: tip/sched/core patch link: https://lore.kernel.org/r/1689842053-5291-2-git-send-email-Kenan.Liu%40linux.alibaba.com patch subject: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology. config: powerpc-allyesconfig (https://download.01.org/0day-ci/archive/20230720/[email protected]/config) compiler: powerpc-linux-gcc (GCC) 12.3.0 reproduce: (https://download.01.org/0day-ci/archive/20230720/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
In file included from include/linux/smp.h:113,
from include/linux/lockdep.h:14,
from include/linux/rcupdate.h:29,
from include/linux/rculist.h:11,
from include/linux/pid.h:5,
from include/linux/sched.h:14,
from include/linux/ratelimit.h:6,
from include/linux/dev_printk.h:16,
from include/linux/device.h:15,
from include/linux/energy_model.h:5,
from kernel/sched/fair.c:23:
kernel/sched/fair.c: In function 'explore_core_smp_topology':
arch/powerpc/include/asm/smp.h:139:22: error: implicit declaration of function 'cpu_smt_mask'; did you mean 'cpu_cpu_mask'? [-Werror=implicit-function-declaration]
139 | #define cpu_smt_mask cpu_smt_mask
| ^~~~~~~~~~~~
kernel/sched/fair.c:149:42: note: in expansion of macro 'cpu_smt_mask'
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^~~~~~~~~~~~
>> arch/powerpc/include/asm/smp.h:139:22: warning: initialization of 'const struct cpumask *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
139 | #define cpu_smt_mask cpu_smt_mask
| ^~~~~~~~~~~~
kernel/sched/fair.c:149:42: note: in expansion of macro 'cpu_smt_mask'
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^~~~~~~~~~~~
In file included from include/linux/bitmap.h:9,
from include/linux/cpumask.h:12,
from include/linux/energy_model.h:4:
include/linux/cpumask.h:28:37: error: invalid type argument of '->' (have 'int')
28 | #define cpumask_bits(maskp) ((maskp)->bits)
| ^~
include/linux/find.h:559:48: note: in definition of macro 'for_each_set_bit'
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^~~~
include/linux/cpumask.h:282:31: note: in expansion of macro 'cpumask_bits'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
include/linux/find.h:559:69: warning: left-hand operand of comma expression has no effect [-Wunused-value]
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^
include/linux/cpumask.h:282:9: note: in expansion of macro 'for_each_set_bit'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
kernel/sched/fair.c: At top level:
kernel/sched/fair.c:146:13: warning: 'explore_core_smp_topology' defined but not used [-Wunused-function]
146 | static void explore_core_smp_topology(void)
| ^~~~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:129:13: warning: 'core_smt_topo_detect' defined but not used [-Wunused-variable]
129 | static bool core_smt_topo_detect;
| ^~~~~~~~~~~~~~~~~~~~cc1: some warnings being treated as errors
vim +139 arch/powerpc/include/asm/smp.h
f3232321db5848 Srikar Dronamraju 2020-08-07 138
f3232321db5848 Srikar Dronamraju 2020-08-07 @139 #define cpu_smt_mask cpu_smt_mask
f3232321db5848 Srikar Dronamraju 2020-08-07 140 #ifdef CONFIG_SCHED_SMT
f3232321db5848 Srikar Dronamraju 2020-08-07 141 static inline const struct cpumask *cpu_smt_mask(int cpu)
f3232321db5848 Srikar Dronamraju 2020-08-07 142 {
f3232321db5848 Srikar Dronamraju 2020-08-07 143 if (has_big_cores)
f3232321db5848 Srikar Dronamraju 2020-08-07 144 return per_cpu(cpu_smallcore_map, cpu);
f3232321db5848 Srikar Dronamraju 2020-08-07 145
f3232321db5848 Srikar Dronamraju 2020-08-07 146 return per_cpu(cpu_sibling_map, cpu);
f3232321db5848 Srikar Dronamraju 2020-08-07 147 }
f3232321db5848 Srikar Dronamraju 2020-08-07 148 #endif /* CONFIG_SCHED_SMT */
f3232321db5848 Srikar Dronamraju 2020-08-07 149 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology.
In reply to Kenan.Liukernel test robot noticed the following build errors:
Hi Kenan.Liu,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on linus/master v6.5-rc2 next-20230720] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kenan-Liu/sched-fair-Adjust-CFS-loadbalance-for-machine-with-qemu-native-CPU-topology/20230720-170238 base: tip/sched/core patch link: https://lore.kernel.org/r/1689842053-5291-2-git-send-email-Kenan.Liu%40linux.alibaba.com patch subject: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology. config: hexagon-randconfig-r036-20230720 (https://download.01.org/0day-ci/archive/20230720/[email protected]/config) compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a) reproduce: (https://download.01.org/0day-ci/archive/20230720/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from kernel/sched/fair.c:28:
In file included from include/linux/highmem.h:12:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/hexagon/include/asm/io.h:334:
include/asm-generic/io.h:547:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
547 | val = __raw_readb(PCI_IOBASE + addr);
| ~~~~~~~~~~ ^
include/asm-generic/io.h:560:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
560 | val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
| ~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:37:51: note: expanded from macro '__le16_to_cpu'
37 | #define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
| ^
In file included from kernel/sched/fair.c:28:
In file included from include/linux/highmem.h:12:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/hexagon/include/asm/io.h:334:
include/asm-generic/io.h:573:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
573 | val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
| ~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:35:51: note: expanded from macro '__le32_to_cpu'
35 | #define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
| ^
In file included from kernel/sched/fair.c:28:
In file included from include/linux/highmem.h:12:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/hexagon/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/hexagon/include/asm/io.h:334:
include/asm-generic/io.h:584:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
584 | __raw_writeb(value, PCI_IOBASE + addr);
| ~~~~~~~~~~ ^
include/asm-generic/io.h:594:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
594 | __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
| ~~~~~~~~~~ ^
include/asm-generic/io.h:604:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
604 | __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
| ~~~~~~~~~~ ^
>> kernel/sched/fair.c:149:35: error: call to undeclared function 'cpu_smt_mask'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^
kernel/sched/fair.c:149:35: note: did you mean 'cpu_cpu_mask'?
include/linux/topology.h:243:37: note: 'cpu_cpu_mask' declared here
243 | static inline const struct cpumask *cpu_cpu_mask(int cpu)
| ^
>> kernel/sched/fair.c:149:24: error: incompatible integer to pointer conversion initializing 'const struct cpumask *' with an expression of type 'int' [-Wint-conversion]
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^ ~~~~~~~~~~~~~~~~~
>> kernel/sched/fair.c:158:2: error: member reference type 'int' is not a pointer
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/cpumask.h:282:24: note: expanded from macro 'for_each_cpu'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/cpumask.h:28:39: note: expanded from macro 'cpumask_bits'
28 | #define cpumask_bits(maskp) ((maskp)->bits)
| ^
include/linux/find.h:559:41: note: expanded from macro 'for_each_set_bit'
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^~~~
kernel/sched/fair.c:6243:6: warning: no previous prototype for function 'init_cfs_bandwidth' [-Wmissing-prototypes]
6243 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
| ^
kernel/sched/fair.c:6243:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
6243 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
| ^
| static
kernel/sched/fair.c:12868:6: warning: no previous prototype for function 'free_fair_sched_group' [-Wmissing-prototypes]
12868 | void free_fair_sched_group(struct task_group *tg) { }
| ^
kernel/sched/fair.c:12868:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
12868 | void free_fair_sched_group(struct task_group *tg) { }
| ^
| static
kernel/sched/fair.c:12870:5: warning: no previous prototype for function 'alloc_fair_sched_group' [-Wmissing-prototypes]
12870 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
| ^
kernel/sched/fair.c:12870:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
12870 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
| ^
| static
kernel/sched/fair.c:12875:6: warning: no previous prototype for function 'online_fair_sched_group' [-Wmissing-prototypes]
12875 | void online_fair_sched_group(struct task_group *tg) { }
| ^
kernel/sched/fair.c:12875:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
12875 | void online_fair_sched_group(struct task_group *tg) { }
| ^
| static
kernel/sched/fair.c:12877:6: warning: no previous prototype for function 'unregister_fair_sched_group' [-Wmissing-prototypes]
12877 | void unregister_fair_sched_group(struct task_group *tg) { }
| ^
kernel/sched/fair.c:12877:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
12877 | void unregister_fair_sched_group(struct task_group *tg) { }
| ^
| static 11 warnings and 3 errors generated.
vim +/cpu_smt_mask +149 kernel/sched/fair.c
144
145 #ifdef CONFIG_SMP
146 static void explore_core_smp_topology(void)
147 {
148 int cpu = smp_processor_id(), sibling;
> 149 const struct cpumask *smt_mask = cpu_smt_mask(cpu);
150
151 if (nr_cpu_ids <= 2)
152 return;
153
154 smt_nr_cpu = cpumask_weight(smt_mask);
155 if (smt_nr_cpu < 2)
156 return;
157
> 158 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 159 if (cpu == sibling)
160 continue;
161 if (abs(cpu - sibling) == 1)
162 smt_neighbour_topo = true;
163 }
164 }
165
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology.
In reply to Kenan.Liukernel test robot noticed the following build errors:
Hi Kenan.Liu,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on linus/master v6.5-rc2 next-20230720] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kenan-Liu/sched-fair-Adjust-CFS-loadbalance-for-machine-with-qemu-native-CPU-topology/20230720-170238 base: tip/sched/core patch link: https://lore.kernel.org/r/1689842053-5291-2-git-send-email-Kenan.Liu%40linux.alibaba.com patch subject: [RFC PATCH 1/2] sched/fair: Adjust CFS loadbalance for machine with qemu native CPU topology. config: powerpc-allyesconfig (https://download.01.org/0day-ci/archive/20230720/[email protected]/config) compiler: powerpc-linux-gcc (GCC) 12.3.0 reproduce: (https://download.01.org/0day-ci/archive/20230720/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from include/linux/smp.h:113,
from include/linux/lockdep.h:14, from include/linux/rcupdate.h:29, from include/linux/rculist.h:11, from include/linux/pid.h:5,
from include/linux/sched.h:14, from include/linux/ratelimit.h:6, from include/linux/dev_printk.h:16, from include/linux/device.h:15, from include/linux/energy_model.h:5, from kernel/sched/fair.c:23:
kernel/sched/fair.c: In function 'explore_core_smp_topology':
>> arch/powerpc/include/asm/smp.h:139:22: error: implicit declaration of function 'cpu_smt_mask'; did you mean 'cpu_cpu_mask'? [-Werror=implicit-function-declaration]
139 | #define cpu_smt_mask cpu_smt_mask
| ^~~~~~~~~~~~
kernel/sched/fair.c:149:42: note: in expansion of macro 'cpu_smt_mask'
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^~~~~~~~~~~~
arch/powerpc/include/asm/smp.h:139:22: warning: initialization of 'const struct cpumask *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
139 | #define cpu_smt_mask cpu_smt_mask
| ^~~~~~~~~~~~
kernel/sched/fair.c:149:42: note: in expansion of macro 'cpu_smt_mask'
149 | const struct cpumask *smt_mask = cpu_smt_mask(cpu);
| ^~~~~~~~~~~~
In file included from include/linux/bitmap.h:9,
from include/linux/cpumask.h:12,
from include/linux/energy_model.h:4:
include/linux/cpumask.h:28:37: error: invalid type argument of '->' (have 'int')
28 | #define cpumask_bits(maskp) ((maskp)->bits)
| ^~
include/linux/find.h:559:48: note: in definition of macro 'for_each_set_bit'
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^~~~
include/linux/cpumask.h:282:31: note: in expansion of macro 'cpumask_bits'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
include/linux/find.h:559:69: warning: left-hand operand of comma expression has no effect [-Wunused-value]
559 | for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
| ^
include/linux/cpumask.h:282:9: note: in expansion of macro 'for_each_set_bit'
282 | for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
| ^~~~~~~~~~~~~~~~
kernel/sched/fair.c:158:9: note: in expansion of macro 'for_each_cpu'
158 | for_each_cpu(sibling, cpu_smt_mask(cpu)) {
| ^~~~~~~~~~~~
kernel/sched/fair.c: At top level:
kernel/sched/fair.c:146:13: warning: 'explore_core_smp_topology' defined but not used [-Wunused-function]
146 | static void explore_core_smp_topology(void)
| ^~~~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:129:13: warning: 'core_smt_topo_detect' defined but not used [-Wunused-variable]
129 | static bool core_smt_topo_detect;
| ^~~~~~~~~~~~~~~~~~~~cc1: some warnings being treated as errors
vim +139 arch/powerpc/include/asm/smp.h
f3232321db5848 Srikar Dronamraju 2020-08-07 138
f3232321db5848 Srikar Dronamraju 2020-08-07 @139 #define cpu_smt_mask cpu_smt_mask
f3232321db5848 Srikar Dronamraju 2020-08-07 140 #ifdef CONFIG_SCHED_SMT
f3232321db5848 Srikar Dronamraju 2020-08-07 141 static inline const struct cpumask *cpu_smt_mask(int cpu)
f3232321db5848 Srikar Dronamraju 2020-08-07 142 {
f3232321db5848 Srikar Dronamraju 2020-08-07 143 if (has_big_cores)
f3232321db5848 Srikar Dronamraju 2020-08-07 144 return per_cpu(cpu_smallcore_map, cpu);
f3232321db5848 Srikar Dronamraju 2020-08-07 145
f3232321db5848 Srikar Dronamraju 2020-08-07 146 return per_cpu(cpu_sibling_map, cpu);
f3232321db5848 Srikar Dronamraju 2020-08-07 147 }
f3232321db5848 Srikar Dronamraju 2020-08-07 148 #endif /* CONFIG_SCHED_SMT */
f3232321db5848 Srikar Dronamraju 2020-08-07 149 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Quoted 5 lines · Kenan.Liu
From: "Kenan.Liu" <[email protected]>
Multithreading workloads in VM with Qemu may encounter an unexpected
phenomenon: one hyperthread of a physical core is busy while its sibling is idle. Such as:
Is this with vCPU pinning? Without that, guest topology makes no sense what so ever.
Quoted 2 lines
The main reason is that hyperthread index is consecutive in qemu native x86 CPU model which is different from the physical topology.
I'm sorry, what? That doesn't make sense. SMT enumeration is all over the place for Intel, but some actually do have (n,n+1) SMT. On AMD it's always (n,n+1) IIRC.
Quoted 3 lines
As the current kernel scheduler
implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying.
How so?
Quoted 10 lines
This RFC targets to solve the problem by adjusting CFS loabalance policy:
1. Explore CPU topology and adjust CFS loadbalance policy when we found machine with qemu native CPU topology.
2. Export a procfs to control the traverse length when select idle cpu.Kenan.Liu (2):
sched/fair: Adjust CFS loadbalance for machine with qemu native CPU
topology.
sched/fair: Export a param to control the traverse len when select
idle cpu.
NAK, qemu can either provide a fake topology to the guest using normal x86 means (MADT/CPUID) or do some paravirt topology setup, but this is quite insane.
Hi Peter, thanks for your attention,
please refer to my answer to your question inline:
Quoted 8 lines · Peter Zijlstra
Quoted 5 lines · Kenan.Liu
From: "Kenan.Liu" <[email protected]>
Multithreading workloads in VM with Qemu may encounter an unexpected
phenomenon: one hyperthread of a physical core is busy while its sibling is idle. Such as:Is this with vCPU pinning? Without that, guest topology makes no sense what so ever.
vCPU is pinned on host and the imbalance phenomenon we observed is inside VM, not for the vCPU threads on host.
Quoted 10 lines
Quoted 2 lines
The main reason is that hyperthread index is consecutive in qemu native x86 CPU model which is different from the physical topology.
I'm sorry, what? That doesn't make sense. SMT enumeration is all over the place for Intel, but some actually do have (n,n+1) SMT. On AMD it's always (n,n+1) IIRC.
Quoted 3 lines
As the current kernel scheduler
implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying.How so?
The SMT topology in qemu native x86 CPU model is (0,1),…,(n,n+1),…, but nomarlly seen SMT topo in physical machine is like (0,n),(1,n+1),…, n means the total core number of the machine.
The imbalance happens when the number of runnable threads is less than the number of hyperthreads, select_idle_core() would be called to decide which cpu be selected to run the waken-up task.
select_idle_core() will return the checked cpu number if the whole core is idle. On the contrary, if any one HT of the core is busy, select_idle_core() would clear the whole core out from cpumask and check the next core.
select_idle_core():
…
if (idle)
return core;
cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); return -1;
In this manner, except the very beginning of for_each_cpu_wrap() loop, HT with even ID number is always be checked at first, and be returned to the caller if the whole core is idle, so the odd indexed HT almost has no chance to be selected.
select_idle_cpu():
…
for_each_cpu_wrap(cpu, cpus, target + 1) {
if (has_idle_core) {
i = select_idle_core(p, cpu, cpus, &idle_cpu);
And this will NOT happen when the SMT topo is (0,n),(1,n+1),…, because when the loop starts from the bottom half of SMT number, HTs with larger number will be checked first, when it starts from the top half, their siblings with smaller number take the first place of inner core searching.
Quoted 6 lines
Quoted 6 lines
This RFC targets to solve the problem by adjusting CFS loabalance policy:
1. Explore CPU topology and adjust CFS loadbalance policy when we found machine with qemu native CPU topology.
2. Export a procfs to control the traverse length when select idle cpu.
Quoted 8 lines
Quoted 5 lines
Kenan.Liu (2):
sched/fair: Adjust CFS loadbalance for machine with qemu native CPU
topology.
sched/fair: Export a param to control the traverse len when select
idle cpu.NAK, qemu can either provide a fake topology to the guest using normal x86 means (MADT/CPUID) or do some paravirt topology setup, but this is quite insane.
Thanks,
Kenan.Liu
Vincent Guittot
maintainerQuoted 65 lines · Kenan.Liu
Hi Peter, thanks for your attention,
please refer to my answer to your question inline:
Quoted 8 lines · Peter Zijlstra
Quoted 5 lines · Kenan.Liu
From: "Kenan.Liu" <[email protected]>
Multithreading workloads in VM with Qemu may encounter an unexpected
phenomenon: one hyperthread of a physical core is busy while its sibling is idle. Such as:Is this with vCPU pinning? Without that, guest topology makes no sense what so ever.
vCPU is pinned on host and the imbalance phenomenon we observed is inside VM, not for the vCPU threads on host.
Quoted 10 lines
Quoted 2 lines
The main reason is that hyperthread index is consecutive in qemu native x86 CPU model which is different from the physical topology.
I'm sorry, what? That doesn't make sense. SMT enumeration is all over the place for Intel, but some actually do have (n,n+1) SMT. On AMD it's always (n,n+1) IIRC.
Quoted 3 lines
As the current kernel scheduler
implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying.How so?
The SMT topology in qemu native x86 CPU model is (0,1),…,(n,n+1),…, but nomarlly seen SMT topo in physical machine is like (0,n),(1,n+1),…, n means the total core number of the machine.
The imbalance happens when the number of runnable threads is less than the number of hyperthreads,
select_idle_core()would be called to decide which cpu be selected to run the waken-up task.
select_idle_core()will return the checked cpu number if the whole core is idle. On the contrary, if any one HT of the core is busy,select_idle_core()would clear the whole core out from cpumask and check the next core.
select_idle_core():… if (idle) return core; cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); return -1;In this manner, except the very beginning of
for_each_cpu_wrap()loop, HT with even ID number is always be checked at first, and be returned to the caller if the whole core is idle, so the odd indexed HT almost has no chance to be selected.
select_idle_cpu():… for_each_cpu_wrap(cpu, cpus, target + 1) { if (has_idle_core) { i = select_idle_core(p, cpu, cpus, &idle_cpu);
Quoted 4 lines
And this will NOT happen when the SMT topo is (0,n),(1,n+1),…, because when the loop starts from the bottom half of SMT number, HTs with larger number will be checked first, when it starts from the top half, their siblings with smaller number take the first place of inner core searching.
But why is it a problem ? Your system is almost idle and 1 HT per core is used. Who cares to select evenly one HT or the other as long as we select an idle core in priority ?
This seems related to
https://lore.kernel.org/lkml/BYAPR21MB1688FE804787663C425C2202D753A@BYAPR21MB1688.namprd21.prod.outlook.com/
we concluded that it was not a problem
Quoted 19 lines
Quoted 8 lines
Quoted 8 lines
This RFC targets to solve the problem by adjusting CFS loabalance policy:
1. Explore CPU topology and adjust CFS loadbalance policy when we found machine with qemu native CPU topology.
2. Export a procfs to control the traverse length when select idle cpu.Quoted 8 lines
Quoted 5 lines
Kenan.Liu (2):
sched/fair: Adjust CFS loadbalance for machine with qemu native CPU
topology.
sched/fair: Export a param to control the traverse len when select
idle cpu.NAK, qemu can either provide a fake topology to the guest using normal x86 means (MADT/CPUID) or do some paravirt topology setup, but this is quite insane.
Thanks,
Kenan.Liu
Peter Zijlstra
maintainerQuoted 34 lines · Vincent Guittot
Quoted 32 lines · Kenan.Liu
The SMT topology in qemu native x86 CPU model is (0,1),…,(n,n+1),…, but nomarlly seen SMT topo in physical machine is like (0,n),(1,n+1),…, n means the total core number of the machine.
The imbalance happens when the number of runnable threads is less than the number of hyperthreads,
select_idle_core()would be called to decide which cpu be selected to run the waken-up task.
select_idle_core()will return the checked cpu number if the whole core is idle. On the contrary, if any one HT of the core is busy,select_idle_core()would clear the whole core out from cpumask and check the next core.
select_idle_core():… if (idle) return core; cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); return -1;In this manner, except the very beginning of
for_each_cpu_wrap()loop, HT with even ID number is always be checked at first, and be returned to the caller if the whole core is idle, so the odd indexed HT almost has no chance to be selected.
select_idle_cpu():… for_each_cpu_wrap(cpu, cpus, target + 1) { if (has_idle_core) { i = select_idle_core(p, cpu, cpus, &idle_cpu);
Quoted 8 lines
Quoted 4 lines
And this will NOT happen when the SMT topo is (0,n),(1,n+1),…, because when the loop starts from the bottom half of SMT number, HTs with larger number will be checked first, when it starts from the top half, their siblings with smaller number take the first place of inner core searching.
But why is it a problem ? Your system is almost idle and 1 HT per core is used. Who cares to select evenly one HT or the other as long as we select an idle core in priority ?
Right, why is this a problem? Hyperthreads are supposed to be symmetric, it doesn't matter which of the two are active, the important thing is to only have one active if we can.
(Unlike Power7, they have asymmetric SMT)
Quoted 44 lines · Peter Zijlstra
Quoted 34 lines · Vincent Guittot
Quoted 32 lines · Kenan.Liu
The SMT topology in qemu native x86 CPU model is (0,1),…,(n,n+1),…, but nomarlly seen SMT topo in physical machine is like (0,n),(1,n+1),…, n means the total core number of the machine.
The imbalance happens when the number of runnable threads is less than the number of hyperthreads,
select_idle_core()would be called to decide which cpu be selected to run the waken-up task.
select_idle_core()will return the checked cpu number if the whole core is idle. On the contrary, if any one HT of the core is busy,select_idle_core()would clear the whole core out from cpumask and check the next core.
select_idle_core():… if (idle) return core; cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); return -1;In this manner, except the very beginning of
for_each_cpu_wrap()loop, HT with even ID number is always be checked at first, and be returned to the caller if the whole core is idle, so the odd indexed HT almost has no chance to be selected.
select_idle_cpu():… for_each_cpu_wrap(cpu, cpus, target + 1) { if (has_idle_core) { i = select_idle_core(p, cpu, cpus, &idle_cpu);Quoted 8 lines
Quoted 4 lines
And this will NOT happen when the SMT topo is (0,n),(1,n+1),…, because when the loop starts from the bottom half of SMT number, HTs with larger number will be checked first, when it starts from the top half, their siblings with smaller number take the first place of inner core searching.
But why is it a problem ? Your system is almost idle and 1 HT per core is used. Who cares to select evenly one HT or the other as long as we select an idle core in priority ?
Quoted 6 lines
Right, why is this a problem? Hyperthreads are supposed to be symmetric, it doesn't matter which of the two are active, the important thing is to only have one active if we can.
(Unlike Power7, they have asymmetric SMT)
hi Peter and Vincent,
Some upper-level monitoring logic may take the CPU usage as a metric for computing resource scaling. Imbalanced scheduling can create the illusion of CPU resource scarcity, leading to more frequent triggering of resource expansion by the upper-level scheduling system. However, this is actually a waste of resources. So, we think this may be a problem.
Could you please take a further look at PATCH#2? We found that the default 'nr' value did not perform well under our scenario, and we believe that adjustable variables would be more appropriate.
Our scenario is as follows:
16 processes are running in a 32 CPU VM, with 8 threads per process, they are all running the same job.
The expected result is that the CPU usage is evenly distributed, but we found that even-numbered cores were used for scheduling decisions and consumed more CPU resources (5%~20%), mainly because of the default value of nr=4. In this scenario, we found that nr=2 is more suitable.
Thanks,
Ben
Peter Zijlstra
maintainerQuoted 7 lines · On Mon, Jul 24, 2023 at 02:57:57PM +0800, [email protected] wrote:
Quoted 6 lines
Right, why is this a problem? Hyperthreads are supposed to be symmetric, it doesn't matter which of the two are active, the important thing is to only have one active if we can.
(Unlike Power7, they have asymmetric SMT)
Quoted 7 lines
hi Peter and Vincent,
Some upper-level monitoring logic may take the CPU usage as a metric for computing resource scaling. Imbalanced scheduling can create the illusion of CPU resource scarcity, leading to more frequent triggering of resource expansion by the upper-level scheduling system. However, this is actually a waste of resources. So, we think this may be a problem.
This is a problem of your monitoring logic -- there is absolutely no functional problem with the kernel AFAICT.
Teach the thing about SMT instead.
Quoted 3 lines
Could you please take a further look at PATCH#2? We found that the default 'nr' value did not perform well under our scenario, and we believe that adjustable variables would be more appropriate.
That patch is tweaking default disabled code -- which we should be removing sometime soon I suppose. So no.
Peter Zijlstra
maintainerQuoted 9 lines · Kenan.Liu
Quoted 4 lines
Quoted 3 lines
As the current kernel scheduler
implementation, hyperthread with an even ID number will be picked up in a much higher probability during load-balancing and load-deploying.How so?
The SMT topology in qemu native x86 CPU model is (0,1),…,(n,n+1),…, but nomarlly seen SMT topo in physical machine is like (0,n),(1,n+1),…, n means the total core number of the machine.
That is only common on Intel hardware, AMD (and some Intel) will in fact enumerate SMT like Qemu does.
RFC PATCH 2/2 sched/fair: Export a param to control the traverse len when select idle cpu.
In reply to Kenan.LiuFrom: "Kenan.Liu" <[email protected]>
The variable 'nr' decides the length of traverse when we try to find an idle cpu in function select_idle_cpu(). A fixed value such as 4 may not perform well in all scenes and may lead to un-acceptable overhead. Export two sysctl parameters to enable adjustments.
--- kernel/sched/fair.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-)
kernel/sched/fair.c +63 -2
@@ -125,6 +125,9 @@static unsigned int normalized_sysctl_sched_wakeup_granularity =1000000UL ; const_debug unsigned int sysctl_sched_migration_cost = 500000UL; +unsigned int __read_mostly sysctl_sched_idle_search_nr_default =4 ; +unsigned int __read_mostly sysctl_sched_idle_search_nr_threshold =4 ; +static bool smt_neighbour_topo;static bool core_smt_topo_detect;static unsigned int smt_nr_cpu =2 ;
@@ -207,6 +210,50 @@ int __weak arch_asym_cpu_priority(int cpu)#endif #ifdef CONFIG_SYSCTL +static int sched_set_idle_search_nr_default(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + static DEFINE_MUTEX(mutex); + unsigned int old_nr; + int ret; + + mutex_lock(&mutex); + old_nr = sysctl_sched_idle_search_nr_default; + ret = proc_douintvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + if (sysctl_sched_idle_search_nr_default == 0) { + sysctl_sched_idle_search_nr_default = old_nr; + mutex_unlock(&mutex); + return -EINVAL; + } + } + + mutex_unlock(&mutex); + return ret; +} + +static int sched_set_idle_search_nr_threshold(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + static DEFINE_MUTEX(mutex); + unsigned int old_threshold; + int ret; + + mutex_lock(&mutex); + old_threshold = sysctl_sched_idle_search_nr_threshold; + ret = proc_douintvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + if (sysctl_sched_idle_search_nr_threshold == 0) { + sysctl_sched_idle_search_nr_threshold = old_threshold; + mutex_unlock(&mutex); + return -EINVAL; + } + } + + mutex_unlock(&mutex); + return ret; +} + static struct ctl_table sched_fair_sysctls[] = { { .procname = "sched_child_runs_first",
@@ -235,6 +282,20 @@ int __weak arch_asym_cpu_priority(int cpu) .extra1 = SYSCTL_ZERO, },#endif /* CONFIG_NUMA_BALANCING */ + { + .procname = "sched_cfs_idle_search_nr_default", + .data = &sysctl_sched_idle_search_nr_default, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_set_idle_search_nr_default, + }, + { + .procname = "sched_cfs_idle_search_nr_threshold", + .data = &sysctl_sched_idle_search_nr_threshold, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_set_idle_search_nr_threshold, + }, {} };
@@ -7027,10 +7088,10 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool avg_cost = this_sd->avg_scan_cost +1 ; span_avg = sd->span_weight * avg_idle; -if (span_avg >4 *avg_cost) +if (span_avg > sysctl_sched_idle_search_nr_threshold * avg_cost) nr =div_u64 (span_avg, avg_cost);else - nr =4 ; + nr = sysctl_sched_idle_search_nr_default; time =cpu_clock (this); }
Signature · 1 line
1.8.3.1