sched: Make select_idle_sibling search domain configurable
Subsystems (6)
PATCH 1/1 sched: Make select_idle_sibling search domain configurable
The scope of select_idle_sibling idle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as the sd_llc is only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.
With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:
sysctl_sched_wake_idle_domain == 2 ("MC" domain) sysctl_sched_wake_idle_presearch_domain == 1 ("DIE" domain)
Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.
Heuristics including sd->avg_scan_cost and sds->have_idle_cores are only active for the main search.
On a 128 core (2 socket * 64 core, 256 hw threads) AMD machine ran hackbench as
hackbench -g 20 -f 20 --loops 10000
A snapshot of run time was
Baseline: 11.8
With the patch: 7.6 (configured as in the example above)
--- block/blk-mq.c | 2 +- block/blk-softirq.c | 2 +- include/linux/cpuset.h | 10 +- include/linux/sched/topology.h | 11 +- kernel/cgroup/cpuset.c | 32 ++++-- kernel/sched/core.c | 10 +- kernel/sched/fair.c | 191 +++++++++++++++++++++------------ kernel/sched/sched.h | 9 +- kernel/sched/topology.c | 87 ++++++++++----- kernel/sysctl.c | 25 +++++ 10 files changed, 256 insertions(+), 123 deletions(-)
block/blk-mq.c +1 -1
@@ -626,7 +626,7 @@ void blk_mq_force_complete_rq(struct request *rq) cpu =get_cpu ();if (!test_bit (QUEUE_FLAG_SAME_FORCE , &q->queue_flags)) - shared =cpus_share_cache (cpu, ctx->cpu); + shared =cpus_share_sis (cpu, ctx->cpu);if (cpu != ctx->cpu && !shared &&cpu_online (ctx->cpu)) { rq->csd.func = __blk_mq_complete_request_remote;
block/blk-softirq.c +1 -1
@@ -108,7 +108,7 @@ void __blk_complete_request(struct request *req)*/ if (test_bit (QUEUE_FLAG_SAME_COMP , &q->queue_flags) && ccpu !=-1 ) {if (!test_bit (QUEUE_FLAG_SAME_FORCE , &q->queue_flags)) - shared =cpus_share_cache (cpu, ccpu); + shared =cpus_share_sis (cpu, ccpu); }else ccpu = cpu;
include/linux/cpuset.h +8 -2
@@ -117,6 +117,7 @@ static inline int cpuset_do_slab_mem_spread(void)extern bool current_cpuset_is_being_rebound (void );extern void rebuild_sched_domains (void ); +extern void rebuild_sched_domains_force (void );extern void cpuset_print_current_mems_allowed (void );
@@ -173,7 +174,7 @@ static inline void cpuset_force_rebuild(void) { }static inline void cpuset_update_active_cpus (void ) { -partition_sched_domains (1 ,NULL ,NULL ); +partition_sched_domains (1 ,NULL ,NULL ,0 ); }static inline void cpuset_wait_for_hotplug (void ) { }
@@ -259,7 +260,12 @@ static inline bool current_cpuset_is_being_rebound(void)static inline void rebuild_sched_domains (void ) { -partition_sched_domains (1 ,NULL ,NULL ); +partition_sched_domains (1 ,NULL ,NULL ,0 ); +} + +static inline void rebuild_sched_domains_force (void ) +{ +partition_sched_domains (1 ,NULL ,NULL ,1 ); }static inline void cpuset_print_current_mems_allowed (void )
include/linux/sched/topology.h +6 -5
@@ -151,16 +151,17 @@ static inline struct cpumask *sched_domain_span(struct sched_domain *sd)extern void partition_sched_domains_locked (int ndoms_new,cpumask_var_t doms_new[], -struct sched_domain_attr *dattr_new); +struct sched_domain_attr *dattr_new, +int force_update);extern void partition_sched_domains (int ndoms_new,cpumask_var_t doms_new[], -struct sched_domain_attr *dattr_new); +struct sched_domain_attr *dattr_new,int force_update);/* Allocate an array of sched domains, for partition_sched_domains(). */ cpumask_var_t *alloc_sched_domains (unsigned int ndoms);void free_sched_domains (cpumask_var_t doms[],unsigned int ndoms); -bool cpus_share_cache (int this_cpu,int that_cpu); +bool cpus_share_sis (int this_cpu,int that_cpu);typedef const struct cpumask *(*sched_domain_mask_f )(int cpu);typedef int (*sched_domain_flags_f )(void );
@@ -199,7 +200,7 @@ struct sched_domain_attr;static inline void partition_sched_domains_locked (int ndoms_new,cpumask_var_t doms_new[], -struct sched_domain_attr *dattr_new) +struct sched_domain_attr *dattr_new,int force_update) { }
@@ -209,7 +210,7 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], { } -static inline bool cpus_share_cache (int this_cpu,int that_cpu) +static inline bool cpus_share_sis (int this_cpu,int that_cpu) {return true; }
kernel/cgroup/cpuset.c +21 -11
@@ -962,10 +962,10 @@ static void rebuild_root_domains(void)static void partition_and_rebuild_sched_domains (int ndoms_new,cpumask_var_t doms_new[], -struct sched_domain_attr *dattr_new) +struct sched_domain_attr *dattr_new,int force_update) {mutex_lock (&sched_domains_mutex); -partition_sched_domains_locked (ndoms_new, doms_new, dattr_new); +partition_sched_domains_locked (ndoms_new, doms_new, dattr_new, force_update);rebuild_root_domains ();mutex_unlock (&sched_domains_mutex); }
@@ -981,7 +981,7 @@ partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],* * Call with cpuset_mutex held. Takes get_online_cpus(). */ -static void rebuild_sched_domains_locked (void ) +static void rebuild_sched_domains_locked (int force_update) {struct sched_domain_attr *attr;cpumask_var_t *doms;
@@ -1007,23 +1007,33 @@ static void rebuild_sched_domains_locked(void) ndoms =generate_sched_domains (&doms, &attr);/* Have scheduler rebuild the domains */ - partition_and_rebuild_sched_domains(ndoms, doms, attr); + partition_and_rebuild_sched_domains(ndoms, doms, attr, force_update); }#else /* !CONFIG_SMP */ -static void rebuild_sched_domains_locked (void ) +static void rebuild_sched_domains_locked (int force_update) { }#endif /* CONFIG_SMP */ -void rebuild_sched_domains (void ) +void __rebuild_sched_domains (int force_update) {get_online_cpus ();percpu_down_write (&cpuset_rwsem); -rebuild_sched_domains_locked (); +rebuild_sched_domains_locked (force_update);percpu_up_write (&cpuset_rwsem);put_online_cpus (); } +void rebuild_sched_domains (void ) +{ +__rebuild_sched_domains (0 ); +} + +void rebuild_sched_domains_force (void ) +{ +__rebuild_sched_domains (1 ); +} +/** * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
@@ -1437,7 +1447,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp)rcu_read_unlock ();if (need_rebuild_sched_domains) -rebuild_sched_domains_locked (); +rebuild_sched_domains_locked (0 ); }/**
@@ -1837,7 +1847,7 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val) cs->relax_domain_level = val;if (!cpumask_empty (cs->cpus_allowed) &&is_sched_load_balance (cs)) -rebuild_sched_domains_locked (); +rebuild_sched_domains_locked (0 ); }return 0 ;
@@ -1903,7 +1913,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,spin_unlock_irq (&callback_lock);if (!cpumask_empty (trialcs->cpus_allowed) && balance_flag_changed) -rebuild_sched_domains_locked (); +rebuild_sched_domains_locked (0 );if (spread_flag_changed)update_tasks_flags (cs);
@@ -1994,7 +2004,7 @@ static int update_prstate(struct cpuset *cs, int val)if (parent->child_ecpus_count)update_sibling_cpumasks (parent, cs, &tmp); -rebuild_sched_domains_locked (); +rebuild_sched_domains_locked (0 ); out:free_cpumasks (NULL , &tmp);return err;
kernel/sched/core.c +5 -5
@@ -2350,9 +2350,9 @@ void wake_up_if_idle(int cpu)rcu_read_unlock (); } -bool cpus_share_cache (int this_cpu,int that_cpu) +bool cpus_share_sis (int this_cpu,int that_cpu) { -return per_cpu (sd_llc_id, this_cpu) ==per_cpu (sd_llc_id, that_cpu); +return per_cpu (sd_sis_id, this_cpu) ==per_cpu (sd_sis_id, that_cpu); }static inline bool ttwu_queue_cond (int cpu,int wake_flags)
@@ -2361,7 +2361,7 @@ static inline bool ttwu_queue_cond(int cpu, int wake_flags)* If the CPU does not share cache, then queue the task on the * remote rqs wakelist to avoid accessing remote data. */ -if (!cpus_share_cache (smp_processor_id (), cpu)) +if (!cpus_share_sis (smp_processor_id (), cpu))return true;/*
@@ -6501,7 +6501,7 @@ static void cpuset_cpu_active(void)* operation in the resume sequence, just build a single sched * domain, ignoring cpusets. */ -partition_sched_domains (1 ,NULL ,NULL ); +partition_sched_domains (1 ,NULL ,NULL ,0 );if (--num_cpus_frozen)return ;/*
@@ -6522,7 +6522,7 @@ static int cpuset_cpu_inactive(unsigned int cpu)cpuset_update_active_cpus (); }else { num_cpus_frozen++; -partition_sched_domains (1 ,NULL ,NULL ); +partition_sched_domains (1 ,NULL ,NULL ,0 ); }return 0 ; }
kernel/sched/fair.c +124 -67
356 lines not on this page · raw message
kernel/sched/sched.h +5 -4
@@ -1415,10 +1415,11 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)return sd; } -DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); -DECLARE_PER_CPU (int, sd_llc_size); -DECLARE_PER_CPU (int, sd_llc_id); -DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_sis); +DECLARE_PER_CPU (int, sd_sis_size); +DECLARE_PER_CPU (int, sd_sis_id); +DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_sis_shared); +DECLARE_PER_CPU(struct sched_domain *, sd_sis_pre); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
kernel/sched/topology.c +60 -27
137 lines not on this page · raw message
kernel/sysctl.c +25 -0
@@ -144,6 +144,10 @@ static const int cap_last_cap = CAP_LAST_CAP;#ifdef CONFIG_DETECT_HUNG_TASK static unsigned long hung_task_timeout_max = (LONG_MAX /HZ );#endif +#ifdef CONFIG_SMP +extern int sysctl_sched_wake_idle_domain; +extern int sysctl_sched_wake_idle_presearch_domain; +#endif #ifdef CONFIG_INOTIFY_USER #include <linux/inotify.h>
@@ -202,6 +206,11 @@ static int max_extfrag_threshold = 1000;#endif /* CONFIG_SYSCTL */ +#ifdef CONFIG_SMP +int proc_sched_wake_idle_domain_handler (struct ctl_table *table, +int write,void __user *buffer,size_t *lenp,loff_t *ppos); +#endif +#if defined(CONFIG_BPF_SYSCALL ) && defined(CONFIG_SYSCTL ) static int bpf_stats_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp,
@@ -1834,6 +1843,22 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_ONE, },#endif +#ifdef CONFIG_SMP + { + .procname = "sched_wake_idle_domain", + .data = &sysctl_sched_wake_idle_domain, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_sched_wake_idle_domain_handler, + }, + { + .procname = "sched_wake_idle_presearch_domain", + .data = &sysctl_sched_wake_idle_presearch_domain, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_sched_wake_idle_domain_handler, + }, +#endif #ifdef CONFIG_PROVE_LOCKING { .procname = "prove_locking",
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
kernel test robot
build botI love your patch! Yet something to improve:
Hi Xi,
I love your patch! Yet something to improve:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/auto-latest linus/master v5.8-rc7] [cannot apply to block/for-next cgroup/for-next next-20200727] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879 config: nios2-defconfig (attached as .config) compiler: nios2-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=nios2
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
In file included from kernel/kthread.c:20:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':
>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
177 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from include/linux/cpuset.h:13,
from kernel/kthread.c:20:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/kthread.c:20:
include/linux/cpuset.h: In function 'rebuild_sched_domains':
include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
263 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from include/linux/cpuset.h:13,
from kernel/kthread.c:20:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/kthread.c:20:
include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
268 | partition_sched_domains(1, NULL, NULL, 1);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from include/linux/cpuset.h:13,
from kernel/kthread.c:20:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~--
In file included from kernel/sched/sched.h:44,
from kernel/sched/core.c:13:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':
>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
177 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/core.c:13:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/core.c:13:
include/linux/cpuset.h: In function 'rebuild_sched_domains':
include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
263 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/core.c:13:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/core.c:13:
include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
268 | partition_sched_domains(1, NULL, NULL, 1);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/core.c:13:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/core.c: In function 'ttwu_stat':
kernel/sched/core.c:2319:13: warning: variable 'rq' set but not used [-Wunused-but-set-variable]
2319 | struct rq *rq;
| ^~--
In file included from kernel/sched/sched.h:44,
from kernel/sched/fair.c:23:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':
>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
177 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/fair.c:23:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/fair.c:23:
include/linux/cpuset.h: In function 'rebuild_sched_domains':
include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
263 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/fair.c:23:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/fair.c:23:
include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
268 | partition_sched_domains(1, NULL, NULL, 1);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/fair.c:23:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c: At top level:
kernel/sched/fair.c:5364:6: warning: no previous prototype for 'init_cfs_bandwidth' [-Wmissing-prototypes]
5364 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
| ^~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:11165:6: warning: no previous prototype for 'free_fair_sched_group' [-Wmissing-prototypes]
11165 | void free_fair_sched_group(struct task_group *tg) { }
| ^~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:11167:5: warning: no previous prototype for 'alloc_fair_sched_group' [-Wmissing-prototypes]
11167 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
| ^~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:11172:6: warning: no previous prototype for 'online_fair_sched_group' [-Wmissing-prototypes]
11172 | void online_fair_sched_group(struct task_group *tg) { }
| ^~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/fair.c:11174:6: warning: no previous prototype for 'unregister_fair_sched_group' [-Wmissing-prototypes]
11174 | void unregister_fair_sched_group(struct task_group *tg) { }
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~--
In file included from kernel/sched/sched.h:44,
from kernel/sched/rt.c:6:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':
>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
177 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/rt.c:6:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/rt.c:6:
include/linux/cpuset.h: In function 'rebuild_sched_domains':
include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
263 | partition_sched_domains(1, NULL, NULL, 0);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/rt.c:6:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:44,
from kernel/sched/rt.c:6:
include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
268 | partition_sched_domains(1, NULL, NULL, 1);
| ^~~~~~~~~~~~~~~~~~~~~~~
In file included from kernel/sched/sched.h:31,
from kernel/sched/rt.c:6:
include/linux/sched/topology.h:208:1: note: declared here
208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
| ^~~~~~~~~~~~~~~~~~~~~~~
kernel/sched/rt.c: At top level:
kernel/sched/rt.c:253:6: warning: no previous prototype for 'free_rt_sched_group' [-Wmissing-prototypes]
253 | void free_rt_sched_group(struct task_group *tg) { }
| ^~~~~~~~~~~~~~~~~~~ kernel/sched/rt.c:255:5: warning: no previous prototype for 'alloc_rt_sched_group' [-Wmissing-prototypes]
255 | int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
| ^~~~~~~~~~~~~~~~~~~~
kernel/sched/rt.c:668:6: warning: no previous prototype for 'sched_rt_bandwidth_account' [-Wmissing-prototypes]
668 | bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
vim +/partition_sched_domains +177 include/linux/cpuset.h
174
175 static inline void cpuset_update_active_cpus(void)
176 {
> 177 partition_sched_domains(1, NULL, NULL, 0);
178 }
179
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kernel test robot
build botI love your patch! Yet something to improve:
Hi Xi,
I love your patch! Yet something to improve:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/auto-latest linus/master v5.8-rc7] [cannot apply to block/for-next cgroup/for-next next-20200727] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879 config: arc-allyesconfig (attached as .config) compiler: arc-elf-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arc
If you fix the issue, kindly add following tag as appropriate
All error/warnings (new ones prefixed by >>):
kernel/sched/fair.c: In function 'select_idle_sibling':
>> kernel/sched/fair.c:6285:27: error: passing argument 2 of 'select_idle_core' from incompatible pointer type [-Werror=incompatible-pointer-types]
6285 | r = select_idle_core(p, cpus, target);
| ^~~~
| |
| struct cpumask *
kernel/sched/fair.c:6096:80: note: expected 'struct sched_domain *' but argument is of type 'struct cpumask *'
6096 | static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target)
| ~~~~~~~~~~~~~~~~~~~~~^~
kernel/sched/fair.c: At top level:
>> kernel/sched/fair.c:6754:5: warning: no previous prototype for 'proc_sched_wake_idle_domain_handler' [-Wmissing-prototypes]
6754 | int proc_sched_wake_idle_domain_handler(struct ctl_table *table,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from include/linux/perf_event.h:25,
from include/linux/trace_events.h:10,
from include/trace/syscall.h:7,
from include/linux/syscalls.h:85,
from kernel/sched/sched.h:65,
from kernel/sched/fair.c:23:
arch/arc/include/asm/perf_event.h:126:23: warning: 'arc_pmu_cache_map' defined but not used [-Wunused-const-variable=]
126 | static const unsigned arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
| ^~~~~~~~~~~~~~~~~
arch/arc/include/asm/perf_event.h:91:27: warning: 'arc_pmu_ev_hw_map' defined but not used [-Wunused-const-variable=]
91 | static const char * const arc_pmu_ev_hw_map[] = {
| ^~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
--
>> kernel/cgroup/cpuset.c:1018:6: warning: no previous prototype for '__rebuild_sched_domains' [-Wmissing-prototypes] 1018 | void __rebuild_sched_domains(int force_update)
| ^~~~~~~~~~~~~~~~~~~~~~~
vim +/select_idle_core +6285 kernel/sched/fair.c
6196
6197 /*
6198 * Try and locate an idle core/thread in the sis domain.
6199 */
6200 static int select_idle_sibling(struct task_struct *p, int prev, int target)
6201 {
6202 struct sched_domain *sd_asym;
6203 struct sched_domain *sd[2];
6204 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6205 int i, r, recent_used_cpu;
6206
6207 /*
6208 * For asymmetric CPU capacity systems, our domain of interest is
6209 * sd_asym_cpucapacity rather than sd_sis.
6210 */
6211 if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6212 sd_asym = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
6213 /*
6214 * On an asymmetric CPU capacity system where an exclusive
6215 * cpuset defines a symmetric island (i.e. one unique
6216 * capacity_orig value through the cpuset), the key will be set
6217 * but the CPUs within that cpuset will not have a domain with
6218 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
6219 * capacity path.
6220 */
6221 if (!sd_asym)
6222 goto symmetric;
6223
6224 i = select_idle_capacity(p, sd_asym, target);
6225 return ((unsigned)i < nr_cpumask_bits) ? i : target;
6226 }
6227
6228 symmetric:
6229 if (available_idle_cpu(target) || sched_idle_cpu(target))
6230 return target;
6231
6232 /*
6233 * If the previous CPU is cache affine and idle, don't be stupid:
6234 */
6235 if (prev != target && cpus_share_sis(prev, target) &&
6236 (available_idle_cpu(prev) || sched_idle_cpu(prev)))
6237 return prev;
6238
6239 /*
6240 * Allow a per-cpu kthread to stack with the wakee if the
6241 * kworker thread and the tasks previous CPUs are the same.
6242 * The assumption is that the wakee queued work for the
6243 * per-cpu kthread that is now complete and the wakeup is
6244 * essentially a sync wakeup. An obvious example of this
6245 * pattern is IO completions.
6246 */
6247 if (is_per_cpu_kthread(current) &&
6248 prev == smp_processor_id() &&
6249 this_rq()->nr_running <= 1) {
6250 return prev;
6251 }
6252
6253 /* Check a recently used CPU as a potential idle candidate: */
6254 recent_used_cpu = p->recent_used_cpu;
6255 if (recent_used_cpu != prev &&
6256 recent_used_cpu != target &&
6257 cpus_share_sis(recent_used_cpu, target) &&
6258 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
6259 cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr)) {
6260 /*
6261 * Replace recent_used_cpu with prev as it is a potential
6262 * candidate for the next wake:
6263 */
6264 p->recent_used_cpu = prev;
6265 return recent_used_cpu;
6266 }
6267
6268 for (i = 0; ; i++) {
6269 if (i == 0) {
6270 sd[0] = rcu_dereference(per_cpu(sd_sis_pre, target));
6271 if (!sd[0])
6272 continue;
6273 cpumask_and(cpus, sched_domain_span(sd[0]), p->cpus_ptr);
6274 } else if (i == 1) {
6275 sd[1] = rcu_dereference(per_cpu(sd_sis, target));
6276 if (!sd[1])
6277 continue;
6278 cpumask_and(cpus, sched_domain_span(sd[1]), p->cpus_ptr);
6279 if (sd[0])
6280 cpumask_andnot(cpus, cpus, sched_domain_span(sd[0]));
6281 } else {
6282 break;
6283 }
6284
> 6285 r = select_idle_core(p, cpus, target); 6286 if ((unsigned)r < nr_cpumask_bits)
6287 return r;
6288
6289 r = select_idle_cpu(p, cpus, (i == 1), sd[i]->span_weight, target);
6290 if ((unsigned)r < nr_cpumask_bits)
6291 return r;
6292
6293 r = select_idle_smt(p, target);
6294 if ((unsigned)r < nr_cpumask_bits)
6295 return r;
6296 }
6297
6298 return target;
6299 }
6300
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
[email protected]
maintainerQuoted 11 lines · Xi Wang
The scope of
select_idle_siblingidle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as thesd_llcis only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:
Quoted 6 lines
sysctl_sched_wake_idle_domain== 2 ("MC" domain)sysctl_sched_wake_idle_presearch_domain== 1 ("DIE" domain)Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.
*groan*, this is horrific :-(
It is also in direct conflict with people wanting to make it smaller.
On top of that, a domain number is a terrible terrible interface. They aren't even available without SCHED_DEBUG on.
What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?
Also, this commit:
051f3ca02e46 ("sched/topology: Introduce NUMA identity node sched domain")
seems to suggest L3 is actually bigger. Suravee, can you please comment?
Quoted 32 lines · On Tue, Jul 28, 2020 at 3:39 AM <[email protected]> wrote:
Quoted 11 lines · Xi Wang
The scope of
select_idle_siblingidle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as thesd_llcis only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:
Quoted 6 lines
sysctl_sched_wake_idle_domain== 2 ("MC" domain)sysctl_sched_wake_idle_presearch_domain== 1 ("DIE" domain)Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.
*groan*, this is horrific :-(
It is also in direct conflict with people wanting to make it smaller.
On top of that, a domain number is a terrible terrible interface. They aren't even available without
SCHED_DEBUGon.What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?
Quoted 5 lines
Also, this commit:
051f3ca02e46("sched/topology: Introduce NUMA identity node sched domain")seems to suggest L3 is actually bigger. Suravee, can you please comment?
I think 051f3ca02e46 was still saying 4 cores sharing an L3 but there is another numa layer which is 8 cores or 2 * 4 core L3 groups. This should be the chiplet layer.
I don't have precise data but some anecdotes are: The latency difference between inter 4 core group access and inter 8 core group access is not huge. Also my experience from Intel machines was that accessing L3 data across numa domains (also across cpu socket) was not too bad until the link bandwidth was saturated. I am hoping the bandwidth situation is better for AMD as L3 groups are smaller. Another factor is sometimes the trade off is spending 10s of us of sched overhead vs time slicing at ~12.5ms latency.
What makes the decision trickly is the configuration can depend on applications and the scale of the system. For a system with 8 cores, running it the old way with 2 * 4 core LLCs might be the best decision. For a system with a lot more cores, the number of threads on the machine would also scale up, which means more potential to create a dynamic imbalance. I have another (even more horrific) patch for auto configuring the sysctls, which has (nnext is the size of the next higher sched domain):
/*
* Widen the range of idle core search if llc domain is too small, both in
* absolute sense and when compared to the next higher level domain. */
if (nllc < min(24, nnext / 4)) {
sysctl_sched_wake_idle_domain = next_level;
sysctl_sched_wake_idle_presearch_domain = llc_level;
/* Also make new idle search domain params more like default llc */
sysctl_sched_wake_idle_domain_tune_flags = 1;
}-Xi
Quoted 66 lines · Xi Wang
Quoted 32 lines · On Tue, Jul 28, 2020 at 3:39 AM <[email protected]> wrote:
Quoted 11 lines · Xi Wang
The scope of
select_idle_siblingidle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as thesd_llcis only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:
Quoted 6 lines
sysctl_sched_wake_idle_domain== 2 ("MC" domain)sysctl_sched_wake_idle_presearch_domain== 1 ("DIE" domain)Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.
*groan*, this is horrific :-(
It is also in direct conflict with people wanting to make it smaller.
On top of that, a domain number is a terrible terrible interface. They aren't even available without
SCHED_DEBUGon.What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?
Quoted 5 lines
Also, this commit:
051f3ca02e46("sched/topology: Introduce NUMA identity node sched domain")seems to suggest L3 is actually bigger. Suravee, can you please comment?
I think
051f3ca02e46was still saying 4 cores sharing an L3 but there is another numa layer which is 8 cores or 2 * 4 core L3 groups. This should be the chiplet layer.I don't have precise data but some anecdotes are: The latency difference between inter 4 core group access and inter 8 core group access is not huge. Also my experience from Intel machines was that accessing L3 data across numa domains (also across cpu socket) was not too bad until the link bandwidth was saturated. I am hoping the bandwidth situation is better for AMD as L3 groups are smaller. Another factor is sometimes the trade off is spending 10s of us of sched overhead vs time slicing at ~12.5ms latency.
What makes the decision trickly is the configuration can depend on applications and the scale of the system. For a system with 8 cores, running it the old way with 2 * 4 core LLCs might be the best decision. For a system with a lot more cores, the number of threads on the machine would also scale up, which means more potential to create a dynamic imbalance. I have another (even more horrific) patch for auto configuring the sysctls, which has (nnext is the size of the next higher sched domain):
/*
* Widen the range of idle core search if llc domain is too small, both in
* absolute sense and when compared to the next higher level domain. */
Quoted 8 lines
if (nllc < min(24, nnext / 4)) { sysctl_sched_wake_idle_domain = next_level; sysctl_sched_wake_idle_presearch_domain = llc_level; /* Also make new idle search domain params more like default llc */ sysctl_sched_wake_idle_domain_tune_flags = 1; }-Xi
Overall I think the current numa defaults should be considered broken for large AMD machines due to latency and core utilization problems. We need to have some solution for it. Simply Moving up the llc domain should work, but we likely also want to avoid some of the obvious overheads. Maybe switching to a CCX local first search order inside select_idle_core would simplify it? It appears the configuration part is still difficult to clean up.
-Xi
kernel test robot
build botI love your patch! Perhaps something to improve:
Hi Xi,
I love your patch! Perhaps something to improve:
[auto build test WARNING on tip/sched/core]
[also build test WARNING on tip/auto-latest v5.8]
[cannot apply to block/for-next cgroup/for-next linus/master next-20200803] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879 config: i386-randconfig-s002-20200803 (attached as .config) compiler: gcc-9 (Debian 9.3.0-14) 9.3.0 reproduce: # apt-get install sparse # sparse version: v0.6.2-117-g8c7aee71-dirty # save the attached .config to linux build tree make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' ARCH=i386
If you fix the issue, kindly add following tag as appropriate
sparse warnings: (new ones prefixed by >>)
>> kernel/cgroup/cpuset.c:1018:6: sparse: sparse: symbol '__rebuild_sched_domains' was not declared. Should it be static?
Please review and possibly fold the followup patch.
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kernel test robot
build bot[RFC PATCH] sched: __rebuild_sched_domains() can be static
In reply to Xi WangSigned-off-by: kernel test robot <[email protected]>
--- cpuset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/cgroup/cpuset.c +1 -1
@@ -1015,7 +1015,7 @@ static void rebuild_sched_domains_locked(int force_update) }#endif /* CONFIG_SMP */ -void __rebuild_sched_domains (int force_update) +static void __rebuild_sched_domains (int force_update) {get_online_cpus ();percpu_down_write (&cpuset_rwsem);