Skip to contentSkip to conversation

sched: Make select_idle_sibling search domain configurable

PATCH In review Xi Wang 3 participants 8 messages Jul 28, 2020 – Aug 4, 2020

Subsystems (6)
VFS Block layer Scheduler cgroup +2 more subsystems
lore
Share & tools
lore

Xi Wang

author
#0 lore raw

PATCH 1/1 sched: Make select_idle_sibling search domain configurable

The scope of select_idle_sibling idle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as the sd_llc is only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.

With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:

sysctl_sched_wake_idle_domain == 2 ("MC" domain) sysctl_sched_wake_idle_presearch_domain == 1 ("DIE" domain)

Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.

Heuristics including sd->avg_scan_cost and sds->have_idle_cores are only active for the main search.

On a 128 core (2 socket * 64 core, 256 hw threads) AMD machine ran hackbench as

hackbench -g 20 -f 20 --loops 10000

A snapshot of run time was

Baseline: 11.8
With the patch: 7.6 (configured as in the example above)

---
 block/blk-mq.c                 |   2 +-
 block/blk-softirq.c            |   2 +-
 include/linux/cpuset.h         |  10 +-
 include/linux/sched/topology.h |  11 +-
 kernel/cgroup/cpuset.c         |  32 ++++--
 kernel/sched/core.c            |  10 +-
 kernel/sched/fair.c            | 191 +++++++++++++++++++++------------
 kernel/sched/sched.h           |   9 +-
 kernel/sched/topology.c        |  87 ++++++++++-----
 kernel/sysctl.c                |  25 +++++
 10 files changed, 256 insertions(+), 123 deletions(-)
block/blk-mq.c +1 -1
index 4e0d173beaa3..20aee9f047e2 100644
@@ -626,7 +626,7 @@ void blk_mq_force_complete_rq(struct request *rq)
 
 	cpu = get_cpu();
 	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
-		shared = cpus_share_cache(cpu, ctx->cpu);
+		shared = cpus_share_sis(cpu, ctx->cpu);
 
 	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 		rq->csd.func = __blk_mq_complete_request_remote;
block/blk-softirq.c +1 -1
index 6e7ec87d49fa..dd38ac0e1f2e 100644
@@ -108,7 +108,7 @@ void __blk_complete_request(struct request *req)
 	 */
 	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) && ccpu != -1) {
 		if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
-			shared = cpus_share_cache(cpu, ccpu);
+			shared = cpus_share_sis(cpu, ccpu);
 	} else
 		ccpu = cpu;
 
include/linux/cpuset.h +8 -2
index 04c20de66afc..8b243aa8462e 100644
@@ -117,6 +117,7 @@ static inline int cpuset_do_slab_mem_spread(void)
 extern bool current_cpuset_is_being_rebound(void);
 
 extern void rebuild_sched_domains(void);
+extern void rebuild_sched_domains_force(void);
 
 extern void cpuset_print_current_mems_allowed(void);
 
@@ -173,7 +174,7 @@ static inline void cpuset_force_rebuild(void) { }
 
 static inline void cpuset_update_active_cpus(void)
 {
-	partition_sched_domains(1, NULL, NULL);
+	partition_sched_domains(1, NULL, NULL, 0);
 }
 
 static inline void cpuset_wait_for_hotplug(void) { }
@@ -259,7 +260,12 @@ static inline bool current_cpuset_is_being_rebound(void)
 
 static inline void rebuild_sched_domains(void)
 {
-	partition_sched_domains(1, NULL, NULL);
+	partition_sched_domains(1, NULL, NULL, 0);
+}
+
+static inline void rebuild_sched_domains_force(void)
+{
+	partition_sched_domains(1, NULL, NULL, 1);
 }
 
 static inline void cpuset_print_current_mems_allowed(void)
include/linux/sched/topology.h +6 -5
index fb11091129b3..aff9739cf516 100644
@@ -151,16 +151,17 @@ static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
 
 extern void partition_sched_domains_locked(int ndoms_new,
 					   cpumask_var_t doms_new[],
-					   struct sched_domain_attr *dattr_new);
+					   struct sched_domain_attr *dattr_new,
+					   int force_update);
 
 extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
-				    struct sched_domain_attr *dattr_new);
+				    struct sched_domain_attr *dattr_new, int force_update);
 
 /* Allocate an array of sched domains, for partition_sched_domains(). */
 cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
 
-bool cpus_share_cache(int this_cpu, int that_cpu);
+bool cpus_share_sis(int this_cpu, int that_cpu);
 
 typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
 typedef int (*sched_domain_flags_f)(void);
@@ -199,7 +200,7 @@ struct sched_domain_attr;
 
 static inline void
 partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
-			       struct sched_domain_attr *dattr_new)
+			       struct sched_domain_attr *dattr_new, int force_update)
 {
 }
 
@@ -209,7 +210,7 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
 {
 }
 
-static inline bool cpus_share_cache(int this_cpu, int that_cpu)
+static inline bool cpus_share_sis(int this_cpu, int that_cpu)
 {
 	return true;
 }
kernel/cgroup/cpuset.c +21 -11
index 642415b8c3c9..5087b90c4c47 100644
@@ -962,10 +962,10 @@ static void rebuild_root_domains(void)
 
 static void
 partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
-				    struct sched_domain_attr *dattr_new)
+				    struct sched_domain_attr *dattr_new, int force_update)
 {
 	mutex_lock(&sched_domains_mutex);
-	partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
+	partition_sched_domains_locked(ndoms_new, doms_new, dattr_new, force_update);
 	rebuild_root_domains();
 	mutex_unlock(&sched_domains_mutex);
 }
@@ -981,7 +981,7 @@ partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
  *
  * Call with cpuset_mutex held.  Takes get_online_cpus().
  */
-static void rebuild_sched_domains_locked(void)
+static void rebuild_sched_domains_locked(int force_update)
 {
 	struct sched_domain_attr *attr;
 	cpumask_var_t *doms;
@@ -1007,23 +1007,33 @@ static void rebuild_sched_domains_locked(void)
 	ndoms = generate_sched_domains(&doms, &attr);
 
 	/* Have scheduler rebuild the domains */
-	partition_and_rebuild_sched_domains(ndoms, doms, attr);
+	partition_and_rebuild_sched_domains(ndoms, doms, attr, force_update);
 }
 #else /* !CONFIG_SMP */
-static void rebuild_sched_domains_locked(void)
+static void rebuild_sched_domains_locked(int force_update)
 {
 }
 #endif /* CONFIG_SMP */
 
-void rebuild_sched_domains(void)
+void __rebuild_sched_domains(int force_update)
 {
 	get_online_cpus();
 	percpu_down_write(&cpuset_rwsem);
-	rebuild_sched_domains_locked();
+	rebuild_sched_domains_locked(force_update);
 	percpu_up_write(&cpuset_rwsem);
 	put_online_cpus();
 }
 
+void rebuild_sched_domains(void)
+{
+	__rebuild_sched_domains(0);
+}
+
+void rebuild_sched_domains_force(void)
+{
+	__rebuild_sched_domains(1);
+}
+
 /**
  * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
  * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
@@ -1437,7 +1447,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp)
 	rcu_read_unlock();
 
 	if (need_rebuild_sched_domains)
-		rebuild_sched_domains_locked();
+		rebuild_sched_domains_locked(0);
 }
 
 /**
@@ -1837,7 +1847,7 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
 		cs->relax_domain_level = val;
 		if (!cpumask_empty(cs->cpus_allowed) &&
 		    is_sched_load_balance(cs))
-			rebuild_sched_domains_locked();
+			rebuild_sched_domains_locked(0);
 	}
 
 	return 0;
@@ -1903,7 +1913,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
 	spin_unlock_irq(&callback_lock);
 
 	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
-		rebuild_sched_domains_locked();
+		rebuild_sched_domains_locked(0);
 
 	if (spread_flag_changed)
 		update_tasks_flags(cs);
@@ -1994,7 +2004,7 @@ static int update_prstate(struct cpuset *cs, int val)
 	if (parent->child_ecpus_count)
 		update_sibling_cpumasks(parent, cs, &tmp);
 
-	rebuild_sched_domains_locked();
+	rebuild_sched_domains_locked(0);
 out:
 	free_cpumasks(NULL, &tmp);
 	return err;
kernel/sched/core.c +5 -5
index e15543cb8481..e28548fc63f0 100644
@@ -2350,9 +2350,9 @@ void wake_up_if_idle(int cpu)
 	rcu_read_unlock();
 }
 
-bool cpus_share_cache(int this_cpu, int that_cpu)
+bool cpus_share_sis(int this_cpu, int that_cpu)
 {
-	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
+	return per_cpu(sd_sis_id, this_cpu) == per_cpu(sd_sis_id, that_cpu);
 }
 
 static inline bool ttwu_queue_cond(int cpu, int wake_flags)
@@ -2361,7 +2361,7 @@ static inline bool ttwu_queue_cond(int cpu, int wake_flags)
 	 * If the CPU does not share cache, then queue the task on the
 	 * remote rqs wakelist to avoid accessing remote data.
 	 */
-	if (!cpus_share_cache(smp_processor_id(), cpu))
+	if (!cpus_share_sis(smp_processor_id(), cpu))
 		return true;
 
 	/*
@@ -6501,7 +6501,7 @@ static void cpuset_cpu_active(void)
 		 * operation in the resume sequence, just build a single sched
 		 * domain, ignoring cpusets.
 		 */
-		partition_sched_domains(1, NULL, NULL);
+		partition_sched_domains(1, NULL, NULL, 0);
 		if (--num_cpus_frozen)
 			return;
 		/*
@@ -6522,7 +6522,7 @@ static int cpuset_cpu_inactive(unsigned int cpu)
 		cpuset_update_active_cpus();
 	} else {
 		num_cpus_frozen++;
-		partition_sched_domains(1, NULL, NULL);
+		partition_sched_domains(1, NULL, NULL, 0);
 	}
 	return 0;
 }
kernel/sched/fair.c +124 -67

356 lines not on this page · raw message

kernel/sched/sched.h +5 -4
index 877fb08eb1b0..641a5bacdf77 100644
@@ -1415,10 +1415,11 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
 	return sd;
 }
 
-DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
-DECLARE_PER_CPU(int, sd_llc_size);
-DECLARE_PER_CPU(int, sd_llc_id);
-DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_sis);
+DECLARE_PER_CPU(int, sd_sis_size);
+DECLARE_PER_CPU(int, sd_sis_id);
+DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_sis_shared);
+DECLARE_PER_CPU(struct sched_domain *, sd_sis_pre);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
kernel/sched/topology.c +60 -27

137 lines not on this page · raw message

kernel/sysctl.c +25 -0
index db1ce7af2563..b474851e1a66 100644
@@ -144,6 +144,10 @@ static const int cap_last_cap = CAP_LAST_CAP;
 #ifdef CONFIG_DETECT_HUNG_TASK
 static unsigned long hung_task_timeout_max = (LONG_MAX/HZ);
 #endif
+#ifdef CONFIG_SMP
+extern int sysctl_sched_wake_idle_domain;
+extern int sysctl_sched_wake_idle_presearch_domain;
+#endif
 
 #ifdef CONFIG_INOTIFY_USER
 #include <linux/inotify.h>
@@ -202,6 +206,11 @@ static int max_extfrag_threshold = 1000;
 
 #endif /* CONFIG_SYSCTL */
 
+#ifdef CONFIG_SMP
+int proc_sched_wake_idle_domain_handler(struct ctl_table *table,
+		int write, void __user *buffer, size_t *lenp, loff_t *ppos);
+#endif
+
 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_SYSCTL)
 static int bpf_stats_handler(struct ctl_table *table, int write,
 			     void __user *buffer, size_t *lenp,
@@ -1834,6 +1843,22 @@ static struct ctl_table kern_table[] = {
 		.extra2		= SYSCTL_ONE,
 	},
 #endif
+#ifdef CONFIG_SMP
+	{
+		.procname	= "sched_wake_idle_domain",
+		.data		= &sysctl_sched_wake_idle_domain,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_sched_wake_idle_domain_handler,
+	},
+	{
+		.procname	= "sched_wake_idle_presearch_domain",
+		.data		= &sysctl_sched_wake_idle_presearch_domain,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_sched_wake_idle_domain_handler,
+	},
+#endif
 #ifdef CONFIG_PROVE_LOCKING
 	{
 		.procname	= "prove_locking",
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
#1 lore raw
I love your patch! Yet something to improve:

Hi Xi,

I love your patch! Yet something to improve:

[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/auto-latest linus/master v5.8-rc7] [cannot apply to block/for-next cgroup/for-next next-20200727] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879
config: nios2-defconfig (attached as .config)
compiler: nios2-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=nios2

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from kernel/kthread.c:20:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':

>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
     177 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from include/linux/cpuset.h:13,
                    from kernel/kthread.c:20:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/kthread.c:20:
   include/linux/cpuset.h: In function 'rebuild_sched_domains':
   include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
     263 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from include/linux/cpuset.h:13,
                    from kernel/kthread.c:20:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/kthread.c:20:
   include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
   include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
     268 |  partition_sched_domains(1, NULL, NULL, 1);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from include/linux/cpuset.h:13,
                    from kernel/kthread.c:20:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~

--
In file included from kernel/sched/sched.h:44,
from kernel/sched/core.c:13:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':

>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
     177 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/core.c:13:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/core.c:13:
   include/linux/cpuset.h: In function 'rebuild_sched_domains':
   include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
     263 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/core.c:13:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/core.c:13:
   include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
   include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
     268 |  partition_sched_domains(1, NULL, NULL, 1);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/core.c:13:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   kernel/sched/core.c: In function 'ttwu_stat':
   kernel/sched/core.c:2319:13: warning: variable 'rq' set but not used [-Wunused-but-set-variable]
    2319 |  struct rq *rq;
         |             ^~

--
In file included from kernel/sched/sched.h:44,
from kernel/sched/fair.c:23:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':

>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
     177 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/fair.c:23:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/fair.c:23:
   include/linux/cpuset.h: In function 'rebuild_sched_domains':
   include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
     263 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/fair.c:23:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/fair.c:23:
   include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
   include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
     268 |  partition_sched_domains(1, NULL, NULL, 1);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/fair.c:23:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   kernel/sched/fair.c: At top level:
   kernel/sched/fair.c:5364:6: warning: no previous prototype for 'init_cfs_bandwidth' [-Wmissing-prototypes]
    5364 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
         |      ^~~~~~~~~~~~~~~~~~
   kernel/sched/fair.c:11165:6: warning: no previous prototype for 'free_fair_sched_group' [-Wmissing-prototypes]
   11165 | void free_fair_sched_group(struct task_group *tg) { }
         |      ^~~~~~~~~~~~~~~~~~~~~
   kernel/sched/fair.c:11167:5: warning: no previous prototype for 'alloc_fair_sched_group' [-Wmissing-prototypes]
   11167 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
         |     ^~~~~~~~~~~~~~~~~~~~~~
   kernel/sched/fair.c:11172:6: warning: no previous prototype for 'online_fair_sched_group' [-Wmissing-prototypes]
   11172 | void online_fair_sched_group(struct task_group *tg) { }
         |      ^~~~~~~~~~~~~~~~~~~~~~~
   kernel/sched/fair.c:11174:6: warning: no previous prototype for 'unregister_fair_sched_group' [-Wmissing-prototypes]
   11174 | void unregister_fair_sched_group(struct task_group *tg) { }
         |      ^~~~~~~~~~~~~~~~~~~~~~~~~~~

--
In file included from kernel/sched/sched.h:44,
from kernel/sched/rt.c:6:
include/linux/cpuset.h: In function 'cpuset_update_active_cpus':

>> include/linux/cpuset.h:177:2: error: too many arguments to function 'partition_sched_domains'
     177 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/rt.c:6:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/rt.c:6:
   include/linux/cpuset.h: In function 'rebuild_sched_domains':
   include/linux/cpuset.h:263:2: error: too many arguments to function 'partition_sched_domains'
     263 |  partition_sched_domains(1, NULL, NULL, 0);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/rt.c:6:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:44,
                    from kernel/sched/rt.c:6:
   include/linux/cpuset.h: In function 'rebuild_sched_domains_force':
   include/linux/cpuset.h:268:2: error: too many arguments to function 'partition_sched_domains'
     268 |  partition_sched_domains(1, NULL, NULL, 1);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   In file included from kernel/sched/sched.h:31,
                    from kernel/sched/rt.c:6:
   include/linux/sched/topology.h:208:1: note: declared here
     208 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
         | ^~~~~~~~~~~~~~~~~~~~~~~
   kernel/sched/rt.c: At top level:
   kernel/sched/rt.c:253:6: warning: no previous prototype for 'free_rt_sched_group' [-Wmissing-prototypes]
     253 | void free_rt_sched_group(struct task_group *tg) { }
         |      ^~~~~~~~~~~~~~~~~~~

kernel/sched/rt.c:255:5: warning: no previous prototype for 'alloc_rt_sched_group' [-Wmissing-prototypes]
255 | int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
| ^~~~~~~~~~~~~~~~~~~~
kernel/sched/rt.c:668:6: warning: no previous prototype for 'sched_rt_bandwidth_account' [-Wmissing-prototypes]
668 | bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~

vim +/partition_sched_domains +177 include/linux/cpuset.h

174
175 static inline void cpuset_update_active_cpus(void)
176 {

 > 177		partition_sched_domains(1, NULL, NULL, 0);

178 }
179

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#2 lore raw
In reply to Xi Wang
I love your patch! Yet something to improve:

Hi Xi,

I love your patch! Yet something to improve:

[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/auto-latest linus/master v5.8-rc7] [cannot apply to block/for-next cgroup/for-next next-20200727] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879
config: arc-allyesconfig (attached as .config)
compiler: arc-elf-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arc

If you fix the issue, kindly add following tag as appropriate

All error/warnings (new ones prefixed by >>):

kernel/sched/fair.c: In function 'select_idle_sibling':

>> kernel/sched/fair.c:6285:27: error: passing argument 2 of 'select_idle_core' from incompatible pointer type [-Werror=incompatible-pointer-types]
    6285 |   r = select_idle_core(p, cpus, target);
         |                           ^~~~
         |                           |
         |                           struct cpumask *
   kernel/sched/fair.c:6096:80: note: expected 'struct sched_domain *' but argument is of type 'struct cpumask *'
    6096 | static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target)
         |                                                           ~~~~~~~~~~~~~~~~~~~~~^~
   kernel/sched/fair.c: At top level:
>> kernel/sched/fair.c:6754:5: warning: no previous prototype for 'proc_sched_wake_idle_domain_handler' [-Wmissing-prototypes]
    6754 | int proc_sched_wake_idle_domain_handler(struct ctl_table *table,
         |     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   In file included from include/linux/perf_event.h:25,
                    from include/linux/trace_events.h:10,
                    from include/trace/syscall.h:7,
                    from include/linux/syscalls.h:85,
                    from kernel/sched/sched.h:65,
                    from kernel/sched/fair.c:23:
   arch/arc/include/asm/perf_event.h:126:23: warning: 'arc_pmu_cache_map' defined but not used [-Wunused-const-variable=]
     126 | static const unsigned arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
         |                       ^~~~~~~~~~~~~~~~~
   arch/arc/include/asm/perf_event.h:91:27: warning: 'arc_pmu_ev_hw_map' defined but not used [-Wunused-const-variable=]
      91 | static const char * const arc_pmu_ev_hw_map[] = {
         |                           ^~~~~~~~~~~~~~~~~
   cc1: some warnings being treated as errors
--
>> kernel/cgroup/cpuset.c:1018:6: warning: no previous prototype for '__rebuild_sched_domains' [-Wmissing-prototypes]

1018 | void __rebuild_sched_domains(int force_update)
| ^~~~~~~~~~~~~~~~~~~~~~~

vim +/select_idle_core +6285 kernel/sched/fair.c

6196
6197 /*
6198 * Try and locate an idle core/thread in the sis domain.
6199 */
6200 static int select_idle_sibling(struct task_struct *p, int prev, int target)
6201 {
6202 struct sched_domain *sd_asym;
6203 struct sched_domain *sd[2];
6204 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6205 int i, r, recent_used_cpu;
6206
6207 /*
6208 * For asymmetric CPU capacity systems, our domain of interest is
6209 * sd_asym_cpucapacity rather than sd_sis.
6210 */
6211 if (static_branch_unlikely(&sched_asym_cpucapacity)) {
6212 sd_asym = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
6213 /*
6214 * On an asymmetric CPU capacity system where an exclusive
6215 * cpuset defines a symmetric island (i.e. one unique
6216 * capacity_orig value through the cpuset), the key will be set
6217 * but the CPUs within that cpuset will not have a domain with
6218 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
6219 * capacity path.
6220 */
6221 if (!sd_asym)
6222 goto symmetric;
6223
6224 i = select_idle_capacity(p, sd_asym, target);
6225 return ((unsigned)i < nr_cpumask_bits) ? i : target;
6226 }
6227
6228 symmetric:
6229 if (available_idle_cpu(target) || sched_idle_cpu(target))
6230 return target;
6231
6232 /*
6233 * If the previous CPU is cache affine and idle, don't be stupid:
6234 */
6235 if (prev != target && cpus_share_sis(prev, target) &&
6236 (available_idle_cpu(prev) || sched_idle_cpu(prev)))
6237 return prev;
6238
6239 /*
6240 * Allow a per-cpu kthread to stack with the wakee if the
6241 * kworker thread and the tasks previous CPUs are the same.
6242 * The assumption is that the wakee queued work for the
6243 * per-cpu kthread that is now complete and the wakeup is
6244 * essentially a sync wakeup. An obvious example of this
6245 * pattern is IO completions.
6246 */
6247 if (is_per_cpu_kthread(current) &&
6248 prev == smp_processor_id() &&
6249 this_rq()->nr_running <= 1) {
6250 return prev;
6251 }
6252
6253 /* Check a recently used CPU as a potential idle candidate: */
6254 recent_used_cpu = p->recent_used_cpu;
6255 if (recent_used_cpu != prev &&

  6256		    recent_used_cpu != target &&
  6257		    cpus_share_sis(recent_used_cpu, target) &&
  6258		    (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
  6259		    cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr)) {
  6260			/*
  6261			 * Replace recent_used_cpu with prev as it is a potential
  6262			 * candidate for the next wake:
  6263			 */
  6264			p->recent_used_cpu = prev;
  6265			return recent_used_cpu;
  6266		}
  6267	
  6268		for (i = 0; ; i++) {
  6269			if (i == 0) {
  6270				sd[0] = rcu_dereference(per_cpu(sd_sis_pre, target));
  6271				if (!sd[0])
  6272					continue;
  6273				cpumask_and(cpus, sched_domain_span(sd[0]), p->cpus_ptr);
  6274			} else if (i == 1) {
  6275				sd[1] = rcu_dereference(per_cpu(sd_sis, target));
  6276				if (!sd[1])
  6277					continue;
  6278				cpumask_and(cpus, sched_domain_span(sd[1]), p->cpus_ptr);
  6279				if (sd[0])
  6280					cpumask_andnot(cpus, cpus, sched_domain_span(sd[0]));
  6281			} else {
  6282				break;
  6283			}
  6284	
> 6285			r = select_idle_core(p, cpus, target);

6286 if ((unsigned)r < nr_cpumask_bits)
6287 return r;
6288
6289 r = select_idle_cpu(p, cpus, (i == 1), sd[i]->span_weight, target);
6290 if ((unsigned)r < nr_cpumask_bits)
6291 return r;
6292
6293 r = select_idle_smt(p, target);
6294 if ((unsigned)r < nr_cpumask_bits)
6295 return r;
6296 }
6297
6298 return target;
6299 }
6300

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]

#3 lore raw
In reply to Xi Wang
Quoted 11 lines · Xi Wang

The scope of select_idle_sibling idle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as the sd_llc is only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.

With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:

Quoted 6 lines

sysctl_sched_wake_idle_domain == 2 ("MC" domain) sysctl_sched_wake_idle_presearch_domain == 1 ("DIE" domain)

Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.

*groan*, this is horrific :-(

It is also in direct conflict with people wanting to make it smaller.

On top of that, a domain number is a terrible terrible interface. They aren't even available without SCHED_DEBUG on.

What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?

Also, this commit:

051f3ca02e46 ("sched/topology: Introduce NUMA identity node sched domain")

seems to suggest L3 is actually bigger. Suravee, can you please comment?

author
#4 lore raw
Quoted 32 lines · On Tue, Jul 28, 2020 at 3:39 AM <[email protected]> wrote:
Quoted 11 lines · Xi Wang

The scope of select_idle_sibling idle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as the sd_llc is only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.

With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:

Quoted 6 lines

sysctl_sched_wake_idle_domain == 2 ("MC" domain) sysctl_sched_wake_idle_presearch_domain == 1 ("DIE" domain)

Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.

*groan*, this is horrific :-(

It is also in direct conflict with people wanting to make it smaller.

On top of that, a domain number is a terrible terrible interface. They aren't even available without SCHED_DEBUG on.

What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?

Quoted 5 lines

Also, this commit:

051f3ca02e46 ("sched/topology: Introduce NUMA identity node sched domain")

seems to suggest L3 is actually bigger. Suravee, can you please comment?

I think 051f3ca02e46 was still saying 4 cores sharing an L3 but there is another numa layer which is 8 cores or 2 * 4 core L3 groups. This should be the chiplet layer.

I don't have precise data but some anecdotes are: The latency difference between inter 4 core group access and inter 8 core group access is not huge. Also my experience from Intel machines was that accessing L3 data across numa domains (also across cpu socket) was not too bad until the link bandwidth was saturated. I am hoping the bandwidth situation is better for AMD as L3 groups are smaller. Another factor is sometimes the trade off is spending 10s of us of sched overhead vs time slicing at ~12.5ms latency.

What makes the decision trickly is the configuration can depend on applications and the scale of the system. For a system with 8 cores, running it the old way with 2 * 4 core LLCs might be the best decision. For a system with a lot more cores, the number of threads on the machine would also scale up, which means more potential to create a dynamic imbalance. I have another (even more horrific) patch for auto configuring the sysctls, which has (nnext is the size of the next higher sched domain):

/*
* Widen the range of idle core search if llc domain is too small, both in
* absolute sense and when compared to the next higher level domain. */

if (nllc < min(24, nnext / 4)) {
        sysctl_sched_wake_idle_domain = next_level;
        sysctl_sched_wake_idle_presearch_domain = llc_level;
        /* Also make new idle search domain params more like default llc */
        sysctl_sched_wake_idle_domain_tune_flags = 1;
}

-Xi

author
#5 lore raw
Quoted 66 lines · Xi Wang
Quoted 32 lines · On Tue, Jul 28, 2020 at 3:39 AM <[email protected]> wrote:
Quoted 11 lines · Xi Wang

The scope of select_idle_sibling idle cpu search is LLC. This becomes a problem for the AMD CCX architecture, as the sd_llc is only 4 cores. On a many core machine, the range of search is too small to reach a satisfactory level of statistical multiplexing / efficient utilization of short idle time slices.

With this patch idle sibling search is detached from LLC and it becomes run time configurable. To reduce search and migration overheads, a presearch domain is added. The presearch domain will be searched first before the "main search" domain, e.g.:

Quoted 6 lines

sysctl_sched_wake_idle_domain == 2 ("MC" domain) sysctl_sched_wake_idle_presearch_domain == 1 ("DIE" domain)

Presearch will go through 4 cores of a CCX. If no idle cpu is found during presearch, full search will go through the remaining cores of a cpu socket.

*groan*, this is horrific :-(

It is also in direct conflict with people wanting to make it smaller.

On top of that, a domain number is a terrible terrible interface. They aren't even available without SCHED_DEBUG on.

What is the inter-L3 latency? Going by this that had better be awesome. And if this Infinity Fabric stuff if highly effective in effectively merging L3s -- analogous to what Intel does with it's cache slices, then should we not change the AMD topology setup instead of this 'thing'?

Quoted 5 lines

Also, this commit:

051f3ca02e46 ("sched/topology: Introduce NUMA identity node sched domain")

seems to suggest L3 is actually bigger. Suravee, can you please comment?

I think 051f3ca02e46 was still saying 4 cores sharing an L3 but there is another numa layer which is 8 cores or 2 * 4 core L3 groups. This should be the chiplet layer.

I don't have precise data but some anecdotes are: The latency difference between inter 4 core group access and inter 8 core group access is not huge. Also my experience from Intel machines was that accessing L3 data across numa domains (also across cpu socket) was not too bad until the link bandwidth was saturated. I am hoping the bandwidth situation is better for AMD as L3 groups are smaller. Another factor is sometimes the trade off is spending 10s of us of sched overhead vs time slicing at ~12.5ms latency.

What makes the decision trickly is the configuration can depend on applications and the scale of the system. For a system with 8 cores, running it the old way with 2 * 4 core LLCs might be the best decision. For a system with a lot more cores, the number of threads on the machine would also scale up, which means more potential to create a dynamic imbalance. I have another (even more horrific) patch for auto configuring the sysctls, which has (nnext is the size of the next higher sched domain):

/*
* Widen the range of idle core search if llc domain is too small, both in
* absolute sense and when compared to the next higher level domain. */

Quoted 8 lines
if (nllc < min(24, nnext / 4)) {
        sysctl_sched_wake_idle_domain = next_level;
        sysctl_sched_wake_idle_presearch_domain = llc_level;
        /* Also make new idle search domain params more like default llc */
        sysctl_sched_wake_idle_domain_tune_flags = 1;
}

-Xi

Overall I think the current numa defaults should be considered broken for large AMD machines due to latency and core utilization problems. We need to have some solution for it. Simply Moving up the llc domain should work, but we likely also want to avoid some of the obvious overheads. Maybe switching to a CCX local first search order inside select_idle_core would simplify it? It appears the configuration part is still difficult to clean up.

-Xi

#6 lore raw
In reply to Xi Wang
I love your patch! Perhaps something to improve:

Hi Xi,

I love your patch! Perhaps something to improve:

[auto build test WARNING on tip/sched/core]
[also build test WARNING on tip/auto-latest v5.8]
[cannot apply to block/for-next cgroup/for-next linus/master next-20200803] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Xi-Wang/sched-Make-select_idle_sibling-search-domain-configurable/20200728-150328
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 13efa616124f7eec7d6a58adeeef31864aa03879
config: i386-randconfig-s002-20200803 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-14) 9.3.0
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.2-117-g8c7aee71-dirty
        # save the attached .config to linux build tree
        make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' ARCH=i386 

If you fix the issue, kindly add following tag as appropriate

sparse warnings: (new ones prefixed by >>)

>> kernel/cgroup/cpuset.c:1018:6: sparse: sparse: symbol '__rebuild_sched_domains' was not declared. Should it be static?

Please review and possibly fold the followup patch.

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#7 lore raw

[RFC PATCH] sched: __rebuild_sched_domains() can be static

In reply to Xi Wang
Signed-off-by: kernel test robot <[email protected]>
---
 cpuset.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/cgroup/cpuset.c +1 -1
index 5087b90c4c47b..c871b4f4a7a3e 100644
@@ -1015,7 +1015,7 @@ static void rebuild_sched_domains_locked(int force_update)
 }
 #endif /* CONFIG_SMP */
 
-void __rebuild_sched_domains(int force_update)
+static void __rebuild_sched_domains(int force_update)
 {
 	get_online_cpus();
 	percpu_down_write(&cpuset_rwsem);