Customize sched domain via cpuset

The implementation is here.

- Add 2 new cpuset files:
sched_wake_idle_far
sched_balance_newidle_far

- Modify partition_sched_domains() and build_sched_domains() to take flags parameter passed from cpuset.

- Fill newidle_idx for node domains which currently unused but might be required for sched_balance_newidle_far.

Signed-off-by Hidetoshi Seto <[email protected]>

---
 include/asm-ia64/topology.h |    2
 include/asm-sh/topology.h   |    2
 include/asm-x86/topology.h  |    2
 include/linux/sched.h       |    4 +
 kernel/cpuset.c             |   89 ++++++++++++++++++++++++++++++++++++++++++--
 kernel/sched.c              |   38 ++++++++++++++++--
 kernel/sched_fair.c         |    4 +
 7 files changed, 128 insertions(+), 13 deletions(-)

Index: GIT-torvalds/kernel/sched_fair.c
===================================================================

kernel/sched_fair.c +3 -1

@@ -957,7 +957,9 @@ static int wake_idle(int cpu, struct tas
 		return cpu;
 
 	for_each_domain(cpu, sd) {
-		if (sd->flags & SD_WAKE_IDLE) {
+		if ((sd->flags & SD_WAKE_IDLE)
+		    || ((sd->flags & SD_WAKE_IDLE_FAR)
+			&& !task_hot(p, task_rq(p)->clock, sd))) {
 			cpus_and(tmp, sd->span, p->cpus_allowed);
 			for_each_cpu_mask(i, tmp) {
 				if (idle_cpu(i)) {

Index: GIT-torvalds/kernel/cpuset.c
===================================================================

kernel/cpuset.c +86 -3

@@ -126,6 +126,8 @@ typedef enum {
 	CS_MEM_EXCLUSIVE,
 	CS_MEMORY_MIGRATE,
 	CS_SCHED_LOAD_BALANCE,
+	CS_SCHED_BALANCE_NEWIDLE_FAR,
+	CS_SCHED_WAKE_IDLE_FAR,
 	CS_SPREAD_PAGE,
 	CS_SPREAD_SLAB,
 } cpuset_flagbits_t;

@@ -146,6 +148,16 @@ static inline int is_sched_load_balance(
 	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
 }
 
+static inline int is_sched_balance_newidle_far(const struct cpuset *cs)
+{
+	return test_bit(CS_SCHED_BALANCE_NEWIDLE_FAR, &cs->flags);
+}
+
+static inline int is_sched_wake_idle_far(const struct cpuset *cs)
+{
+	return test_bit(CS_SCHED_WAKE_IDLE_FAR, &cs->flags);
+}
+
 static inline int is_memory_migrate(const struct cpuset *cs)
 {
 	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);

@@ -161,6 +173,11 @@ static inline int is_spread_slab(const s
 	return test_bit(CS_SPREAD_SLAB, &cs->flags);
 }
 
+static inline int is_sched_custom_domain(const struct cpuset *cs)
+{
+	return is_sched_balance_newidle_far(cs) || is_sched_wake_idle_far(cs);
+}
+
 /*
  * Increment this integer everytime any cpuset changes its
  * mems_allowed value.  Users of cpusets can track this generation

@@ -553,12 +570,14 @@ static void rebuild_sched_domains(void)
 	int csn;		/* how many cpuset ptrs in csa so far */
 	int i, j, k;		/* indices for partition finding loops */
 	cpumask_t *doms;	/* resulting partition; i.e. sched domains */
+	int *flags;		/* flags for custom sched domains */
 	int ndoms;		/* number of sched domains in result */
 	int nslot;		/* next empty doms[] cpumask_t slot */
 
 	q = NULL;
 	csa = NULL;
 	doms = NULL;
+	flags = NULL;
 
 	/* Special case for the 99% of systems with one, full, sched domain */
 	if (is_sched_load_balance(&top_cpuset)) {

@@ -566,6 +585,13 @@ static void rebuild_sched_domains(void)
 		doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
 		if (!doms)
 			goto rebuild;
+		if (is_sched_custom_domain(&top_cpuset)) {
+			flags = kzalloc(sizeof(int), GFP_KERNEL);
+			if (flags && is_sched_balance_newidle_far(&top_cpuset))
+				*flags |= SD_BALANCE_NEWIDLE;
+			if (flags && is_sched_wake_idle_far(&top_cpuset))
+				*flags |= SD_WAKE_IDLE_FAR;
+		}
 		*doms = top_cpuset.cpus_allowed;
 		goto rebuild;
 	}

@@ -622,6 +648,7 @@ restart:
 	doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL);
 	if (!doms)
 		goto rebuild;
+	flags = kzalloc(ndoms * sizeof(int), GFP_KERNEL);
 
 	for (nslot = 0, i = 0; i < csn; i++) {
 		struct cpuset *a = csa[i];

@@ -650,6 +677,13 @@ restart:
 				if (apn == b->pn) {
 					cpus_or(*dp, *dp, b->cpus_allowed);
 					b->pn = -1;
+					if (flags
+					    && is_sched_balance_newidle_far(b))
+						*(flags + nslot) |=
+							SD_BALANCE_NEWIDLE;
+					if (flags && is_sched_wake_idle_far(b))
+						*(flags + nslot) |=
+							SD_WAKE_IDLE_FAR;
 				}
 			}
 			nslot++;

@@ -660,7 +694,7 @@ restart:
 rebuild:
 	/* Have scheduler rebuild sched domains */
 	get_online_cpus();
-	partition_sched_domains(ndoms, doms);
+	partition_sched_domains(ndoms, doms, flags);
 	put_online_cpus();
 
 done:

@@ -668,6 +702,7 @@ done:
 		kfifo_free(q);
 	kfree(csa);
 	/* Don't kfree(doms) -- partition_sched_domains() does that. */
+	/* Don't kfree(flags) -- partition_sched_domains() does that. */
 }
 
 static inline int started_after_time(struct task_struct *t1,

@@ -1011,10 +1046,26 @@ static int update_memory_pressure_enable
 	return 0;
 }
 
+static int need_rebuild_domains(struct cpuset *cs, struct cpuset *tcs)
+{
+	if (is_sched_load_balance(cs) != is_sched_load_balance(tcs))
+		return 1;
+	if (!is_sched_load_balance(tcs))
+		return 0;
+	if (is_sched_balance_newidle_far(cs) !=
+					is_sched_balance_newidle_far(tcs))
+		return 1;
+	if (is_sched_wake_idle_far(cs) != is_sched_wake_idle_far(tcs))
+		return 1;
+	return 0;
+}
+
 /*
  * update_flag - read a 0 or a 1 in a file and update associated flag
  * bit:	the bit to update (CS_CPU_EXCLUSIVE, CS_MEM_EXCLUSIVE,
  *				CS_SCHED_LOAD_BALANCE,
+ *				CS_SCHED_BALANCE_NEW_IDLE_FAR,
+ *				CS_SCHED_WAKE_IDLE_FAR,
  *				CS_NOTIFY_ON_RELEASE, CS_MEMORY_MIGRATE,
  *				CS_SPREAD_PAGE, CS_SPREAD_SLAB)
  * cs:	the cpuset to update

@@ -1043,8 +1094,7 @@ static int update_flag(cpuset_flagbits_t
 		return err;
 
 	cpus_nonempty = !cpus_empty(trialcs.cpus_allowed);
-	balance_flag_changed = (is_sched_load_balance(cs) !=
-		 			is_sched_load_balance(&trialcs));
+	balance_flag_changed = need_rebuild_domains(cs, &trialcs);
 
 	mutex_lock(&callback_mutex);
 	cs->flags = trialcs.flags;

@@ -1202,6 +1252,8 @@ typedef enum {
 	FILE_CPU_EXCLUSIVE,
 	FILE_MEM_EXCLUSIVE,
 	FILE_SCHED_LOAD_BALANCE,
+	FILE_SCHED_BALANCE_NEWIDLE_FAR,
+	FILE_SCHED_WAKE_IDLE_FAR,
 	FILE_MEMORY_PRESSURE_ENABLED,
 	FILE_MEMORY_PRESSURE,
 	FILE_SPREAD_PAGE,

@@ -1256,6 +1308,12 @@ static ssize_t cpuset_common_file_write(
 	case FILE_SCHED_LOAD_BALANCE:
 		retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, buffer);
 		break;
+	case FILE_SCHED_BALANCE_NEWIDLE_FAR:
+		retval = update_flag(CS_SCHED_BALANCE_NEWIDLE_FAR, cs, buffer);
+		break;
+	case FILE_SCHED_WAKE_IDLE_FAR:
+		retval = update_flag(CS_SCHED_WAKE_IDLE_FAR, cs, buffer);
+		break;
 	case FILE_MEMORY_MIGRATE:
 		retval = update_flag(CS_MEMORY_MIGRATE, cs, buffer);
 		break;

@@ -1354,6 +1412,12 @@ static ssize_t cpuset_common_file_read(s
 	case FILE_SCHED_LOAD_BALANCE:
 		*s++ = is_sched_load_balance(cs) ? '1' : '0';
 		break;
+	case FILE_SCHED_BALANCE_NEWIDLE_FAR:
+		*s++ = is_sched_balance_newidle_far(cs) ? '1' : '0';
+		break;
+	case FILE_SCHED_WAKE_IDLE_FAR:
+		*s++ = is_sched_wake_idle_far(cs) ? '1' : '0';
+		break;
 	case FILE_MEMORY_MIGRATE:
 		*s++ = is_memory_migrate(cs) ? '1' : '0';
 		break;

@@ -1424,6 +1488,20 @@ static struct cftype cft_sched_load_bala
 	.private = FILE_SCHED_LOAD_BALANCE,
 };
 
+static struct cftype cft_sched_balance_newidle_far = {
+	.name = "sched_balance_newidle_far",
+	.read = cpuset_common_file_read,
+	.write = cpuset_common_file_write,
+	.private = FILE_SCHED_BALANCE_NEWIDLE_FAR,
+};
+
+static struct cftype cft_sched_wake_idle_far = {
+	.name = "sched_wake_idle_far",
+	.read = cpuset_common_file_read,
+	.write = cpuset_common_file_write,
+	.private = FILE_SCHED_WAKE_IDLE_FAR,
+};
+
 static struct cftype cft_memory_migrate = {
 	.name = "memory_migrate",
 	.read = cpuset_common_file_read,

@@ -1475,6 +1553,11 @@ static int cpuset_populate(struct cgroup
 		return err;
 	if ((err = cgroup_add_file(cont, ss, &cft_sched_load_balance)) < 0)
 		return err;
+	if ((err = cgroup_add_file(cont, ss,
+					&cft_sched_balance_newidle_far)) < 0)
+		return err;
+	if ((err = cgroup_add_file(cont, ss, &cft_sched_wake_idle_far)) < 0)
+		return err;
 	if ((err = cgroup_add_file(cont, ss, &cft_memory_pressure)) < 0)
 		return err;
 	if ((err = cgroup_add_file(cont, ss, &cft_spread_page)) < 0)

Index: GIT-torvalds/include/linux/sched.h
===================================================================

include/linux/sched.h +3 -1

@@ -704,6 +704,7 @@ enum cpu_idle_type {
 #define SD_POWERSAVINGS_BALANCE	256	/* Balance for power savings */
 #define SD_SHARE_PKG_RESOURCES	512	/* Domain members share cpu pkg resources */
 #define SD_SERIALIZE		1024	/* Only a single load balancing instance */
+#define SD_WAKE_IDLE_FAR	2048	/* Gain latency sacrificing cache hit */
 
 #define BALANCE_FOR_MC_POWER	\
 	(sched_smt_power_savings ? SD_POWERSAVINGS_BALANCE : 0)

@@ -789,7 +790,8 @@ struct sched_domain {
 #endif
 };
 
-extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new);
+extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
+				    int *flags_new);
 extern int arch_reinit_sched_domains(void);
 
 #endif	/* CONFIG_SMP */

Index: GIT-torvalds/kernel/sched.c
===================================================================

kernel/sched.c +33 -5

@@ -6586,7 +6586,7 @@ static void init_sched_groups_power(int
  * Build sched domains for a given set of cpus and attach the sched domains
  * to the individual cpus
  */
-static int build_sched_domains(const cpumask_t *cpu_map)
+static int __build_sched_domains(const cpumask_t *cpu_map, int flags)
 {
 	int i;
 	struct root_domain *rd;

@@ -6627,6 +6627,7 @@ static int build_sched_domains(const cpu
 			sd = &per_cpu(allnodes_domains, i);
 			*sd = SD_ALLNODES_INIT;
 			sd->span = *cpu_map;
+			/* prohibit "sd->flags |= flags" for allnodes_domain */
 			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
 			p = sd;
 			sd_allnodes = 1;

@@ -6636,6 +6637,7 @@ static int build_sched_domains(const cpu
 		sd = &per_cpu(node_domains, i);
 		*sd = SD_NODE_INIT;
 		sd->span = sched_domain_node_span(cpu_to_node(i));
+		sd->flags |= flags;
 		sd->parent = p;
 		if (p)
 			p->child = sd;

@@ -6646,6 +6648,7 @@ static int build_sched_domains(const cpu
 		sd = &per_cpu(phys_domains, i);
 		*sd = SD_CPU_INIT;
 		sd->span = nodemask;
+		sd->flags |= flags;
 		sd->parent = p;
 		if (p)
 			p->child = sd;

@@ -6657,6 +6660,7 @@ static int build_sched_domains(const cpu
 		*sd = SD_MC_INIT;
 		sd->span = cpu_coregroup_map(i);
 		cpus_and(sd->span, sd->span, *cpu_map);
+		sd->flags |= flags;
 		sd->parent = p;
 		p->child = sd;
 		cpu_to_core_group(i, cpu_map, &sd->groups);

@@ -6668,6 +6672,7 @@ static int build_sched_domains(const cpu
 		*sd = SD_SIBLING_INIT;
 		sd->span = per_cpu(cpu_sibling_map, i);
 		cpus_and(sd->span, sd->span, *cpu_map);
+		sd->flags |= flags;
 		sd->parent = p;
 		p->child = sd;
 		cpu_to_cpu_group(i, cpu_map, &sd->groups);

@@ -6840,8 +6845,14 @@ error:
 #endif
 }
 
+static int build_sched_domains(const cpumask_t *cpu_map)
+{
+	return __build_sched_domains(cpu_map, 0);
+}
+
 static cpumask_t *doms_cur;	/* current sched domains */
 static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
+static int *flags_cur;		/* custom flags of domains in 'doms_cur' */
 
 /*
  * Special case: If a kmalloc of a doms_cur partition (array of

@@ -6868,6 +6879,7 @@ static int arch_init_sched_domains(const
 	doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
 	if (!doms_cur)
 		doms_cur = &fallback_doms;
+	flags_cur = NULL;
 	cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
 	err = build_sched_domains(doms_cur);
 	register_sched_domain_sysctl();

@@ -6896,6 +6908,16 @@ static void detach_destroy_domains(const
 	arch_destroy_sched_domains(cpu_map);
 }
 
+/* handle null as 0s array */
+static inline int flags_equal(int *cur, int idx_cur, int *new, int idx_new)
+{
+	if (!new)
+		return (!cur || !cur[idx_cur]);
+	if (!cur)
+		return (!new[idx_new]);
+	return (cur[idx_cur] == new[idx_new]);
+}
+
 /*
  * Partition sched domains as specified by the 'ndoms_new'
  * cpumasks in the array doms_new[] of cpumasks. This compares

@@ -6917,7 +6939,7 @@ static void detach_destroy_domains(const
  *
  * Call with hotplug lock held
  */
-void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
+void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, int *flags_new)
 {
 	int i, j;
 

@@ -6929,13 +6951,15 @@ void partition_sched_domains(int ndoms_n
 	if (doms_new == NULL) {
 		ndoms_new = 1;
 		doms_new = &fallback_doms;
+		flags_new = NULL;
 		cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
 	}
 
 	/* Destroy deleted domains */
 	for (i = 0; i < ndoms_cur; i++) {
 		for (j = 0; j < ndoms_new; j++) {
-			if (cpus_equal(doms_cur[i], doms_new[j]))
+			if (cpus_equal(doms_cur[i], doms_new[j])
+			    && flags_equal(flags_cur, i, flags_new, j))
 				goto match1;
 		}
 		/* no match - a current sched domain not in new doms_new[] */

@@ -6947,11 +6971,13 @@ match1:
 	/* Build new domains */
 	for (i = 0; i < ndoms_new; i++) {
 		for (j = 0; j < ndoms_cur; j++) {
-			if (cpus_equal(doms_new[i], doms_cur[j]))
+			if (cpus_equal(doms_new[i], doms_cur[j])
+			    && flags_equal(flags_new, i, flags_cur, j))
 				goto match2;
 		}
 		/* no match - add a new doms_new */
-		build_sched_domains(doms_new + i);
+		__build_sched_domains(doms_new + i,
+					flags_new ? flags_new[i] : 0);
 match2:
 		;
 	}

@@ -6959,7 +6985,9 @@ match2:
 	/* Remember the new sched domains */
 	if (doms_cur != &fallback_doms)
 		kfree(doms_cur);
+	kfree(flags_cur);	/* kfree(NULL) is safe */
 	doms_cur = doms_new;
+	flags_cur = flags_new;
 	ndoms_cur = ndoms_new;
 
 	register_sched_domain_sysctl();

Index: GIT-torvalds/include/asm-ia64/topology.h
===================================================================

include/asm-ia64/topology.h +1 -1

@@ -93,7 +93,7 @@ void build_cpu_to_node_map(void);
 	.cache_nice_tries	= 2,			\
 	.busy_idx		= 3,			\
 	.idle_idx		= 2,			\
-	.newidle_idx		= 0, /* unused */	\
+	.newidle_idx		= 2,			\
 	.wake_idx		= 1,			\
 	.forkexec_idx		= 1,			\
 	.flags			= SD_LOAD_BALANCE	\

Index: GIT-torvalds/include/asm-sh/topology.h
===================================================================

include/asm-sh/topology.h +1 -1

@@ -16,7 +16,7 @@
 	.cache_nice_tries	= 2,			\
 	.busy_idx		= 3,			\
 	.idle_idx		= 2,			\
-	.newidle_idx		= 0,			\
+	.newidle_idx		= 2,			\
 	.wake_idx		= 1,			\
 	.forkexec_idx		= 1,			\
 	.flags			= SD_LOAD_BALANCE	\

Index: GIT-torvalds/include/asm-x86/topology.h
===================================================================

include/asm-x86/topology.h +1 -2

@@ -129,7 +129,7 @@ extern unsigned long node_remap_size[];
 
 # define SD_CACHE_NICE_TRIES	2
 # define SD_IDLE_IDX		2
-# define SD_NEWIDLE_IDX		0
+# define SD_NEWIDLE_IDX		2
 # define SD_FORKEXEC_IDX	1
 
 #endif
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

Quoted 11 lines · Hidetoshi Seto

The implementation is here.

- Add 2 new cpuset files:
sched_wake_idle_far
sched_balance_newidle_far

- Modify partition_sched_domains() and build_sched_domains() to take flags parameter passed from cpuset.

- Fill newidle_idx for node domains which currently unused but might be required for sched_balance_newidle_far.

Just to be clear; the same effect can be had by poking into:

/proc/sys/kernel/sched_domain/$cpu/$domain/flags

but this interface you now propose gives a more stable interface in that you'd have to re-do your setting after every cpuset change (admittedly those are rare, but I see how it could be a nuisance).

Or do you actually add something that wasn't available through the initial domain interface?

Quoted 17 lines · Peter Zijlstra

Quoted 11 lines · Hidetoshi Seto

The implementation is here.

- Add 2 new cpuset files:
sched_wake_idle_far
sched_balance_newidle_far

- Modify partition_sched_domains() and build_sched_domains() to take flags parameter passed from cpuset.

- Fill newidle_idx for node domains which currently unused but might be required for sched_balance_newidle_far.

Just to be clear; the same effect can be had by poking into:

/proc/sys/kernel/sched_domain/$cpu/$domain/flags

Quoted 3 lines

but this interface you now propose gives a more stable interface in that you'd have to re-do your setting after every cpuset change (admittedly those are rare, but I see how it could be a nuisance).

And the sysctl entry "sched_domain" is not available unless SCHED_DEBUG.

So it is common sense that this sysctl is not open to public yet, and that the expected users are scheduler developers, Ingo and friends.

Quoted 2 lines

Or do you actually add something that wasn't available through the initial domain interface?

At this time I have no idea, but it would be possible if there are something unreasonable on global system but acceptable on a part.

In other words, we can invent other sched_* families which has special effect that "default scheduler" never have.