Skip to contentSkip to conversation

sched_ext: Enhance built-in idle selection with allowed CPUs

PATCH Superseded Andrea Righi 2 participants 13 messages Mar 7, 2025 – Mar 10, 2025

Subsystems (5)

v2 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Andrea Righi

author reviewer
#0 lore raw

PATCH v2 1/1 sched_ext: Enhance built-in idle selection with allowed CPUs

Many scx schedulers define their own concept of scheduling domains to represent topology characteristics, such as heterogeneous architectures (e.g., big.LITTLE, P-cores/E-cores), or to categorize tasks based on specific properties (e.g., setting the soft-affinity of certain tasks to a subset of CPUs).

Currently, there is no mechanism to share these domains with the built-in idle CPU selection policy. As a result, schedulers often implement their own idle CPU selection policies, which are typically similar to one another, leading to a lot of code duplication.

To address this, extend the built-in idle CPU selection policy introducing the concept of allowed CPUs.

With this concept, BPF schedulers can apply the built-in idle CPU selection policy to a subset of allowed CPUs, allowing them to implement their own scheduling domains while still using the topology optimizations of the built-in policy, preventing code duplication across different schedulers.

To implement this introduce a new helper kfunc scx_bpf_select_cpu_and() that accepts a cpumask of allowed CPUs:

s32 scx_bpf_select_cpu_and(struct task_struct *p,

			   const struct cpumask *cpus_allowed,
			   s32 prev_cpu, u64 wake_flags, u64 flags);

Example usage
=============

s32 BPF_STRUCT_OPS(foo_select_cpu, struct task_struct *p,

		   s32 prev_cpu, u64 wake_flags)
{
	const struct cpumask *dom = task_domain(p) ?: p->cpus_ptr;
	s32 cpu;

	/*
	 * Pick an idle CPU in the task's domain.
	 */
	cpu = scx_bpf_select_cpu_and(p, dom, prev_cpu, wake_flags, 0);
	if (cpu >= 0) {
		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
		return cpu;
	}

	return prev_cpu;
}

Results
=======

Load distribution on a 4 sockets / 4 cores per socket system, simulated using virtme-ng, running a modified version of scx_bpfland that uses the new helper scx_bpf_select_cpu_and() and 0xff00 as allowed domain:

     $ vng --cpu 16,sockets=4,cores=4,threads=1
     ...
     $ stress-ng -c 16
     ...
     $ htop
     ...
       0[                         0.0%]   8[||||||||||||||||||||||||100.0%]
       1[                         0.0%]   9[||||||||||||||||||||||||100.0%]
       2[                         0.0%]  10[||||||||||||||||||||||||100.0%]
       3[                         0.0%]  11[||||||||||||||||||||||||100.0%]
       4[                         0.0%]  12[||||||||||||||||||||||||100.0%]
       5[                         0.0%]  13[||||||||||||||||||||||||100.0%]
       6[                         0.0%]  14[||||||||||||||||||||||||100.0%]
       7[                         0.0%]  15[||||||||||||||||||||||||100.0%]

With scx_bpf_select_cpu_dfl() tasks would be distributed evenly across all the available CPUs.

ChangeLog v1 -> v2:
- rename scx_bpf_select_cpu_pref() to scx_bpf_select_cpu_and() and always select idle CPUs strictly within the allowed domain
- rename preferred CPUs -> allowed CPU
- drop %SCX_PICK_IDLE_IN_PREF (not required anymore)
- deprecate scx_bpf_select_cpu_dfl() in favor of scx_bpf_select_cpu_and() and provide all the required backward compatibility boilerplate

Andrea Righi (6):
sched_ext: idle: Honor idle flags in the built-in idle selection policy
sched_ext: idle: Refactor scx_select_cpu_dfl()
sched_ext: idle: Introduce the concept of allowed CPUs
sched_ext: idle: Introduce scx_bpf_select_cpu_and() selftests/sched_ext: Add test for scx_bpf_select_cpu_and()
sched_ext: idle: Deprecate scx_bpf_select_cpu_dfl()

 Documentation/scheduler/sched-ext.rst              |  11 +-
 kernel/sched/ext.c                                 |  13 +-
 kernel/sched/ext_idle.c                            | 243 +++++++++++++++------
 kernel/sched/ext_idle.h                            |   3 +-
 tools/sched_ext/include/scx/common.bpf.h           |   5 +-
 tools/sched_ext/include/scx/compat.bpf.h           |  37 ++++
 tools/sched_ext/scx_flatcg.bpf.c                   |  12 +-
 tools/sched_ext/scx_simple.bpf.c                   |   9 +-
 tools/testing/selftests/sched_ext/Makefile         |   1 +
 .../testing/selftests/sched_ext/allowed_cpus.bpf.c |  91 ++++++++
 tools/testing/selftests/sched_ext/allowed_cpus.c   |  57 +++++
 .../selftests/sched_ext/enq_select_cpu_fails.bpf.c |  12 +-
 .../selftests/sched_ext/enq_select_cpu_fails.c     |   2 +-
 tools/testing/selftests/sched_ext/exit.bpf.c       |   6 +-
 .../sched_ext/select_cpu_dfl_nodispatch.bpf.c      |  13 +-
 .../sched_ext/select_cpu_dfl_nodispatch.c          |   2 +-
 16 files changed, 405 insertions(+), 112 deletions(-)
 create mode 100644 tools/testing/selftests/sched_ext/allowed_cpus.bpf.c
 create mode 100644 tools/testing/selftests/sched_ext/allowed_cpus.c
author reviewer
#1 lore raw

PATCH 1/6 sched_ext: idle: Honor idle flags in the built-in idle selection policy

Enable passing idle flags (%SCX_PICK_IDLE_*) to scx_select_cpu_dfl(), to enforce strict selection criteria, such as selecting an idle CPU strictly within @prev_cpu's node or choosing only a fully idle SMT core.

This functionality will be exposed through a dedicated kfunc in a separate patch.

---
 kernel/sched/ext.c      |  2 +-
 kernel/sched/ext_idle.c | 41 ++++++++++++++++++++++++++++++-----------
 kernel/sched/ext_idle.h |  2 +-
 3 files changed, 32 insertions(+), 13 deletions(-)
kernel/sched/ext.c +1 -1
index debcd1cf2de9b..5cd878bbd0e39 100644
@@ -3396,7 +3396,7 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 		bool found;
 		s32 cpu;
 
-		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
+		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0, &found);
 		p->scx.selected_cpu = cpu;
 		if (found) {
 			p->scx.slice = SCX_SLICE_DFL;
kernel/sched/ext_idle.c +30 -11

98 lines not on this page · raw message

kernel/sched/ext_idle.h +1 -1
index 68c4307ce4f6f..5c1db6b315f7a 100644
@@ -27,7 +27,7 @@ static inline s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node
 }
 #endif /* CONFIG_SMP */
 
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *found);
+s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags, bool *found);
 void scx_idle_enable(struct sched_ext_ops *ops);
 void scx_idle_disable(void);
 int scx_idle_init(void);
Signature · 1 line
2.48.1
author reviewer
#2 lore raw

PATCH 2/6 sched_ext: idle: Refactor scx_select_cpu_dfl()

In reply to Andrea Righi

Make scx_select_cpu_dfl() more consistent with the other idle-related APIs by returning a negative value when an idle CPU isn't found.

---
 kernel/sched/ext.c      |  9 +++++----
 kernel/sched/ext_idle.c | 38 +++++++++++++++++++-------------------
 kernel/sched/ext_idle.h |  2 +-
 3 files changed, 25 insertions(+), 24 deletions(-)
kernel/sched/ext.c +5 -4
index 5cd878bbd0e39..8c9f36baf7dfd 100644
@@ -3393,16 +3393,17 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 		else
 			return prev_cpu;
 	} else {
-		bool found;
 		s32 cpu;
 
-		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0, &found);
-		p->scx.selected_cpu = cpu;
-		if (found) {
+		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0);
+		if (cpu >= 0) {
 			p->scx.slice = SCX_SLICE_DFL;
 			p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
 			__scx_add_event(SCX_EV_ENQ_SLICE_DFL, 1);
+		} else {
+			cpu = prev_cpu;
 		}
+		p->scx.selected_cpu = cpu;
 
 		if (rq_bypass)
 			__scx_add_event(SCX_EV_BYPASS_DISPATCH, 1);
kernel/sched/ext_idle.c +19 -19
index 16981456ec1ed..4f8a6e46a37a4 100644
@@ -418,15 +418,13 @@ void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops)
  * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because
  * we never call ops.select_cpu() for them, see select_task_rq().
  */
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags, bool *found)
+s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags)
 {
 	const struct cpumask *llc_cpus = NULL;
 	const struct cpumask *numa_cpus = NULL;
 	int node = scx_cpu_node_if_enabled(prev_cpu);
 	s32 cpu;
 
-	*found = false;
-
 	/*
 	 * This is necessary to protect llc_cpus.
 	 */
@@ -465,7 +463,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		if (cpus_share_cache(cpu, prev_cpu) &&
 		    scx_idle_test_and_clear_cpu(prev_cpu)) {
 			cpu = prev_cpu;
-			goto cpu_found;
+			goto out_unlock;
 		}
 
 		/*
@@ -487,7 +485,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		    (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) &&
 		    !cpumask_empty(idle_cpumask(waker_node)->cpu)) {
 			if (cpumask_test_cpu(cpu, p->cpus_ptr))
-				goto cpu_found;
+				goto out_unlock;
 		}
 	}
 
@@ -502,7 +500,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		if (cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) &&
 		    scx_idle_test_and_clear_cpu(prev_cpu)) {
 			cpu = prev_cpu;
-			goto cpu_found;
+			goto out_unlock;
 		}
 
 		/*
@@ -511,7 +509,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		if (llc_cpus) {
 			cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE);
 			if (cpu >= 0)
-				goto cpu_found;
+				goto out_unlock;
 		}
 
 		/*
@@ -520,7 +518,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		if (numa_cpus) {
 			cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE);
 			if (cpu >= 0)
-				goto cpu_found;
+				goto out_unlock;
 		}
 
 		/*
@@ -533,7 +531,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 		 */
 		cpu = scx_pick_idle_cpu(p->cpus_ptr, node, flags | SCX_PICK_IDLE_CORE);
 		if (cpu >= 0)
-			goto cpu_found;
+			goto out_unlock;
 
 		/*
 		 * Give up if we're strictly looking for a full-idle SMT
@@ -550,7 +548,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 	 */
 	if (scx_idle_test_and_clear_cpu(prev_cpu)) {
 		cpu = prev_cpu;
-		goto cpu_found;
+		goto out_unlock;
 	}
 
 	/*
@@ -559,7 +557,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 	if (llc_cpus) {
 		cpu = pick_idle_cpu_in_node(llc_cpus, node, 0);
 		if (cpu >= 0)
-			goto cpu_found;
+			goto out_unlock;
 	}
 
 	/*
@@ -568,7 +566,7 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 	if (numa_cpus) {
 		cpu = pick_idle_cpu_in_node(numa_cpus, node, 0);
 		if (cpu >= 0)
-			goto cpu_found;
+			goto out_unlock;
 	}
 
 	/*
@@ -581,13 +579,8 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64
 	 */
 	cpu = scx_pick_idle_cpu(p->cpus_ptr, node, flags);
 	if (cpu >= 0)
-		goto cpu_found;
+		goto out_unlock;
 
-	cpu = prev_cpu;
-	goto out_unlock;
-
-cpu_found:
-	*found = true;
 out_unlock:
 	rcu_read_unlock();
 
@@ -819,6 +812,9 @@ __bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
 __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
 				       u64 wake_flags, bool *is_idle)
 {
+#ifdef CONFIG_SMP
+	s32 cpu;
+#endif
 	if (!ops_cpu_valid(prev_cpu, NULL))
 		goto prev_cpu;
 
@@ -829,7 +825,11 @@ __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
 		goto prev_cpu;
 
 #ifdef CONFIG_SMP
-	return scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0, is_idle);
+	cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0);
+	if (cpu >= 0) {
+		*is_idle = true;
+		return cpu;
+	}
 #endif
 
 prev_cpu:
kernel/sched/ext_idle.h +1 -1
index 5c1db6b315f7a..511cc2221f7a8 100644
@@ -27,7 +27,7 @@ static inline s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node
 }
 #endif /* CONFIG_SMP */
 
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags, bool *found);
+s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags);
 void scx_idle_enable(struct sched_ext_ops *ops);
 void scx_idle_disable(void);
 int scx_idle_init(void);
Signature · 1 line
2.48.1
author reviewer
#3 lore raw

PATCH 3/6 sched_ext: idle: Introduce the concept of allowed CPUs

In reply to Andrea Righi

Many scx schedulers define their own concept of scheduling domains to represent topology characteristics, such as heterogeneous architectures (e.g., big.LITTLE, P-cores/E-cores), or to categorize tasks based on specific properties (e.g., setting the soft-affinity of certain tasks to a subset of CPUs).

Currently, there is no mechanism to share these domains with the built-in idle CPU selection policy. As a result, schedulers often implement their own idle CPU selection policies, which are typically similar to one another, leading to a lot of code duplication.

To address this, introduce the concept of allowed domain (represented as a cpumask) that can be used by the BPF schedulers to apply the built-in idle CPU selection policy to a subset of preferred CPUs.

With this concept the idle CPU selection policy becomes the following:
- always prioritize CPUs from fully idle SMT cores (if SMT is enabled),
- select the same CPU if it's idle and in the allowed domain,
- select an idle CPU within the same LLC domain, if the LLC domain is a subset of the allowed domain,
- select an idle CPU within the same node, if the node domain is a subset of the allowed domain,
- select an idle CPU within the allowed domain.

If the allowed domain is empty or NULL, the behavior of the built-in idle CPU selection policy remains unchanged.

This only introduces the core concept of allowed domain. This functionality will be exposed through a dedicated kfunc in a separate patch.

---
 kernel/sched/ext.c      |   2 +-
 kernel/sched/ext_idle.c | 128 +++++++++++++++++++++++++++++-----------
 kernel/sched/ext_idle.h |   3 +-
 3 files changed, 97 insertions(+), 36 deletions(-)
kernel/sched/ext.c +1 -1
index 8c9f36baf7dfd..1e9414ffeff01 100644
@@ -3395,7 +3395,7 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 	} else {
 		s32 cpu;
 
-		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, 0);
+		cpu = scx_select_cpu_dfl(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
 		if (cpu >= 0) {
 			p->scx.slice = SCX_SLICE_DFL;
 			p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
kernel/sched/ext_idle.c +94 -34

210 lines not on this page · raw message

kernel/sched/ext_idle.h +2 -1
index 511cc2221f7a8..977f49905f2c7 100644
@@ -27,7 +27,8 @@ static inline s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node
 }
 #endif /* CONFIG_SMP */
 
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags);
+s32 scx_select_cpu_dfl(struct task_struct *p, const struct cpumask *cpus_allowed,
+		       s32 prev_cpu, u64 wake_flags, u64 flags);
 void scx_idle_enable(struct sched_ext_ops *ops);
 void scx_idle_disable(void);
 int scx_idle_init(void);
Signature · 1 line
2.48.1
reviewer
#4 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Hello,

Quoted 2 lines · Andrea Righi

Many scx schedulers define their own concept of scheduling domains to represent topology characteristics, such as heterogeneous architectures

I'm not sure "domain" is a good choice given that sched_domain is already an established construct in kernel and means something specific.

Quoted 4 lines

(e.g., big.LITTLE, P-cores/E-cores), or to categorize tasks based on specific properties (e.g., setting the soft-affinity of certain tasks to a subset of CPUs).

Quoted 8 lines

Currently, there is no mechanism to share these domains with the built-in idle CPU selection policy. As a result, schedulers often implement their own idle CPU selection policies, which are typically similar to one another, leading to a lot of code duplication.

To address this, introduce the concept of allowed domain (represented as a cpumask) that can be used by the BPF schedulers to apply the built-in idle CPU selection policy to a subset of preferred CPUs.

We don't need a new term here, do we? All that's being added is an extra mask when picking CPUs.

Quoted 5 lines

With this concept the idle CPU selection policy becomes the following:
- always prioritize CPUs from fully idle SMT cores (if SMT is enabled),
- select the same CPU if it's idle and in the allowed domain,
- select an idle CPU within the same LLC domain, if the LLC domain is a subset of the allowed domain,

Why not select from the intersection of the same LLC domain and the cpumask?

Quoted 2 lines

- select an idle CPU within the same node, if the node domain is a subset of the allowed domain,

Ditto.

Quoted 8 lines

- select an idle CPU within the allowed domain.

If the allowed domain is empty or NULL, the behavior of the built-in idle CPU selection policy remains unchanged.

This only introduces the core concept of allowed domain. This functionality will be exposed through a dedicated kfunc in a separate patch.

...

kernel/sched/ext_idle.c void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) · line 452

Quoted 3 lines
kernel/sched/ext_idle.c +2 -1
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags)
+s32 scx_select_cpu_dfl(struct task_struct *p, const struct cpumask *cpus_allowed,
+		       s32 prev_cpu, u64 wake_flags, u64 flags)

Maybe rearrange them (p, prev_cpu, wake_flags, and_cpumask, pick_idle_flags) so that the first three args align with select_task_rq() and we don't have three consecutive integer arguments? Two back-to-back flag args increase the chance of subtle bugs.

Thanks.

Signature · 1 line
tejun
author reviewer
#5 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Quoted 8 lines · Tejun Heo

Hello,

Quoted 2 lines · Andrea Righi

Many scx schedulers define their own concept of scheduling domains to represent topology characteristics, such as heterogeneous architectures

I'm not sure "domain" is a good choice given that sched_domain is already an established construct in kernel and means something specific.

Yeah, I agree, we don't want to create ambiguity with sched_domain. How about CPU groups or CPU partitions?

Quoted 10 lines
Quoted 10 lines

(e.g., big.LITTLE, P-cores/E-cores), or to categorize tasks based on specific properties (e.g., setting the soft-affinity of certain tasks to a subset of CPUs).

Currently, there is no mechanism to share these domains with the built-in idle CPU selection policy. As a result, schedulers often implement their own idle CPU selection policies, which are typically similar to one another, leading to a lot of code duplication.

Quoted 6 lines
Quoted 3 lines

To address this, introduce the concept of allowed domain (represented as a cpumask) that can be used by the BPF schedulers to apply the built-in idle CPU selection policy to a subset of preferred CPUs.

We don't need a new term here, do we? All that's being added is an extra mask when picking CPUs.

Right, at the end it's just a cpumask, I'll rephrase this part.

Quoted 8 lines
Quoted 6 lines

With this concept the idle CPU selection policy becomes the following:
- always prioritize CPUs from fully idle SMT cores (if SMT is enabled),
- select the same CPU if it's idle and in the allowed domain,
- select an idle CPU within the same LLC domain, if the LLC domain is a subset of the allowed domain,

Why not select from the intersection of the same LLC domain and the cpumask?

We could do that, but to guarantee the intersection we need to introduce other temporary cpumasks (one for the LLC intersection and another for the NUMA), which is not a big problem, but it can introduce overhead. And most of the time the LLC group is either a subset of the allowed CPUs or vice-versa, so in this case the current logic already works.

The extra cpumask work is needed only when the allowed cpumask spans multiple partial LLCs, which should be rare. So maybe in such cases, we could tolerate the additional overhead of updating an additional temporary cpumask to ensure proper hierarchical semantics (maintaining consistency with the topology hierarchy). WDYT?

Quoted 15 lines
Quoted 3 lines

- select an idle CPU within the same node, if the node domain is a subset of the allowed domain,

Ditto.

Quoted 8 lines

- select an idle CPU within the allowed domain.

If the allowed domain is empty or NULL, the behavior of the built-in idle CPU selection policy remains unchanged.

This only introduces the core concept of allowed domain. This functionality will be exposed through a dedicated kfunc in a separate patch.

...

Quoted 8 lines

kernel/sched/ext_idle.c void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) · line 452

Quoted 3 lines
kernel/sched/ext_idle.c +2 -1
-s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, u64 flags)
+s32 scx_select_cpu_dfl(struct task_struct *p, const struct cpumask *cpus_allowed,
+		       s32 prev_cpu, u64 wake_flags, u64 flags)

Maybe rearrange them (p, prev_cpu, wake_flags, and_cpumask, pick_idle_flags) so that the first three args align with select_task_rq() and we don't have three consecutive integer arguments? Two back-to-back flag args increase the chance of subtle bugs.

Good idea. I even introduced a bug while I was updating the kselftests, because I switched wake_flags and idle flags... so yeah, will definitely do that.

Thanks!
-Andrea

reviewer
#6 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Hello,

Quoted 14 lines · Andrea Righi
Quoted 7 lines
Quoted 5 lines

With this concept the idle CPU selection policy becomes the following:
- always prioritize CPUs from fully idle SMT cores (if SMT is enabled),
- select the same CPU if it's idle and in the allowed domain,
- select an idle CPU within the same LLC domain, if the LLC domain is a subset of the allowed domain,

Why not select from the intersection of the same LLC domain and the cpumask?

We could do that, but to guarantee the intersection we need to introduce other temporary cpumasks (one for the LLC intersection and another for the NUMA), which is not a big problem, but it can introduce overhead. And most of the time the LLC group is either a subset of the allowed CPUs or vice-versa, so in this case the current logic already works.

Quoted 5 lines

The extra cpumask work is needed only when the allowed cpumask spans multiple partial LLCs, which should be rare. So maybe in such cases, we could tolerate the additional overhead of updating an additional temporary cpumask to ensure proper hierarchical semantics (maintaining consistency with the topology hierarchy). WDYT?

Would just using a pre-allocated cpumask to do pre-and on @cpus_allowed work? This won't only be used for topology support (e.g. soft partitioning in scx_layered and scx_mitosis may want to use multi-topology-unit spanning subsets) and I'm not sure assuming and optimizing for that is a good idea for generic API.

We can do something simple now. Note that if we want to optimize it, we can introduce cpumask_any_and_and_distribute(). There already is cpumask_first_and_and(), so the pattern isn't new and the only extra bitops we need to add is find_next_and_and_bit_wrap(). There's already find_first_and_and_bit(), so I don't think it will be all that difficult to add.

Thanks.

Signature · 1 line
tejun
author reviewer
#7 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Quoted 23 lines · Tejun Heo

Hello,

Quoted 14 lines · Andrea Righi
Quoted 7 lines
Quoted 5 lines

With this concept the idle CPU selection policy becomes the following:
- always prioritize CPUs from fully idle SMT cores (if SMT is enabled),
- select the same CPU if it's idle and in the allowed domain,
- select an idle CPU within the same LLC domain, if the LLC domain is a subset of the allowed domain,

Why not select from the intersection of the same LLC domain and the cpumask?

We could do that, but to guarantee the intersection we need to introduce other temporary cpumasks (one for the LLC intersection and another for the NUMA), which is not a big problem, but it can introduce overhead. And most of the time the LLC group is either a subset of the allowed CPUs or vice-versa, so in this case the current logic already works.

Quoted 5 lines

The extra cpumask work is needed only when the allowed cpumask spans multiple partial LLCs, which should be rare. So maybe in such cases, we could tolerate the additional overhead of updating an additional temporary cpumask to ensure proper hierarchical semantics (maintaining consistency with the topology hierarchy). WDYT?

Quoted 5 lines

Would just using a pre-allocated cpumask to do pre-and on @cpus_allowed work? This won't only be used for topology support (e.g. soft partitioning in scx_layered and scx_mitosis may want to use multi-topology-unit spanning subsets) and I'm not sure assuming and optimizing for that is a good idea for generic API.

We can pre-allocate two additional (per-cpu) cpumasks to do:
- cpumask_and(numa_cpus, numa_span(cpu), cpus_allowed)
- cpumask_and(llc_cpus, llc_span(cpu), cpus_allowed)

And update/use them only when it's needed. In this way the API would be generic without making any implicit assumption about @cpus_allowed.

If you don't see any issues, I'll go ahead with this approach.

Quoted 7 lines

We can do something simple now. Note that if we want to optimize it, we can introduce cpumask_any_and_and_distribute(). There already is cpumask_first_and_and(), so the pattern isn't new and the only extra bitops we need to add is find_next_and_and_bit_wrap(). There's already find_first_and_and_bit(), so I don't think it will be all that difficult to add.

Yes, it'd be really nice to have cpumask_any_and_and_distribute(), but I agree that we can start simple and provide this as a separate improvement later on. Looks like a good plan.

Thanks,
-Andrea

reviewer
#8 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Hello,

Quoted 6 lines · Andrea Righi
Quoted 5 lines

Would just using a pre-allocated cpumask to do pre-and on @cpus_allowed work? This won't only be used for topology support (e.g. soft partitioning in scx_layered and scx_mitosis may want to use multi-topology-unit spanning subsets) and I'm not sure assuming and optimizing for that is a good idea for generic API.

Quoted 6 lines

We can pre-allocate two additional (per-cpu) cpumasks to do:
- cpumask_and(numa_cpus, numa_span(cpu), cpus_allowed)
- cpumask_and(llc_cpus, llc_span(cpu), cpus_allowed)

And update/use them only when it's needed. In this way the API would be generic without making any implicit assumption about @cpus_allowed.

I'm not quite following why two masks would be necessary. The user is providing two masks and and'ing those two masks result in a single cpus_allowed mask which can then be passed down to the existing pick functions, no?

Thanks.

Signature · 1 line
tejun
author reviewer
#9 lore raw

Re: [PATCH 3/6] sched_ext: idle: Introduce the concept of allowed CPUs

Quoted 13 lines · Tejun Heo

Hello,

Quoted 10 lines · Andrea Righi
Quoted 5 lines

Would just using a pre-allocated cpumask to do pre-and on @cpus_allowed work? This won't only be used for topology support (e.g. soft partitioning in scx_layered and scx_mitosis may want to use multi-topology-unit spanning subsets) and I'm not sure assuming and optimizing for that is a good idea for generic API.

We can pre-allocate two additional (per-cpu) cpumasks to do:
- cpumask_and(numa_cpus, numa_span(cpu), cpus_allowed)
- cpumask_and(llc_cpus, llc_span(cpu), cpus_allowed)

Quoted 7 lines
Quoted 2 lines

And update/use them only when it's needed. In this way the API would be generic without making any implicit assumption about @cpus_allowed.

I'm not quite following why two masks would be necessary. The user is providing two masks and and'ing those two masks result in a single cpus_allowed mask which can then be passed down to the existing pick functions, no?

When you say the user is providing two masks, you mean p->cpus_ptr and @cpus_allowed, right? Or am I missing something?

So, internally we have three levels of cpumasks, used in this order:
1) p->cpus_ptr & cpus_allowed & llc_span(prev_cpu)
2) p->cpus_ptr & cpus_allowed & numa_span(prev_cpu)
3) p->cpus_ptr & cpus_allowed

The current logic (without @cpus_allowed) is applying LLC and NUMA optimization only for tasks that can run on all CPUs (p->cpus_ptr == all), to avoid doing extra "and" operations internally and simply use llc_span(prev_cpu) and numa_span(prev_cpu).

With @cpus_allowed this optimization doesn't work anymore and we can't just re-apply the current logic to "p->cpus_ptr & cpus_allowed", since it would result in ignoring the LLC and NUMA cpumasks.

Maybe we could use a single pre-allocated temporary cpumask and do the "and" at each step when it's needed, instead of using two separate cpumasks to evaluate "cpus_allowed & llc_span(prev_cpu)" and "cpus_allowed & numa_span(prev_cpu). Is this what you mean?

Thanks,
-Andrea

author reviewer
#10 lore raw

PATCH 4/6 sched_ext: idle: Introduce scx_bpf_select_cpu_and()

In reply to Andrea Righi

Provide a new kfunc that can be used to apply the built-in idle CPU selection policy to a subset of allowed CPU:

s32 scx_bpf_select_cpu_and(struct task_struct *p,

			   const struct cpumask *cpus_allowed,
			   s32 prev_cpu, u64 wake_flags, u64 flags);

This new helper is basically an extension of scx_bpf_select_cpu_dfl(). However, when an idle CPU can't be found, it returns a negative value instead of @prev_cpu, aligning its behavior more closely with scx_bpf_pick_idle_cpu().

It also accepts %SCX_PICK_IDLE_* flags, which can be used to enforce strict selection to @prev_cpu's node (%SCX_PICK_IDLE_IN_NODE), or to request only a full-idle SMT core (%SCX_PICK_IDLE_CORE), while applying the built-in selection logic.

With this helper, BPF schedulers can apply the built-in idle CPU selection policy restricted to a generic CPU domain.

In the future we can also consider to deprecate scx_bpf_select_cpu_dfl() and replace it with scx_bpf_select_cpu_and(), as the latter provides the same functionality, with the addition of the allowed domain logic.

Example usage
=============

Possible usage in ops.select_cpu():

s32 BPF_STRUCT_OPS(foo_select_cpu, struct task_struct *p,

		  s32 prev_cpu, u64 wake_flags)
{
	const struct cpumask *dom = task_domain(p) ?: p->cpus_ptr;
	s32 cpu;

	cpu = scx_bpf_select_cpu_and(p, dom, prev_cpu, wake_flags, 0);
	if (cpu >= 0) {
		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
		return cpu;
	}

	return prev_cpu;
}

Results
=======

Load distribution on a 4 sockets / 4 cores per socket system, simulated using virtme-ng, running a modified version of scx_bpfland that uses the new helper scx_bpf_select_cpu_and() and 0xff00 as allowed domain:

 $ vng --cpu 16,sockets=4,cores=4,threads=1
 ...
 $ stress-ng -c 16
 ...
 $ htop
 ...
   0[                         0.0%]   8[||||||||||||||||||||||||100.0%]
   1[                         0.0%]   9[||||||||||||||||||||||||100.0%]
   2[                         0.0%]  10[||||||||||||||||||||||||100.0%]
   3[                         0.0%]  11[||||||||||||||||||||||||100.0%]
   4[                         0.0%]  12[||||||||||||||||||||||||100.0%]
   5[                         0.0%]  13[||||||||||||||||||||||||100.0%]
   6[                         0.0%]  14[||||||||||||||||||||||||100.0%]
   7[                         0.0%]  15[||||||||||||||||||||||||100.0%]

With scx_bpf_select_cpu_dfl() tasks would be distributed evenly across all the available CPUs.

---
 kernel/sched/ext.c                       |  1 +
 kernel/sched/ext_idle.c                  | 42 ++++++++++++++++++++++++
 tools/sched_ext/include/scx/common.bpf.h |  2 ++
 3 files changed, 45 insertions(+)
kernel/sched/ext.c +1 -0
index 1e9414ffeff01..a3c7c835ba857 100644
@@ -465,6 +465,7 @@ struct sched_ext_ops {
 	 * idle CPU tracking and the following helpers become unavailable:
 	 *
 	 * - scx_bpf_select_cpu_dfl()
+	 * - scx_bpf_select_cpu_and()
 	 * - scx_bpf_test_and_clear_cpu_idle()
 	 * - scx_bpf_pick_idle_cpu()
 	 *
kernel/sched/ext_idle.c +42 -0

54 lines not on this page · raw message

tools/sched_ext/include/scx/common.bpf.h +2 -0
index dc4333d23189f..16e38b46807fd 100644
@@ -48,6 +48,8 @@ static inline void ___vmlinux_h_sanity_check___(void)
 
 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
 s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
+s32 scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed,
+			    s32 prev_cpu, u64 wake_flags, u64 flags) __ksym __weak;
 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
 void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak;
 u32 scx_bpf_dispatch_nr_slots(void) __ksym;
Signature · 1 line
2.48.1
author reviewer
#11 lore raw

PATCH 5/6 selftests/sched_ext: Add test for scx_bpf_select_cpu_and()

In reply to Andrea Righi

Add a selftest to validate the behavior of the built-in idle CPU selection policy applied to a subset of allowed CPUs, using scx_bpf_select_cpu_and().

---
 tools/testing/selftests/sched_ext/Makefile    |  1 +
 .../selftests/sched_ext/allowed_cpus.bpf.c    | 91 +++++++++++++++++++
 .../selftests/sched_ext/allowed_cpus.c        | 57 ++++++++++++
 3 files changed, 149 insertions(+)
 create mode 100644 tools/testing/selftests/sched_ext/allowed_cpus.bpf.c
 create mode 100644 tools/testing/selftests/sched_ext/allowed_cpus.c
tools/testing/selftests/sched_ext/Makefile +1 -0
index f4531327b8e76..e9d5bc575f806 100644
@@ -173,6 +173,7 @@ auto-test-targets :=			\
 	maybe_null			\
 	minimal				\
 	numa				\
+	allowed_cpus			\
 	prog_run			\
 	reload_loop			\
 	select_cpu_dfl			\
tools/testing/selftests/sched_ext/allowed_cpus.bpf.c new file +91 -0

91 lines not on this page · raw message

tools/testing/selftests/sched_ext/allowed_cpus.c new file +57 -0

57 lines not on this page · raw message

Signature · 1 line
2.48.1
author reviewer
#12 lore raw

PATCH 6/6 sched_ext: idle: Deprecate scx_bpf_select_cpu_dfl()

In reply to Andrea Righi

With the introduction of scx_bpf_select_cpu_and(), we can deprecate scx_bpf_select_cpu_dfl(), as it offers only a subset of features and it's also more consistent with other idle-related APIs (returning a negative value when no idle CPU is found).

Therefore, mark scx_bpf_select_cpu_dfl() as deprecated (printing a warning when it's used), update all the scheduler examples and kselftests to adopt the new API, and ensure backward (source and binary) compatibility by providing the necessary macros and hooks.

Support for scx_bpf_select_cpu_dfl() can be maintained until v6.17.

---
 Documentation/scheduler/sched-ext.rst         | 11 +++---
 kernel/sched/ext.c                            |  3 +-
 kernel/sched/ext_idle.c                       | 18 ++-------
 tools/sched_ext/include/scx/common.bpf.h      |  3 +-
 tools/sched_ext/include/scx/compat.bpf.h      | 37 +++++++++++++++++++
 tools/sched_ext/scx_flatcg.bpf.c              | 12 +++---
 tools/sched_ext/scx_simple.bpf.c              |  9 +++--
 .../sched_ext/enq_select_cpu_fails.bpf.c      | 12 +-----
 .../sched_ext/enq_select_cpu_fails.c          |  2 +-
 tools/testing/selftests/sched_ext/exit.bpf.c  |  6 ++-
 .../sched_ext/select_cpu_dfl_nodispatch.bpf.c | 13 +++----
 .../sched_ext/select_cpu_dfl_nodispatch.c     |  2 +-
 12 files changed, 73 insertions(+), 55 deletions(-)
Documentation/scheduler/sched-ext.rst +5 -6
index 0993e41353db7..a3abdbf682681 100644
@@ -142,15 +142,14 @@ optional. The following modified excerpt is from
                        s32 prev_cpu, u64 wake_flags)
     {
             s32 cpu;
-            /* Need to initialize or the BPF verifier will reject the program */
-            bool direct = false;
 
-            cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &direct);
-
-            if (direct)
+            cpu = scx_bpf_select_cpu_and(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
+            if (cpu >= 0)
                     scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+                    return cpu;
+            }
 
-            return cpu;
+            return prev_cpu;
     }
 
     /*
kernel/sched/ext.c +1 -2
index a3c7c835ba857..5614a2f7e8dbb 100644
@@ -464,13 +464,12 @@ struct sched_ext_ops {
 	 * state. By default, implementing this operation disables the built-in
 	 * idle CPU tracking and the following helpers become unavailable:
 	 *
-	 * - scx_bpf_select_cpu_dfl()
 	 * - scx_bpf_select_cpu_and()
 	 * - scx_bpf_test_and_clear_cpu_idle()
 	 * - scx_bpf_pick_idle_cpu()
 	 *
 	 * The user also must implement ops.select_cpu() as the default
-	 * implementation relies on scx_bpf_select_cpu_dfl().
+	 * implementation relies on scx_bpf_select_cpu_and().
 	 *
 	 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
 	 * tracking.
kernel/sched/ext_idle.c +4 -14
index 1977b1368da7f..e127aab4a286b 100644
@@ -855,26 +855,16 @@ __bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
 #endif
 }
 
-/**
- * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu()
- * @p: task_struct to select a CPU for
- * @prev_cpu: CPU @p was on previously
- * @wake_flags: %SCX_WAKE_* flags
- * @is_idle: out parameter indicating whether the returned CPU is idle
- *
- * Can only be called from ops.select_cpu() if the built-in CPU selection is
- * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
- * @p, @prev_cpu and @wake_flags match ops.select_cpu().
- *
- * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
- * currently idle and thus a good candidate for direct dispatching.
- */
+/* Provided for backward binary compatibility, will be removed in v6.17. */
 __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
 				       u64 wake_flags, bool *is_idle)
 {
 #ifdef CONFIG_SMP
 	s32 cpu;
 #endif
+	printk_deferred_once(KERN_WARNING
+			"sched_ext: scx_bpf_select_cpu_dfl() deprecated in favor of scx_bpf_select_cpu_and()");
+
 	if (!ops_cpu_valid(prev_cpu, NULL))
 		goto prev_cpu;
 
tools/sched_ext/include/scx/common.bpf.h +2 -1
index 16e38b46807fd..9b830f17b6380 100644
@@ -47,7 +47,8 @@ static inline void ___vmlinux_h_sanity_check___(void)
 }
 
 s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
+s32 scx_bpf_select_cpu_dfl(struct task_struct *p,
+			   s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym __weak;
 s32 scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed,
 			    s32 prev_cpu, u64 wake_flags, u64 flags) __ksym __weak;
 void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
tools/sched_ext/include/scx/compat.bpf.h +37 -0
index 9252e1a00556f..2f43b3fa16b73 100644
@@ -225,6 +225,43 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
 	 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) :			\
 	 scx_bpf_pick_any_cpu(cpus_allowed, flags))
 
+/**
+ * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu().
+ * We will preserve this compatible helper until v6.17.
+ *
+ * @p: task_struct to select a CPU for
+ * @prev_cpu: CPU @p was on previously
+ * @wake_flags: %SCX_WAKE_* flags
+ * @is_idle: out parameter indicating whether the returned CPU is idle
+ *
+ * Can only be called from ops.select_cpu() if the built-in CPU selection is
+ * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
+ * @p, @prev_cpu and @wake_flags match ops.select_cpu().
+ *
+ * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
+ * currently idle and thus a good candidate for direct dispatching.
+ */
+#define scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle)		\
+({										\
+	s32 __cpu;								\
+										\
+	if (bpf_ksym_exists(scx_bpf_select_cpu_and)) {				\
+		__cpu = scx_bpf_select_cpu_and((p), (p)->cpus_ptr,		\
+					       (prev_cpu), (wake_flags), 0);	\
+		if (__cpu >= 0) {						\
+			*(is_idle) = true;					\
+		} else {							\
+			*(is_idle) = false;					\
+			__cpu = (prev_cpu);					\
+		}								\
+	} else {								\
+		__cpu = scx_bpf_select_cpu_dfl((p), (prev_cpu),			\
+					       (wake_flags), (is_idle));	\
+	}									\
+										\
+	__cpu;									\
+})
+
 /*
  * Define sched_ext_ops. This may be expanded to define multiple variants for
  * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH().
tools/sched_ext/scx_flatcg.bpf.c +6 -6
index 2c720e3ecad59..3117f3ab2ff52 100644
@@ -317,15 +317,12 @@ static void set_bypassed_at(struct task_struct *p, struct fcg_task_ctx *taskc)
 s32 BPF_STRUCT_OPS(fcg_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
 {
 	struct fcg_task_ctx *taskc;
-	bool is_idle = false;
 	s32 cpu;
 
-	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
-
 	taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
 	if (!taskc) {
 		scx_bpf_error("task_ctx lookup failed");
-		return cpu;
+		return prev_cpu;
 	}
 
 	/*
@@ -333,13 +330,16 @@ s32 BPF_STRUCT_OPS(fcg_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake
 	 * idle. Follow it and charge the cgroup later in fcg_stopping() after
 	 * the fact.
 	 */
-	if (is_idle) {
+	cpu = scx_bpf_select_cpu_and(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
+	if (cpu >= 0) {
 		set_bypassed_at(p, taskc);
 		stat_inc(FCG_STAT_LOCAL);
 		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+
+		return cpu;
 	}
 
-	return cpu;
+	return prev_cpu;
 }
 
 void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
tools/sched_ext/scx_simple.bpf.c +5 -4
index e6de99dba7db6..6ad01dd5b34ab 100644
@@ -54,16 +54,17 @@ static void stat_inc(u32 idx)
 
 s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
 {
-	bool is_idle = false;
 	s32 cpu;
 
-	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
-	if (is_idle) {
+	cpu = scx_bpf_select_cpu_and(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
+	if (cpu >= 0) {
 		stat_inc(0);	/* count local queueing */
 		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+
+		return cpu;
 	}
 
-	return cpu;
+	return prev_cpu;
 }
 
 void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c +1 -11
index a7cf868d5e311..6b9d1cd0bb948 100644
@@ -9,10 +9,6 @@
 
 char _license[] SEC("license") = "GPL";
 
-/* Manually specify the signature until the kfunc is added to the scx repo. */
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
-			   bool *found) __ksym;
-
 s32 BPF_STRUCT_OPS(enq_select_cpu_fails_select_cpu, struct task_struct *p,
 		   s32 prev_cpu, u64 wake_flags)
 {
@@ -22,14 +18,8 @@ s32 BPF_STRUCT_OPS(enq_select_cpu_fails_select_cpu, struct task_struct *p,
 void BPF_STRUCT_OPS(enq_select_cpu_fails_enqueue, struct task_struct *p,
 		    u64 enq_flags)
 {
-	/*
-	 * Need to initialize the variable or the verifier will fail to load.
-	 * Improving these semantics is actively being worked on.
-	 */
-	bool found = false;
-
 	/* Can only call from ops.select_cpu() */
-	scx_bpf_select_cpu_dfl(p, 0, 0, &found);
+	scx_bpf_select_cpu_and(p, p->cpus_ptr, 0, 0, 0);
 
 	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
 }
tools/testing/selftests/sched_ext/enq_select_cpu_fails.c +1 -1
index a80e3a3b3698c..c964444998667 100644
@@ -52,7 +52,7 @@ static void cleanup(void *ctx)
 
 struct scx_test enq_select_cpu_fails = {
 	.name = "enq_select_cpu_fails",
-	.description = "Verify we fail to call scx_bpf_select_cpu_dfl() "
+	.description = "Verify we fail to call scx_bpf_select_cpu_and() "
 		       "from ops.enqueue()",
 	.setup = setup,
 	.run = run,
tools/testing/selftests/sched_ext/exit.bpf.c +4 -2
index 4bc36182d3ffc..172c0cd339947 100644
@@ -20,12 +20,14 @@ UEI_DEFINE(uei);
 s32 BPF_STRUCT_OPS(exit_select_cpu, struct task_struct *p,
 		   s32 prev_cpu, u64 wake_flags)
 {
-	bool found;
+	s32 cpu;
 
 	if (exit_point == EXIT_SELECT_CPU)
 		EXIT_CLEANLY();
 
-	return scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
+	cpu = scx_bpf_select_cpu_and(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
+
+	return cpu >= 0 ? cpu : prev_cpu;
 }
 
 void BPF_STRUCT_OPS(exit_enqueue, struct task_struct *p, u64 enq_flags)
tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c +6 -7
index 815f1d5d61ac4..42e344bc9e743 100644
@@ -27,10 +27,6 @@ struct {
 	__type(value, struct task_ctx);
 } task_ctx_stor SEC(".maps");
 
-/* Manually specify the signature until the kfunc is added to the scx repo. */
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
-			   bool *found) __ksym;
-
 s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_select_cpu, struct task_struct *p,
 		   s32 prev_cpu, u64 wake_flags)
 {
@@ -43,10 +39,13 @@ s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_select_cpu, struct task_struct *p,
 		return -ESRCH;
 	}
 
-	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags,
-				     &tctx->force_local);
+	cpu = scx_bpf_select_cpu_and(p, p->cpus_ptr, prev_cpu, wake_flags, 0);
+	if (cpu >= 0) {
+		tctx->force_local = true;
+		return cpu;
+	}
 
-	return cpu;
+	return prev_cpu;
 }
 
 void BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_enqueue, struct task_struct *p,
tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c +1 -1
index 9b5d232efb7f6..2f450bb14e8d9 100644
@@ -66,7 +66,7 @@ static void cleanup(void *ctx)
 
 struct scx_test select_cpu_dfl_nodispatch = {
 	.name = "select_cpu_dfl_nodispatch",
-	.description = "Verify behavior of scx_bpf_select_cpu_dfl() in "
+	.description = "Verify behavior of scx_bpf_select_cpu_and() in "
 		       "ops.select_cpu()",
 	.setup = setup,
 	.run = run,
Signature · 1 line
2.48.1