sched_ext: Fix idle CPU state initialization and validation
Subsystems (3)
Andrea Righi
reviewerPATCH v3 1/1 sched_ext: Fix idle CPU state initialization and validation
The built-in idle masks are initialized with all online CPUs marked idle, but idle tracking currently starts only after sched_ext is fully enabled. This leaves busy CPUs incorrectly advertised as idle during ops.init() and until their next idle transition.
Moreover, the allowed_cpus selftest checks that a remotely selected CPU is no longer present in the idle mask. An idle-to-idle re-pick can re-advertise the CPU before the test performs this check, making the validation inherently racy.
Fix this by enabling built-in idle tracking before ops.init() and refreshing the state of every online CPU under its rq lock. Early tracking updates the built-in masks, while ops.update_idle() notifications remain suppressed until the scheduler is fully enabled.
Also rework the allowed_cpus kselftest to replace the racy remote-CPU check with a stable local invariant that relies on the newly reliable idle-mask state.
Changes in v3:
- Reuse the built-in idle-selection static key instead of introducing a dedicated idle-tracking key (Kuba Piecuch)
- Check the local CPU-idle invariant from both ops.select_cpu() and ops.enqueue() (Kuba Piecuch)
- Inspect the idle mask without modifying it and perform the check before calling scx_bpf_select_cpu_and() (Kuba Piecuch)
- Link to v2: https://lore.kernel.org/all/[email protected]/
Changes in v2:
- Move idle-mask initialization from the selftest into the sched_ext core (Kuba Piecuch)
- Add a dedicated idle-tracking static key so transitions are tracked before the scheduler is fully enabled (Kuba Piecuch)
- Rework the allowed_cpus selftest to validate the stable local CPU-idle invariant
- Link to v1: https://lore.kernel.org/all/[email protected]/
Andrea Righi (2):
sched_ext: Initialize idle masks before ops.init() selftests/sched_ext: Make allowed_cpus idle validation race-free
kernel/sched/ext/ext.h | 5 ++- kernel/sched/ext/idle.c | 35 ++++++++++++++-- .../testing/selftests/sched_ext/allowed_cpus.bpf.c | 49 ++++++++++++++++++---- 3 files changed, 78 insertions(+), 11 deletions(-)
Andrea Righi
reviewerPATCH 1/2 sched_ext: Initialize idle masks before ops.init()
The built-in idle masks are reset with all online CPUs marked idle, but idle state tracking starts only after the scheduler is fully enabled. As a result, ops.init() can observe busy CPUs as idle, and those CPUs remain incorrectly advertised until their next idle transition.
Enable built-in idle tracking before ops.init() and refresh every online CPU under its rq lock. Once a CPU is refreshed, later transitions keep its state accurate. Keep ops.update_idle() notifications disabled until the scheduler is fully enabled.
--- kernel/sched/ext/ext.h | 5 ++++- kernel/sched/ext/idle.c | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-)
kernel/sched/ext/ext.h +4 -1
@@ -59,11 +59,14 @@ static inline void init_sched_ext_class(void) {}#endif /* CONFIG_SCHED_CLASS_EXT */ #ifdef CONFIG_SCHED_CLASS_EXT +DECLARE_STATIC_KEY_FALSE (scx_builtin_idle_enabled); +void __scx_update_idle (struct rq *rq,bool idle,bool do_notify);static inline void scx_update_idle (struct rq *rq,bool idle,bool do_notify) { -if (scx_enabled ()) +if (scx_enabled () || +static_branch_unlikely (&scx_builtin_idle_enabled))__scx_update_idle (rq, idle, do_notify); }#else
kernel/sched/ext/idle.c +32 -3
@@ -15,7 +15,7 @@#include "sub.h" /* Enable/disable built-in idle CPU selection policy */ -static DEFINE_STATIC_KEY_FALSE (scx_builtin_idle_enabled); +DEFINE_STATIC_KEY_FALSE (scx_builtin_idle_enabled);/* Enable/disable per-node idle cpumasks */ static DEFINE_STATIC_KEY_FALSE (scx_builtin_idle_per_node);
@@ -810,6 +810,15 @@ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)if (static_branch_likely (&scx_builtin_idle_enabled))update_builtin_idle (cpu, idle); +/* +* Idle tracking starts before the scheduler is enabled so that the +* built-in idle masks are accurate when ops.init() runs. Suppress +* ops.update_idle() notifications until the scheduler is fully +* enabled. +*/ +if (!scx_enabled ()) +return ; +/* * ops.update_idle() fires on real idle transitions, indicated by * @do_notify and managed by put_prev_task_idle()/set_next_task_idle().
@@ -838,8 +847,8 @@ static void reset_idle_masks(struct sched_ext_ops *ops)int node;/* -* Consider all online cpus idle. Should converge to the actual state -* quickly. +* Seed all online CPUs as idle. refresh_idle_masks() below corrects +* their state before ops.init() runs. */ if (!(ops->flags &SCX_OPS_BUILTIN_IDLE_PER_NODE )) {cpumask_copy (idle_cpumask (NUMA_NO_NODE )->cpu, cpu_online_mask);
@@ -855,6 +864,23 @@ static void reset_idle_masks(struct sched_ext_ops *ops) } } +static void refresh_idle_masks (void ) +{ +int cpu; + +/* +* Idle tracking is already enabled and the online CPU set is stable. +* Once a CPU is refreshed under its rq lock, subsequent transitions +* keep its state up to date. +*/ +for_each_online_cpu (cpu) { +struct rq *rq =cpu_rq (cpu); + + scoped_guard(rq_lock_irqsave, rq) +update_builtin_idle (cpu, rq->curr == rq->idle); + } +} +void scx_idle_enable (struct sched_ext_ops *ops) {if (!ops->update_idle || (ops->flags &SCX_OPS_KEEP_BUILTIN_IDLE ))
@@ -868,6 +894,9 @@ void scx_idle_enable(struct sched_ext_ops *ops)static_branch_disable_cpuslocked (&scx_builtin_idle_per_node);reset_idle_masks (ops); + +if (static_branch_likely (&scx_builtin_idle_enabled)) +refresh_idle_masks (); }void scx_idle_disable (void )
Signature · 1 line
2.55.0
Tejun Heo
reviewerRe: [PATCH 1/2] sched_ext: Initialize idle masks before ops.init()
Hello,
Quoted 9 lines · Andrea Righi
The built-in idle masks are reset with all online CPUs marked idle, but idle state tracking starts only after the scheduler is fully enabled. As a result,
ops.init()can observe busy CPUs as idle, and those CPUs remain incorrectly advertised until their next idle transition.Enable built-in idle tracking before
ops.init()and refresh every online CPU under its rq lock. Once a CPU is refreshed, later transitions keep its state accurate. Keepops.update_idle()notifications disabled until the scheduler is fully enabled.
While a sched is being loaded, bypass mode is on and when we get out of bypass mode, we set RENOTIFY and trigger kick each CPU, which, if the CPU has been or is entering idle, triggers ops.update_idle(). So, BPF implemented idle tracking gets the actual idle state update when bypass goes off, which makes sense. Would the problem you were seeing go away if we just clear all idle bits on load instead of setting them? The lifting of bypass mode at the end should set idle bits for all actually idle CPUs.
Thanks.
Signature · 1 line
tejun
Andrea Righi
reviewerRe: [PATCH 1/2] sched_ext: Initialize idle masks before ops.init()
Hi Tejun,
Quoted 13 lines · Tejun Heo
Hello,
Quoted 9 lines · Andrea Righi
The built-in idle masks are reset with all online CPUs marked idle, but idle state tracking starts only after the scheduler is fully enabled. As a result,
ops.init()can observe busy CPUs as idle, and those CPUs remain incorrectly advertised until their next idle transition.Enable built-in idle tracking before
ops.init()and refresh every online CPU under its rq lock. Once a CPU is refreshed, later transitions keep its state accurate. Keepops.update_idle()notifications disabled until the scheduler is fully enabled.
Quoted 7 lines
While a sched is being loaded, bypass mode is on and when we get out of bypass mode, we set RENOTIFY and trigger kick each CPU, which, if the CPU has been or is entering idle, triggers
ops.update_idle(). So, BPF implemented idle tracking gets the actual idle state update when bypass goes off, which makes sense. Would the problem you were seeing go away if we just clear all idle bits on load instead of setting them? The lifting of bypass mode at the end should set idle bits for all actually idle CPUs.
Yes, I think clearing the masks should be sufficient: it makes the initial state conservative, so ops.init() sees no idle CPUs instead of potentially seeing busy CPUs as idle.
Once __scx_enabled is set, exiting bypass arms the idle re-notification and reschedules every online CPU, an idle-to-idle re-pick then updates the built-in mask as well and triggers ops.update_idle(). Busy CPUs remain clear. The mask is temporarily incomplete, including during ops.init(), but that should be safe and it will quickly converge to the actual idle state. So everything should work. I'll send a new version with this logic.
Thanks,
-Andrea
Andrea Righi
reviewerPATCH 2/2 selftests/sched_ext: Make allowed_cpus idle validation race-free
In reply to Andrea RighiA remotely selected CPU can be re-advertised as idle by an idle-to-idle re-pick before the BPF program validates the selection. Checking that the selected CPU remains absent from the idle mask is therefore inherently racy.
Validate a stable local invariant instead: a CPU executing ops.select_cpu() or ops.enqueue() in a non-idle scheduling context must not be advertised as idle. Read the idle mask without modifying it and also validate selected CPUs against the requested domain and task affinity.
--- .../selftests/sched_ext/allowed_cpus.bpf.c | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-)
tools/testing/selftests/sched_ext/allowed_cpus.bpf.c +42 -7
@@ -15,15 +15,48 @@ UEI_DEFINE(uei); private(PREF_CPUS) struct bpf_cpumask __kptr * allowed_cpumask;static void -validate_idle_cpu (const struct task_struct *p,const struct cpumask *allowed,s32 cpu) +validate_local_idle_state (void ) { -if (scx_bpf_test_and_clear_cpu_idle (cpu)) -scx_bpf_error ("CPU %d should be marked as busy" , cpu); +const struct cpumask *idle; +struct task_struct *curr; +s32 cpu =bpf_get_smp_processor_id (); +bool cpu_is_idle, curr_is_idle; -if (bpf_cpumask_subset (allowed, p->cpus_ptr) && - !bpf_cpumask_test_cpu (cpu, allowed)) +bpf_rcu_read_lock (); + curr =scx_bpf_cpu_curr (cpu); + curr_is_idle = curr && (curr->flags &PF_IDLE ); +bpf_rcu_read_unlock (); + + idle =scx_bpf_get_idle_cpumask (); + cpu_is_idle =bpf_cpumask_test_cpu (cpu, idle); +scx_bpf_put_idle_cpumask (idle); + +/* +* Unlike a remote selected CPU, the local CPU cannot go through an +* idle re-pick while this callback is running. If it is running a +* non-idle scheduling context, it must not be advertised as idle. +*/ +if (!curr_is_idle && cpu_is_idle) +scx_bpf_error ("running CPU %d should be marked as busy" , cpu); +} + +static void +validate_selected_cpu (const struct task_struct *p,s32 cpu) +{ +const struct cpumask *allowed =cast_mask (allowed_cpumask); + +if (!allowed) { +scx_bpf_error ("allowed domain not initialized" ); +return ; + } + +if (!bpf_cpumask_test_cpu (cpu, allowed))scx_bpf_error ("CPU %d not in the allowed domain for %d (%s)" , cpu, p->pid, p->comm); + +if (!bpf_cpumask_test_cpu (cpu, p->cpus_ptr)) +scx_bpf_error ("CPU %d not in the affinity mask for %d (%s)" , + cpu, p->pid, p->comm); } s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,
@@ -32,6 +65,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,const struct cpumask *allowed;s32 cpu; +validate_local_idle_state (); allowed =cast_mask (allowed_cpumask);if (!allowed) {scx_bpf_error ("allowed domain not initialized" );
@@ -43,7 +77,7 @@ s32 BPF_STRUCT_OPS(allowed_cpus_select_cpu,*/ cpu =scx_bpf_select_cpu_and (p, prev_cpu, wake_flags, allowed,0 );if (cpu >=0 ) { -validate_idle_cpu (p, allowed, cpu); +validate_selected_cpu (p, cpu);scx_bpf_dsq_insert (p,SCX_DSQ_LOCAL ,SCX_SLICE_DFL ,0 );return cpu;
@@ -59,6 +93,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags)scx_bpf_dsq_insert (p,SCX_DSQ_GLOBAL ,SCX_SLICE_DFL ,0 ); +validate_local_idle_state (); allowed =cast_mask (allowed_cpumask);if (!allowed) {scx_bpf_error ("allowed domain not initialized" );
@@ -71,7 +106,7 @@ void BPF_STRUCT_OPS(allowed_cpus_enqueue, struct task_struct *p, u64 enq_flags)*/ cpu =scx_bpf_select_cpu_and (p, prev_cpu,0 , allowed,0 );if (cpu >=0 ) { -validate_idle_cpu (p, allowed, cpu); +validate_selected_cpu (p, cpu);scx_bpf_kick_cpu (cpu,SCX_KICK_IDLE ); } }
Signature · 1 line
2.55.0