sched_ext: add p->scx.tid and SCX_OPS_TID_TO_TASK lookup
Subsystems (2)
Tejun Heo
reviewerPATCH v2 1/1 sched_ext: add p->scx.tid and SCX_OPS_TID_TO_TASK lookup
BPF schedulers that can't hold task_struct pointers (arena-backed ones in particular) key tasks by pid. During exit, pid is released before the task finishes passing through scheduler callbacks, so a dying task becomes invisible to the BPF side mid-schedule. scx_qmap hits this: an exiting task's dispatch callback can't recover its queue entry, stalling dispatch until SCX_EXIT_ERROR_STALL.
Add a unique non-zero u64 p->scx.tid assigned at fork that survives the full task lifetime including exit. scx_bpf_tid_to_task() looks up the task; unlike bpf_task_from_pid(), it handles exiting tasks.
The lookup costs an rhashtable insert/remove under scx_tasks_lock, so root schedulers opt in via SCX_OPS_TID_TO_TASK. Sub-schedulers that set the flag to declare a dependency are rejected at attach if root didn't opt in.
scx_qmap converted: keys tasks by tid and enables SCX_OPS_ENQ_EXITING. Pre-patch it stalls within seconds under a non-leader-exec workload; with the patch it runs cleanly.
v2: Guard scx_root deref in scx_bpf_tid_to_task() error path. The kfunc
is registered via scx_kfunc_set_any and reachable from tracing and syscall programs when no scheduler is attached (Cheng-Yang Chou).
--- include/linux/sched/ext.h | 9 + kernel/sched/ext.c | 146 +++++++++++++++++++++++++++++-- kernel/sched/ext_internal.h | 20 +++- tools/sched_ext/include/scx/common.bpf.h | 1 tools/sched_ext/scx_qmap.bpf.c | 13 +- 5 files changed, 172 insertions(+), 17 deletions(-)
include/linux/sched/ext.h +9 -0
@@ -203,6 +203,15 @@ struct sched_ext_entity {u64 core_sched_at;/* see scx_prio_less() */ #endif +/* +* Unique non-zero task ID assigned at fork. Persists across exec and +* is never reused. Lets BPF schedulers identify tasks without storing +* kernel pointers - arena-backed schedulers being one example. See +* scx_bpf_tid_to_task(). +*/ +u64 tid; +struct rhash_head tid_hash_node;/* see SCX_OPS_TID_TO_TASK */ +/* BPF scheduler modifiable fields */ /*
kernel/sched/ext.c +137 -9
255 lines not on this page · raw message
kernel/sched/ext_internal.h +18 -2
@@ -13,6 +13,9 @@ enum scx_consts {SCX_DSP_MAX_LOOPS =32 ,SCX_WATCHDOG_MAX_TIMEOUT =30 *HZ , +/* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ +SCX_TID_CHUNK =1024 , +SCX_EXIT_BT_LEN =64 ,SCX_EXIT_MSG_LEN =1024 , SCX_EXIT_DUMP_DFL_LEN = 32768,
@@ -138,7 +141,8 @@ enum scx_ops_flags {* To mask this problem, by default, unhashed tasks are automatically * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't * depend on pid lookups and wants to handle these tasks directly, the -* following flag can be used. +* following flag can be used. With %SCX_OPS_TID_TO_TASK, +* scx_bpf_tid_to_task() can find exiting tasks reliably. */ SCX_OPS_ENQ_EXITING = 1LLU << 2,
@@ -189,6 +193,17 @@ enum scx_ops_flags {*/ SCX_OPS_ALWAYS_ENQ_IMMED =1LLU <<7 , +/* +* Maintain a mapping from p->scx.tid to task_struct so the BPF +* scheduler can recover task pointers from stored tids via +* scx_bpf_tid_to_task(). +* +* Only the root scheduler turns this on. A sub-sched may set the flag +* to declare a dependency on the lookup; if the root scheduler hasn't +* enabled it, attaching the sub-sched is rejected. +*/ +SCX_OPS_TID_TO_TASK =1LLU <<8 , +SCX_OPS_ALL_FLAGS =SCX_OPS_KEEP_BUILTIN_IDLE |SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING |
@@ -196,7 +211,8 @@ enum scx_ops_flags {SCX_OPS_ALLOW_QUEUED_WAKEUP |SCX_OPS_SWITCH_PARTIAL |SCX_OPS_BUILTIN_IDLE_PER_NODE | -SCX_OPS_ALWAYS_ENQ_IMMED , +SCX_OPS_ALWAYS_ENQ_IMMED | +SCX_OPS_TID_TO_TASK ,/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56,
tools/sched_ext/include/scx/common.bpf.h +1 -0
@@ -99,6 +99,7 @@ s32 scx_bpf_task_cpu(const struct task_sstruct rq *scx_bpf_cpu_rq (s32 cpu) __ksym;struct rq *scx_bpf_locked_rq (void ) __ksym;struct task_struct *scx_bpf_cpu_curr (s32 cpu) __ksym__weak ; +struct task_struct *scx_bpf_tid_to_task (u64 tid) __ksym__weak ; u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak;
tools/sched_ext/scx_qmap.bpf.c +7 -6
@@ -127,7 +127,8 @@ struct task_ctx { struct task_ctx __arena *q_next; /* queue link, NULL if tail */ struct task_ctx __arena *q_prev; /* queue link, NULL if head */ struct qmap_fifo __arena *fifo; /* queue we're on, NULL if not queued */ -s32 pid; +u64 tid; +s32 pid;/* for dump only */ bool force_local;/* Dispatch directly to local_dsq */ bool highpri;u64 core_sched_seq;
@@ -547,7 +548,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cif (!taskc)break ; - p =bpf_task_from_pid (taskc->pid); + p =scx_bpf_tid_to_task (taskc->tid);if (!p)continue ;
@@ -598,8 +599,6 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cif (!bpf_cpumask_test_cpu (cpu, p->cpus_ptr))scx_bpf_kick_cpu (scx_bpf_task_cpu (p),0 ); -bpf_task_release (p); - batch--; cpuc->dsp_cnt--;if (!batch || !scx_bpf_dispatch_nr_slots ()) {
@@ -724,6 +723,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_t taskc->q_next =NULL ; taskc->q_prev =NULL ; taskc->fifo =NULL ; + taskc->tid = p->scx.tid; taskc->pid = p->pid; taskc->force_local = false; taskc->highpri = false;
@@ -776,7 +776,7 @@ void BPF_STRUCT_OPS(qmap_dump, struct sc/* * Walk the queue lists without locking - kfunc calls (scx_bpf_dump) * aren't in the verifier's kfunc_spin_allowed() list so we can't hold -* a lock and dump. Best-effort; racing may print stale pids but the +* a lock and dump. Best-effort; racing may print stale tids but the * walk is bounded by bpf_repeat() so it always terminates. */ bpf_for(i, 0, 5) {
@@ -785,7 +785,7 @@ void BPF_STRUCT_OPS(qmap_dump, struct sc bpf_repeat(4096) {if (!taskc)break ; -scx_bpf_dump (" %d" , taskc->pid); +scx_bpf_dump (" %d:%llu" , taskc->pid, taskc->tid); taskc = taskc->q_next; }scx_bpf_dump ("\n" );
@@ -1159,6 +1159,7 @@ void BPF_STRUCT_OPS(qmap_sub_detach, str } SCX_OPS_DEFINE(qmap_ops, + .flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK, .select_cpu = (void *)qmap_select_cpu, .enqueue = (void *)qmap_enqueue, .dequeue = (void *)qmap_dequeue,