sched_ext: Split sub-scheduler implementation into sub.c
Subsystems (2)
Tejun Heo
reviewerPATCH 1/1 sched_ext: Split sub-scheduler implementation into sub.c
Hello,
The sub-scheduler implementation has grown and will keep growing. Move it out of ext.c into a new kernel/sched/ext/sub.c. The first three patches are mechanical prep (prefix file-local helpers, expose shared internals, inline a few trivial helpers) so the move itself stays pure code motion. No functional change.
Based on sched_ext/for-7.3 (5df6a4506d06) with sched_ext/for-7.2-fixes (b7d9c359e5cf) assumed merged.
Tejun Heo (4):
sched_ext: Prefix file-local ext.c helpers exposed by the sub.c split
sched_ext: Expose the ext.c internals used by the sub.c split
sched_ext: Inline small ext.c helpers shared across the sub.c split
sched_ext: Split sub-scheduler implementation into sub.c
kernel/sched/build_policy.c | 2 + kernel/sched/ext/ext.c | 1002 +++++-------------------------------------- kernel/sched/ext/internal.h | 136 +++++- kernel/sched/ext/sub.c | 668 +++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 51 +++ 5 files changed, 973 insertions(+), 886 deletions(-)
Thanks.
--
tejun
Tejun Heo
reviewerPATCH 1/4 sched_ext: Prefix file-local ext.c helpers exposed by the sub.c split
A later change moves the sub-scheduler implementation out of ext.c into its own file, from where it calls a number of file-local ext.c helpers. Give those helpers the scx_ prefix that cross-file sched_ext symbols carry, ahead of the move so the mechanical rename stays out of the code-motion patch. No functional change.
--- kernel/sched/ext/ext.c | 192 ++++++++++++++++++------------------ kernel/sched/ext/internal.h | 2 +- 2 files changed, 97 insertions(+), 97 deletions(-)
kernel/sched/ext/ext.c +96 -96
@@ -369,7 +369,7 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p)return __setscheduler_class (p->policy, p->prio); } -static struct scx_dispatch_q *bypass_dsq (struct scx_sched *sch,s32 cpu) +static struct scx_dispatch_q *scx_bypass_dsq (struct scx_sched *sch,s32 cpu) {return &per_cpu_ptr (sch->pcpu, cpu)->bypass_dsq; }
@@ -392,11 +392,11 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c sch =scx_parent (sch);#endif /* CONFIG_EXT_SUB_SCHED */ -return bypass_dsq (sch, cpu); +return scx_bypass_dsq (sch, cpu); }/** -* bypass_dsp_enabled - Check if bypass dispatch path is enabled +* scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled * @sch: scheduler to check * * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled
@@ -408,9 +408,9 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c* * This function checks bypass_dsp_enable_depth which is managed separately from * bypass_depth to enable this decoupling. See enable_bypass_dsp() and -* disable_bypass_dsp(). +* scx_disable_bypass_dsp(). */ -static bool bypass_dsp_enabled (struct scx_sched *sch) +static bool scx_bypass_dsp_enabled (struct scx_sched *sch) {return unlikely (atomic_read (&sch->bypass_dsp_enable_depth)); }
@@ -1079,7 +1079,7 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) }/** -* ops_sanitize_err - Sanitize a -errno value +* scx_ops_sanitize_err - Sanitize a -errno value * @sch: scx_sched to error out on error * @ops_name: operation to blame on failure * @err: -errno value to sanitize
@@ -1091,7 +1091,7 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)* value fails IS_ERR() test after being encoded with ERR_PTR() and then is * handled as a pointer. */ -static int ops_sanitize_err (struct scx_sched *sch,const char *ops_name,s32 err) +static int scx_ops_sanitize_err (struct scx_sched *sch,const char *ops_name,s32 err) {if (err <0 && err >= -MAX_ERRNO )return err;
@@ -1251,7 +1251,7 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsqschedule_deferred (rq); } -static void schedule_reenq_local (struct rq *rq,u64 reenq_flags) +static void scx_schedule_reenq_local (struct rq *rq,u64 reenq_flags) {struct scx_sched *root =rcu_dereference_sched (scx_root);
@@ -1347,8 +1347,8 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en* to the CPU or dequeued. In both cases, the only way @p can go back to * the BPF sched is through enqueueing. If being inserted into a local * DSQ with IMMED, persist the state until the next enqueueing event in -* do_enqueue_task() so that we can maintain IMMED protection through -* e.g. SAVE/RESTORE cycles and slice extensions. +* scx_do_enqueue_task() so that we can maintain IMMED protection +* through e.g. SAVE/RESTORE cycles and slice extensions. */ if (enq_flags &SCX_ENQ_IMMED ) {if (unlikely (dsq->id !=SCX_DSQ_LOCAL )) {
@@ -1371,7 +1371,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en* done yet, @p can't go on the CPU immediately. Re-enqueue. */ if (unlikely (dsq->nr >1 || !rq_is_open (rq, enq_flags))) -schedule_reenq_local (rq,0 ); +scx_schedule_reenq_local (rq,0 ); } }
@@ -1488,9 +1488,9 @@ static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq } } -static void dispatch_enqueue (struct scx_sched *sch,struct rq *rq, -struct scx_dispatch_q *dsq,struct task_struct *p, -u64 enq_flags) +static void scx_dispatch_enqueue (struct scx_sched *sch,struct rq *rq, +struct scx_dispatch_q *dsq,struct task_struct *p, +u64 enq_flags) {bool is_local = dsq->id ==SCX_DSQ_LOCAL ;
@@ -1638,7 +1638,7 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void dispatch_dequeue (struct rq *rq,struct task_struct *p) +static void scx_dispatch_dequeue (struct rq *rq,struct task_struct *p) {struct scx_dispatch_q *dsq = p->scx.dsq;bool is_local = dsq == &rq->scx.local_dsq;
@@ -1692,8 +1692,8 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) }/* -* Abbreviated version of dispatch_dequeue() that can be used when both @p's rq -* and dsq are locked. +* Abbreviated version of scx_dispatch_dequeue() that can be used when both +* @p's rq and dsq are locked. */ static void dispatch_dequeue_locked (struct task_struct *p,struct scx_dispatch_q *dsq)
@@ -1774,10 +1774,10 @@ static void mark_direct_dispatch(struct scx_sched *sch,* - direct_dispatch(): cleared on the synchronous enqueue path, deferred * dispatch keeps the state until consumed * - process_ddsp_deferred_locals(): cleared after consuming deferred state, -* - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch +* - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch * verdict is ignored (local/global/bypass) -* - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred -* cancellation and holding_cpu races +* - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering +* deferred cancellation and holding_cpu races * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by * the scx_bypass() loop, so that stale state is not reused by a subsequent * scheduler instance
@@ -1838,7 +1838,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, ddsp_enq_flags = p->scx.ddsp_enq_flags;clear_direct_dispatch (p); -dispatch_enqueue (sch, rq, dsq, p, ddsp_enq_flags |SCX_ENQ_CLEAR_OPSS ); +scx_dispatch_enqueue (sch, rq, dsq, p, ddsp_enq_flags |SCX_ENQ_CLEAR_OPSS ); }static bool scx_rq_online (struct rq *rq)
@@ -1853,8 +1853,8 @@ static bool scx_rq_online(struct rq *rq)return likely ((rq->scx.flags &SCX_RQ_ONLINE ) &&cpu_active (cpu_of (rq))); } -static void do_enqueue_task (struct rq *rq,struct task_struct *p,u64 enq_flags, -int sticky_cpu) +static void scx_do_enqueue_task (struct rq *rq,struct task_struct *p,u64 enq_flags, +int sticky_cpu) {struct scx_sched *sch =scx_task_sched (p);struct task_struct **ddsp_taskp;
@@ -1941,7 +1941,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,direct_dispatch (sch, p, enq_flags);return ; local_norefill: -dispatch_enqueue (sch, rq, &rq->scx.local_dsq, p, enq_flags); +scx_dispatch_enqueue (sch, rq, &rq->scx.local_dsq, p, enq_flags);return ; local: dsq = &rq->scx.local_dsq;
@@ -1962,7 +1962,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,touch_core_sched (rq, p);refill_task_slice_dfl (sch, p);clear_direct_dispatch (p); -dispatch_enqueue (sch, rq, dsq, p, enq_flags); +scx_dispatch_enqueue (sch, rq, dsq, p, enq_flags); }static bool task_runnable (const struct task_struct *p)
@@ -2031,7 +2031,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_if (rq->scx.nr_running ==1 )dl_server_start (&rq->ext_server); -do_enqueue_task (rq, p, enq_flags, sticky_cpu); +scx_do_enqueue_task (rq, p, enq_flags, sticky_cpu);if (sticky_cpu >=0 ) p->scx.sticky_cpu =-1 ;
@@ -2167,7 +2167,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ rq->scx.nr_running--;sub_nr_running (rq,1 ); -dispatch_dequeue (rq, p); +scx_dispatch_dequeue (rq, p);clear_direct_dispatch (p);return true; }
@@ -2215,7 +2215,7 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl* - A higher-priority wakes up while SCX dispatch is in progress. */ if (rq->scx.nr_immed) -schedule_reenq_local (rq,0 ); +scx_schedule_reenq_local (rq,0 ); } static void move_local_task_to_local_dsq(struct scx_sched *sch,
@@ -2380,7 +2380,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,* values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still -* points to this CPU. See dispatch_dequeue() for the counterpart. +* points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue.
@@ -2485,14 +2485,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,dispatch_dequeue_locked (p, src_dsq);raw_spin_unlock (&src_dsq->lock); -dispatch_enqueue (sch, dst_rq, dst_dsq, p, enq_flags); +scx_dispatch_enqueue (sch, dst_rq, dst_dsq, p, enq_flags); }return dst_rq; } -static bool consume_dispatch_q (struct scx_sched *sch,struct rq *rq, -struct scx_dispatch_q *dsq,u64 enq_flags) +static bool scx_consume_dispatch_q (struct scx_sched *sch,struct rq *rq, +struct scx_dispatch_q *dsq,u64 enq_flags) {struct task_struct *p; retry:
@@ -2538,11 +2538,11 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,return false; } -static bool consume_global_dsq (struct scx_sched *sch,struct rq *rq) +static bool scx_consume_global_dsq (struct scx_sched *sch,struct rq *rq) {int node =cpu_to_node (cpu_of (rq)); -return consume_dispatch_q (sch, rq, &sch->pnode[node]->global_dsq,0 ); +return scx_consume_dispatch_q (sch, rq, &sch->pnode[node]->global_dsq,0 ); }/**
@@ -2575,8 +2575,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,* If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { -dispatch_enqueue (sch, rq, dst_dsq, p, - enq_flags |SCX_ENQ_CLEAR_OPSS ); +scx_dispatch_enqueue (sch, rq, dst_dsq, p, + enq_flags |SCX_ENQ_CLEAR_OPSS );return ; }
@@ -2614,13 +2614,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,*/ if (src_rq == dst_rq) { p->scx.holding_cpu =-1 ; -dispatch_enqueue (sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); +scx_dispatch_enqueue (sch, dst_rq, &dst_rq->scx.local_dsq, p, + enq_flags); }else if (unlikely (!task_can_run_on_remote_rq (sch, p, dst_rq, true))) { p->scx.holding_cpu =-1 ; fallback = true; -dispatch_enqueue (sch, src_rq,find_global_dsq (sch,task_cpu (p)), - p, enq_flags |SCX_ENQ_GDSQ_FALLBACK ); +scx_dispatch_enqueue (sch, src_rq,find_global_dsq (sch,task_cpu (p)), + p, enq_flags |SCX_ENQ_GDSQ_FALLBACK ); }else {move_remote_task_to_local_dsq (p, enq_flags, src_rq, dst_rq);
@@ -2708,10 +2708,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, goto retry;case SCX_OPSS_QUEUEING :/* -* do_enqueue_task() is in the process of transferring the task -* to the BPF scheduler while holding @p's rq lock. As we aren't -* holding any kernel or BPF resource that the enqueue path may -* depend upon, it's safe to wait. +* scx_do_enqueue_task() is in the process of transferring the +* task to the BPF scheduler while holding @p's rq lock. As we +* aren't holding any kernel or BPF resource that the enqueue +* path may depend upon, it's safe to wait. */ wait_ops_state (p, opss); goto retry;
@@ -2724,10 +2724,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,if (dsq->id ==SCX_DSQ_LOCAL )dispatch_to_local_dsq (sch, rq, dsq, p, enq_flags);else -dispatch_enqueue (sch, rq, dsq, p, enq_flags |SCX_ENQ_CLEAR_OPSS ); +scx_dispatch_enqueue (sch, rq, dsq, p, enq_flags |SCX_ENQ_CLEAR_OPSS ); } -static void flush_dispatch_buf (struct scx_sched *sch,struct rq *rq) +static void scx_flush_dispatch_buf (struct scx_sched *sch,struct rq *rq) {struct scx_dsp_ctx *dspc = &this_cpu_ptr (sch->pcpu)->dsp_ctx;u32 u;
@@ -2771,13 +2771,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&scx_task_on_sched (sch, prev); -if (consume_global_dsq (sch, rq)) +if (scx_consume_global_dsq (sch, rq))return true; -if (bypass_dsp_enabled (sch)) { +if (scx_bypass_dsp_enabled (sch)) {/* if @sch is bypassing, only the bypass DSQs are active */ if (scx_bypassing (sch, cpu)) -return consume_dispatch_q (sch, rq,bypass_dsq (sch, cpu),0 ); +return scx_consume_dispatch_q (sch, rq,scx_bypass_dsq (sch, cpu),0 );#ifdef CONFIG_EXT_SUB_SCHED /*
@@ -2795,7 +2795,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,struct scx_sched_pcpu *pcpu =per_cpu_ptr (sch->pcpu, cpu);if (!(pcpu->bypass_host_seq++ %SCX_BYPASS_HOST_NTH ) && -consume_dispatch_q (sch, rq,bypass_dsq (sch, cpu),0 )) { +scx_consume_dispatch_q (sch, rq,scx_bypass_dsq (sch, cpu),0 )) {__scx_add_event (sch,SCX_EV_SUB_BYPASS_DISPATCH ,1 );return true; }
@@ -2808,8 +2808,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, dspc->rq = rq;/* -* The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, -* the local DSQ might still end up empty after a successful +* The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq +* lock, the local DSQ might still end up empty after a successful * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() * produced some tasks, retry. The BPF scheduler may depend on this * looping behavior to simplify its implementation.
@@ -2828,7 +2828,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, rq->scx.sub_dispatch_prev =NULL ; } -flush_dispatch_buf (sch, rq); +scx_flush_dispatch_buf (sch, rq);if ((prev->scx.flags &SCX_TASK_QUEUED ) && prev->scx.slice) { rq->scx.flags |=SCX_RQ_BAL_KEEP ;
@@ -2836,7 +2836,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, }if (rq->scx.local_dsq.nr)return true; -if (consume_global_dsq (sch, rq)) +if (scx_consume_global_dsq (sch, rq))return true;/*
@@ -2859,8 +2859,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,* queued. Without this fallback, bypassed tasks could stall if the host * scheduler's ops.dispatch() doesn't yield any tasks. */ -if (bypass_dsp_enabled (sch)) -return consume_dispatch_q (sch, rq,bypass_dsq (sch, cpu),0 ); +if (scx_bypass_dsp_enabled (sch)) +return scx_consume_dispatch_q (sch, rq,scx_bypass_dsq (sch, cpu),0 );return false; }
@@ -2939,7 +2939,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)* between the IMMED queueing and the subsequent scheduling event. */ if (unlikely (rq->scx.local_dsq.nr >1 && rq->scx.nr_immed)) -schedule_reenq_local (rq,0 ); +scx_schedule_reenq_local (rq,0 ); rq->scx.flags &= ~SCX_RQ_IN_BALANCE ;return true;
@@ -2955,7 +2955,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)* dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue (rq, p,SCX_DEQ_CORE_SCHED_EXEC ); -dispatch_dequeue (rq, p); +scx_dispatch_dequeue (rq, p); } p->se.exec_start =rq_clock_task (rq);
@@ -3067,10 +3067,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,if (p->scx.slice && !scx_bypassing (sch,cpu_of (rq))) {if (p->scx.flags &SCX_TASK_IMMED ) { p->scx.flags |=SCX_TASK_REENQ_PREEMPTED ; -do_enqueue_task (rq, p,SCX_ENQ_REENQ ,-1 ); +scx_do_enqueue_task (rq, p,SCX_ENQ_REENQ ,-1 ); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK ; }else { -dispatch_enqueue (sch, rq, &rq->scx.local_dsq, p,SCX_ENQ_HEAD ); +scx_dispatch_enqueue (sch, rq, &rq->scx.local_dsq, p,SCX_ENQ_HEAD ); } goto switch_class; }
@@ -3088,9 +3088,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,if (next &&sched_class_above (&ext_sched_class, next->sched_class)) {WARN_ON_ONCE (sched_cpu_cookie_match (rq, p) && !(sch->ops.flags &SCX_OPS_ENQ_LAST )); -do_enqueue_task (rq, p,SCX_ENQ_LAST ,-1 ); +scx_do_enqueue_task (rq, p,SCX_ENQ_LAST ,-1 ); }else { -do_enqueue_task (rq, p,0 ,-1 ); +scx_do_enqueue_task (rq, p,0 ,-1 ); } }
@@ -3562,7 +3562,7 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo ret =SCX_CALL_OP_RET (sch, init_task,NULL , p, &args);if (unlikely (ret)) { - ret =ops_sanitize_err (sch,"init_task" , ret); + ret =scx_ops_sanitize_err (sch,"init_task" , ret);return ret; } }
@@ -4107,7 +4107,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)if (!local_task_should_reenq (p, &reenq_flags, &reason))continue ; -dispatch_dequeue (rq, p); +scx_dispatch_dequeue (rq, p);if (WARN_ON_ONCE (p->scx.flags &SCX_TASK_REENQ_REASON_MASK )) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK ;
@@ -4119,7 +4119,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)list_for_each_entry_safe (p, n, &tasks, scx.dsq_list.node) {list_del_init (&p->scx.dsq_list.node); -do_enqueue_task (rq, p,SCX_ENQ_REENQ ,-1 ); +scx_do_enqueue_task (rq, p,SCX_ENQ_REENQ ,-1 ); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK ; nr_enqueued++;
@@ -4234,7 +4234,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK ; p->scx.flags |= reason; -do_enqueue_task (task_rq, p,SCX_ENQ_REENQ ,-1 ); +scx_do_enqueue_task (task_rq, p,SCX_ENQ_REENQ ,-1 ); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK ;
@@ -4354,7 +4354,7 @@ int scx_tg_online(struct task_group *tg) ret =SCX_CALL_OP_RET (sch, cgroup_init,NULL , tg->css.cgroup, &args);if (ret) - ret =ops_sanitize_err (sch,"cgroup_init" , ret); + ret =scx_ops_sanitize_err (sch,"cgroup_init" , ret); }if (ret ==0 ) tg->scx.flags |=SCX_TG_ONLINE |SCX_TG_INITED ;
@@ -4422,7 +4422,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) p->scx.cgrp_moving_from =NULL ; } -return ops_sanitize_err (sch,"cgroup_prep_move" , ret); +return scx_ops_sanitize_err (sch,"cgroup_prep_move" , ret); }void scx_cgroup_move_task (struct task_struct *p)
@@ -4700,7 +4700,7 @@ static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) goto out_unlock_dsq;/* -* Mark dead by invalidating ->id to prevent dispatch_enqueue() from +* Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks.
@@ -4928,7 +4928,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)*/ WARN_ON_ONCE (!list_empty (&pcpu->deferred_reenq_local.node)); -exit_dsq (bypass_dsq (sch, cpu)); +exit_dsq (scx_bypass_dsq (sch, cpu)); }free_percpu (sch->pcpu);
@@ -5239,7 +5239,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, u32 nr_donor_target, u32 nr_donee_target) {struct rq *donor_rq =cpu_rq (donor); -struct scx_dispatch_q *donor_dsq =bypass_dsq (sch, donor); +struct scx_dispatch_q *donor_dsq =scx_bypass_dsq (sch, donor);struct task_struct *p, *n;struct scx_dsq_list_node cursor =INIT_DSQ_LIST_CURSOR (cursor, donor_dsq,0 );s32 delta =READ_ONCE (donor_dsq->nr) - nr_donor_target;
@@ -5287,7 +5287,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,if (donee >= nr_cpu_ids)continue ; - donee_dsq =bypass_dsq (sch, donee); + donee_dsq =scx_bypass_dsq (sch, donee);/* * $p's rq is not locked but $p's DSQ lock protects its
@@ -5308,7 +5308,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,* between bypass DSQs. */ dispatch_dequeue_locked (p, donor_dsq); -dispatch_enqueue (sch,cpu_rq (donee), donee_dsq, p,SCX_ENQ_NESTED ); +scx_dispatch_enqueue (sch,cpu_rq (donee), donee_dsq, p,SCX_ENQ_NESTED );/* * $donee might have been idle and need to be woken up. No need
@@ -5351,7 +5351,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)/* count the target tasks and CPUs */ for_each_cpu_and (cpu, cpu_online_mask, node_mask) { -u32 nr =READ_ONCE (bypass_dsq (sch, cpu)->nr); +u32 nr =READ_ONCE (scx_bypass_dsq (sch, cpu)->nr); nr_tasks += nr; nr_cpus++;
@@ -5373,7 +5373,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)cpumask_clear (donee_mask);for_each_cpu_and (cpu, cpu_online_mask, node_mask) { -if (READ_ONCE (bypass_dsq (sch, cpu)->nr) < nr_target) +if (READ_ONCE (scx_bypass_dsq (sch, cpu)->nr) < nr_target)cpumask_set_cpu (cpu, donee_mask); }
@@ -5384,7 +5384,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)break ;if (cpumask_test_cpu (cpu, donee_mask))continue ; -if (READ_ONCE (bypass_dsq (sch, cpu)->nr) <= nr_donor_target) +if (READ_ONCE (scx_bypass_dsq (sch, cpu)->nr) <= nr_donor_target)continue ; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask,
@@ -5395,7 +5395,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)resched_cpu (cpu);for_each_cpu_and (cpu, cpu_online_mask, node_mask) { -u32 nr =READ_ONCE (bypass_dsq (sch, cpu)->nr); +u32 nr =READ_ONCE (scx_bypass_dsq (sch, cpu)->nr); after_min =min (nr, after_min); after_max =max (nr, after_max);
@@ -5421,7 +5421,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer)int node;u32 intv_us; -if (!bypass_dsp_enabled (sch)) +if (!scx_bypass_dsp_enabled (sch))return ;for_each_node_with_cpus (node)
@@ -5487,9 +5487,9 @@ static void enable_bypass_dsp(struct scx_sched *sch)* dispatch enabled while a descendant is bypassing, which is all that's * required. * -* bypass_dsp_enabled() test is used to determine whether to enter the -* bypass dispatch handling path from both bypassing and hosting scheds. -* Bump enable depth on both @sch and bypass dispatch host. +* scx_bypass_dsp_enabled() test is used to determine whether to enter +* the bypass dispatch handling path from both bypassing and hosting +* scheds. Bump enable depth on both @sch and bypass dispatch host. */ ret =atomic_inc_return (&sch->bypass_dsp_enable_depth);WARN_ON_ONCE (ret <=0 );
@@ -5509,7 +5509,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) }/* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp (struct scx_sched *sch) +static void scx_disable_bypass_dsp (struct scx_sched *sch) {s32 ret;
@@ -5654,7 +5654,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass)/* disarming must come after moving all tasks out of the bypass DSQs */ if (!bypass) -disable_bypass_dsp (sch); +scx_disable_bypass_dsp (sch); unlock:raw_spin_unlock_irqrestore (&scx_bypass_lock, flags); }
@@ -6003,7 +6003,7 @@ static void scx_sub_disable(struct scx_sched *sch)* DSQs for us. */ synchronize_rcu_expedited (); -disable_bypass_dsp (sch); +scx_disable_bypass_dsp (sch);scx_unlink_sched (sch);
@@ -6810,7 +6810,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, }for_each_possible_cpu (cpu) { - ret =init_dsq (bypass_dsq (sch, cpu),SCX_DSQ_BYPASS , sch); + ret =init_dsq (scx_bypass_dsq (sch, cpu),SCX_DSQ_BYPASS , sch);if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu;
@@ -6963,7 +6963,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,for_each_possible_cpu (cpu) {if (cpu == bypass_fail_cpu)break ; -exit_dsq (bypass_dsq (sch, cpu)); +exit_dsq (scx_bypass_dsq (sch, cpu)); }free_percpu (sch->pcpu); err_free_pnode:
@@ -7007,7 +7007,7 @@ static int check_hotplug_seq(struct scx_sched *sch,return 0 ; } -static int validate_ops (struct scx_sched *sch,const struct sched_ext_ops *ops) +static int scx_validate_ops (struct scx_sched *sch,const struct sched_ext_ops *ops) {/* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
@@ -7170,7 +7170,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)if (sch->ops.init) { ret =SCX_CALL_OP_RET (sch, init,NULL );if (ret) { - ret =ops_sanitize_err (sch,"init" , ret); + ret =scx_ops_sanitize_err (sch,"init" , ret);cpus_read_unlock ();scx_error (sch,"ops.init() failed (%d)" , ret); goto err_disable;
@@ -7203,7 +7203,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)cpus_read_unlock (); - ret =validate_ops (sch, ops); + ret =scx_validate_ops (sch, ops);if (ret) goto err_disable;
@@ -7545,7 +7545,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work)if (sch->ops.init) { ret =SCX_CALL_OP_RET (sch, init,NULL );if (ret) { - ret =ops_sanitize_err (sch,"init" , ret); + ret =scx_ops_sanitize_err (sch,"init" , ret);scx_error (sch,"ops.init() failed (%d)" , ret); goto err_disable; }
@@ -7560,7 +7560,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work)if (ret) goto err_disable; -if (validate_ops (sch, ops)) +if (scx_validate_ops (sch, ops)) goto err_disable;struct scx_sub_attach_args sub_attach_args = {
@@ -7571,7 +7571,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work) ret =SCX_CALL_OP_RET (parent, sub_attach,NULL , &sub_attach_args);if (ret) { - ret =ops_sanitize_err (sch,"sub_attach" , ret); + ret =scx_ops_sanitize_err (sch,"sub_attach" , ret);scx_error (sch,"parent rejected (%d)" , ret); goto err_disable; }
@@ -8830,7 +8830,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,/* * If the BPF scheduler keeps calling this function repeatedly, it can -* cause similar live-lock conditions as consume_dispatch_q(). +* cause similar live-lock conditions as scx_consume_dispatch_q(). */ if (unlikely (READ_ONCE (sch->aborting)))return false;
@@ -8991,7 +8991,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, dspc = &this_cpu_ptr (sch->pcpu)->dsp_ctx; -flush_dispatch_buf (sch, dspc->rq); +scx_flush_dispatch_buf (sch, dspc->rq); dsq =find_user_dsq (sch, dsq_id);if (unlikely (!dsq)) {
@@ -8999,7 +8999,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,return false; } -if (consume_dispatch_q (sch, dspc->rq, dsq, enq_flags)) { +if (scx_consume_dispatch_q (sch, dspc->rq, dsq, enq_flags)) {/* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched
@@ -10683,7 +10683,7 @@ static int __init scx_init(void)/* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH (priv, priv);/* -* cid-form must end exactly at @priv - validate_ops() skips +* cid-form must end exactly at @priv - scx_validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields * past the BPF allocation would be UB. */
kernel/sched/ext/internal.h +1 -1
@@ -1172,7 +1172,7 @@ struct scx_sched {u64 bypass_timestamp;s32 bypass_depth; -/* bypass dispatch path enable state, see bypass_dsp_enabled() */ +/* bypass dispatch path enable state, see scx_bypass_dsp_enabled() */ unsigned long bypass_dsp_claim;atomic_t bypass_dsp_enable_depth;
Signature · 1 line
2.54.0
Tejun Heo
reviewerPATCH 2/4 sched_ext: Expose the ext.c internals used by the sub.c split
In reply to Tejun HeoThe sub-scheduler implementation is about to move into its own sub.c, from where it calls a set of ext.c helpers and shares a few ext.c globals. Make those reachable across the new file boundary ahead of the move.
No functional change.
--- kernel/sched/ext/ext.c | 109 ++++++++++++------------------------ kernel/sched/ext/internal.h | 77 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 73 deletions(-)
kernel/sched/ext/ext.c +36 -73
@@ -20,7 +20,7 @@#include "arena.h" #include "idle.h" -static DEFINE_RAW_SPINLOCK (scx_sched_lock); +DEFINE_RAW_SPINLOCK (scx_sched_lock);/* * NOTE: sched_ext is in the process of growing multiple scheduler support and
@@ -39,14 +39,14 @@ struct scx_sched __rcu *scx_root;static LIST_HEAD (scx_sched_all);#ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { +const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched , hash_node), .insecure_elasticity = true,/* inserted under scx_sched_lock */ }; -static struct rhashtable scx_sched_hash; +struct rhashtable scx_sched_hash;#endif /* see SCX_OPS_TID_TO_TASK */
@@ -68,9 +68,9 @@ static DEFINE_RAW_SPINLOCK(scx_tasks_lock);static LIST_HEAD (scx_tasks);/* ops enable/disable */ -static DEFINE_MUTEX (scx_enable_mutex); +DEFINE_MUTEX (scx_enable_mutex);DEFINE_STATIC_KEY_FALSE (__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM (scx_fork_rwsem); +DEFINE_PERCPU_RWSEM (scx_fork_rwsem);static atomic_t scx_enable_state_var =ATOMIC_INIT (SCX_DISABLED );static DEFINE_RAW_SPINLOCK (scx_bypass_lock);static bool scx_init_task_enabled;
@@ -101,7 +101,7 @@ static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);* tasks for the sub-sched being enabled. Use a global variable instead of a * per-task field as all enables are serialized. */ -static struct scx_sched *scx_enabling_sub_sched; +struct scx_sched *scx_enabling_sub_sched;#else #define scx_enabling_sub_sched (struct scx_sched *)NULL#endif /* CONFIG_EXT_SUB_SCHED */
@@ -676,12 +676,12 @@ struct bpf_iter_scx_dsq { } __attribute__((aligned(8))); -static u32 scx_get_task_state (const struct task_struct *p) +u32 scx_get_task_state (const struct task_struct *p) {return p->scx.flags &SCX_TASK_STATE_MASK ; } -static void scx_set_task_state (struct task_struct *p,u32 state) +void scx_set_task_state (struct task_struct *p,u32 state) {u32 prev_state =scx_get_task_state (p);bool warn = false;
@@ -721,23 +721,6 @@ static void scx_set_task_state(struct task_struct *p, u32 state) p->scx.flags |= state; } -/* -* SCX task iterator. -*/ -struct scx_task_iter { -struct sched_ext_entity cursor; -struct task_struct *locked_task; -struct rq *rq; -struct rq_flags rf; -u32 cnt; -bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED -struct cgroup *cgrp; -struct cgroup_subsys_state *css_pos; -struct css_task_iter css_iter; -#endif -}; -/** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init
@@ -766,7 +749,7 @@ struct scx_task_iter {* All tasks which existed when the iteration started are guaranteed to be * visited as long as they are not dead. */ -static void scx_task_iter_start (struct scx_task_iter *iter,struct cgroup *cgrp) +void scx_task_iter_start (struct scx_task_iter *iter,struct cgroup *cgrp) {memset (iter,0 ,sizeof (*iter));
@@ -805,7 +788,7 @@ static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)* This function can be safely called anytime during an iteration. The next * iterator operation will automatically restore the necessary locking. */ -static void scx_task_iter_unlock (struct scx_task_iter *iter) +void scx_task_iter_unlock (struct scx_task_iter *iter) {__scx_task_iter_rq_unlock (iter);if (iter->list_locked) {
@@ -848,7 +831,7 @@ static void scx_task_iter_relock(struct scx_task_iter *iter,* which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */ -static void scx_task_iter_stop (struct scx_task_iter *iter) +void scx_task_iter_stop (struct scx_task_iter *iter) {#ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) {
@@ -923,7 +906,7 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)* whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */ -static struct task_struct *scx_task_iter_next_locked (struct scx_task_iter *iter) +struct task_struct *scx_task_iter_next_locked (struct scx_task_iter *iter) {struct task_struct *p;
@@ -1186,8 +1169,8 @@ static void schedule_deferred_locked(struct rq *rq)schedule_deferred (rq); } -static void schedule_dsq_reenq (struct scx_sched *sch,struct scx_dispatch_q *dsq, -u64 reenq_flags,struct rq *locked_rq) +void schedule_dsq_reenq (struct scx_sched *sch,struct scx_dispatch_q *dsq, +u64 reenq_flags,struct rq *locked_rq) {struct rq *rq;
@@ -2491,8 +2474,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,return dst_rq; } -static bool scx_consume_dispatch_q (struct scx_sched *sch,struct rq *rq, -struct scx_dispatch_q *dsq,u64 enq_flags) +bool scx_consume_dispatch_q (struct scx_sched *sch,struct rq *rq, +struct scx_dispatch_q *dsq,u64 enq_flags) {struct task_struct *p; retry:
@@ -2538,7 +2521,7 @@ static bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,return false; } -static bool scx_consume_global_dsq (struct scx_sched *sch,struct rq *rq) +bool scx_consume_global_dsq (struct scx_sched *sch,struct rq *rq) {int node =cpu_to_node (cpu_of (rq));
@@ -3548,7 +3531,7 @@ static struct cgroup *tg_cgrp(struct task_group *tg)#endif /* CONFIG_EXT_GROUP_SCHED */ -static int __scx_init_task (struct scx_sched *sch,struct task_struct *p,bool fork) +int __scx_init_task (struct scx_sched *sch,struct task_struct *p,bool fork) {int ret;
@@ -3631,7 +3614,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)SCX_CALL_OP_TASK (sch, set_weight, rq, p, p->scx.weight); } -static void scx_enable_task (struct scx_sched *sch,struct task_struct *p) +void scx_enable_task (struct scx_sched *sch,struct task_struct *p) {__scx_enable_task (sch, p);scx_set_task_state (p,SCX_TASK_ENABLED );
@@ -3665,8 +3648,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)WARN_ON_ONCE (p->scx.flags &SCX_TASK_IN_CUSTODY ); } -static void __scx_disable_and_exit_task (struct scx_sched *sch, -struct task_struct *p) +void __scx_disable_and_exit_task (struct scx_sched *sch,struct task_struct *p) {struct scx_exit_task_args args = { .cancelled = false,
@@ -3700,7 +3682,7 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch,* ran. The task state has not been transitioned, so this mirrors the * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). */ -static void scx_sub_init_cancel_task (struct scx_sched *sch,struct task_struct *p) +void scx_sub_init_cancel_task (struct scx_sched *sch,struct task_struct *p) {struct scx_exit_task_args args = { .cancelled = true };
@@ -3711,8 +3693,7 @@ static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *SCX_CALL_OP_TASK (sch, exit_task,task_rq (p), p, &args); } -static void scx_disable_and_exit_task (struct scx_sched *sch, -struct task_struct *p) +void scx_disable_and_exit_task (struct scx_sched *sch,struct task_struct *p) {__scx_disable_and_exit_task (sch, p);
@@ -4525,7 +4506,7 @@ static struct cgroup *root_cgroup(void)return &cgrp_dfl_root.cgrp; } -static void scx_cgroup_lock (void ) +void scx_cgroup_lock (void ) {#ifdef CONFIG_EXT_GROUP_SCHED percpu_down_write (&scx_cgroup_ops_rwsem);
@@ -4533,7 +4514,7 @@ static void scx_cgroup_lock(void)cgroup_lock (); } -static void scx_cgroup_unlock (void ) +void scx_cgroup_unlock (void ) {cgroup_unlock ();#ifdef CONFIG_EXT_GROUP_SCHED
@@ -4851,7 +4832,7 @@ static void free_exit_info(struct scx_exit_info *ei);static const char *scx_exit_reason (enum scx_exit_kind kind);static bool scx_claim_exit (struct scx_sched *sch,enum scx_exit_kind kind); -static s32 scx_set_cmask_scratch_alloc (struct scx_sched *sch) +s32 scx_set_cmask_scratch_alloc (struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits,SCX_CMASK_NR_WORDS (num_possible_cpus ()));
@@ -5509,7 +5490,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) }/* may be called without holding scx_bypass_lock */ -static void scx_disable_bypass_dsp (struct scx_sched *sch) +void scx_disable_bypass_dsp (struct scx_sched *sch) {s32 ret;
@@ -5557,7 +5538,7 @@ static void scx_disable_bypass_dsp(struct scx_sched *sch)* * - scx_prio_less() reverts to the default core_sched_at order. */ -static void scx_bypass (struct scx_sched *sch,bool bypass) +void scx_bypass (struct scx_sched *sch,bool bypass) {struct scx_sched *pos;unsigned long flags;
@@ -5746,7 +5727,7 @@ static void refresh_watchdog(void)cancel_delayed_work_sync (&scx_watchdog_work); } -static s32 scx_link_sched (struct scx_sched *sch) +s32 scx_link_sched (struct scx_sched *sch) {const char *err_msg ="" ;s32 ret =0 ;
@@ -5795,7 +5776,7 @@ static s32 scx_link_sched(struct scx_sched *sch)return 0 ; } -static void scx_unlink_sched (struct scx_sched *sch) +void scx_unlink_sched (struct scx_sched *sch) { scoped_guard(raw_spinlock_irq, &scx_sched_lock) {#ifdef CONFIG_EXT_SUB_SCHED
@@ -5816,13 +5797,13 @@ static void scx_unlink_sched(struct scx_sched *sch)* @sch. Once @sch becomes empty during disable, there's no point in dumping it. * This prevents calling dump ops on a dead sch. */ -static void scx_disable_dump (struct scx_sched *sch) +void scx_disable_dump (struct scx_sched *sch) {guard (raw_spinlock_irqsave)(&scx_dump_lock); sch->dump_disabled = true; } -static void scx_log_sched_disable (struct scx_sched *sch) +void scx_log_sched_disable (struct scx_sched *sch) {struct scx_exit_info *ei = sch->exit_info;const char *type =scx_parent (sch) ?"sub-scheduler" :"scheduler" ;
@@ -6285,7 +6266,7 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)* as a noop. Syncing the irq_work first is required to guarantee the * kthread work has been queued before waiting for it. */ -static void scx_flush_disable_work (struct scx_sched *sch) +void scx_flush_disable_work (struct scx_sched *sch) {int kind;
@@ -6739,31 +6720,13 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)return pnode; } -/* -* scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid -* starvation. During the READY -> ENABLED task switching loop, the calling -* thread's sched_class gets switched from fair to ext. As fair has higher -* priority than ext, the calling thread can be indefinitely starved under -* fair-class saturation, leading to a system hang. -*/ -struct scx_enable_cmd { -struct kthread_work work; -union { -struct sched_ext_ops *ops; -struct sched_ext_ops_cid *ops_cid; - }; -bool is_cid_type; -struct bpf_map *arena_map;/* arena ref to transfer to sch */ -int ret; -}; -/* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */ -static struct scx_sched *scx_alloc_and_add_sched (struct scx_enable_cmd *cmd, -struct cgroup *cgrp, -struct scx_sched *parent) +struct scx_sched *scx_alloc_and_add_sched (struct scx_enable_cmd *cmd, +struct cgroup *cgrp, +struct scx_sched *parent) {struct sched_ext_ops *ops = cmd->ops;struct scx_sched *sch;
@@ -7007,7 +6970,7 @@ static int check_hotplug_seq(struct scx_sched *sch,return 0 ; } -static int scx_validate_ops (struct scx_sched *sch,const struct sched_ext_ops *ops) +int scx_validate_ops (struct scx_sched *sch,const struct sched_ext_ops *ops) {/* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
kernel/sched/ext/internal.h +77 -0
89 lines not on this page · raw message
Signature · 1 line
2.54.0
Tejun Heo
reviewerPATCH 3/4 sched_ext: Inline small ext.c helpers shared across the sub.c split
In reply to Tejun HeoThe following trivial helpers in ext.c are called from both ext.c and the sub-scheduler code. Define them as static inline in internal.h.
- scx_bypass_dsq()
- scx_bypass_dsp_enabled()
- scx_ops_sanitize_err()
- scx_schedule_reenq_local()
No functional change.
--- kernel/sched/ext/ext.c | 57 ------------------------------------- kernel/sched/ext/internal.h | 57 +++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 57 deletions(-)
kernel/sched/ext/ext.c +0 -57
@@ -369,11 +369,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p)return __setscheduler_class (p->policy, p->prio); } -static struct scx_dispatch_q *scx_bypass_dsq (struct scx_sched *sch,s32 cpu) -{ -return &per_cpu_ptr (sch->pcpu, cpu)->bypass_dsq; -} -static struct scx_dispatch_q *bypass_enq_target_dsq (struct scx_sched *sch,s32 cpu) {#ifdef CONFIG_EXT_SUB_SCHED
@@ -395,26 +390,6 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 creturn scx_bypass_dsq (sch, cpu); } -/** -* scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled -* @sch: scheduler to check -* -* When a descendant scheduler enters bypass mode, bypassed tasks are scheduled -* by the nearest non-bypassing ancestor, or the root scheduler if all ancestors -* are bypassing. In the former case, the ancestor is not itself bypassing but -* its bypass DSQs will be populated with bypassed tasks from descendants. Thus, -* the ancestor's bypass dispatch path must be active even though its own -* bypass_depth remains zero. -* -* This function checks bypass_dsp_enable_depth which is managed separately from -* bypass_depth to enable this decoupling. See enable_bypass_dsp() and -* scx_disable_bypass_dsp(). -*/ -static bool scx_bypass_dsp_enabled (struct scx_sched *sch) -{ -return unlikely (atomic_read (&sch->bypass_dsp_enable_depth)); -} -/** * rq_is_open - Is the rq available for immediate execution of an SCX task? * @rq: rq to test
@@ -1061,28 +1036,6 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) } } -/** -* scx_ops_sanitize_err - Sanitize a -errno value -* @sch: scx_sched to error out on error -* @ops_name: operation to blame on failure -* @err: -errno value to sanitize -* -* Verify @err is a valid -errno. If not, trigger scx_error() and return -* -%EPROTO. This is necessary because returning a rogue -errno up the chain can -* cause misbehaviors. For an example, a large negative return from -* ops.init_task() triggers an oops when passed up the call chain because the -* value fails IS_ERR() test after being encoded with ERR_PTR() and then is -* handled as a pointer. -*/ -static int scx_ops_sanitize_err (struct scx_sched *sch,const char *ops_name,s32 err) -{ -if (err <0 && err >= -MAX_ERRNO ) -return err; - -scx_error (sch,"ops.%s() returned an invalid errno %d" , ops_name, err); -return -EPROTO ; -} -static void deferred_bal_cb_workfn (struct rq *rq) {run_deferred (rq);
@@ -1234,16 +1187,6 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,schedule_deferred (rq); } -static void scx_schedule_reenq_local (struct rq *rq,u64 reenq_flags) -{ -struct scx_sched *root =rcu_dereference_sched (scx_root); - -if (WARN_ON_ONCE (!root)) -return ; - -schedule_dsq_reenq (root, &rq->scx.local_dsq, reenq_flags, rq); -} -/** * touch_core_sched - Update timestamp used for core-sched task ordering * @rq: rq to read clock from, must be locked
kernel/sched/ext/internal.h +57 -0
63 lines not on this page · raw message
Signature · 1 line
2.54.0
Tejun Heo
reviewerPATCH 4/4 sched_ext: Split sub-scheduler implementation into sub.c
In reply to Tejun HeoThe sub-scheduler implementation has grown and will continue to expand. Move the sub-scheduler functions from ext.c into a new kernel/sched/ext/sub.c. sub.h holds the prototypes and the !CONFIG_EXT_SUB_SCHED no-op stubs.
No functional change.
--- kernel/sched/build_policy.c | 2 + kernel/sched/ext/ext.c | 676 +----------------------------------- kernel/sched/ext/sub.c | 668 +++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 51 +++ 4 files changed, 722 insertions(+), 675 deletions(-) create mode 100644 kernel/sched/ext/sub.c create mode 100644 kernel/sched/ext/sub.h
kernel/sched/build_policy.c +2 -0
@@ -66,10 +66,12 @@# include "ext/cid.h" # include "ext/arena.h" # include "ext/idle.h" +# include "ext/sub.h" # include "ext/ext.c" # include "ext/cid.c" # include "ext/arena.c" # include "ext/idle.c" +# include "ext/sub.c" #endif #include "syscalls.c"
kernel/sched/ext/ext.c +1 -675
724 lines not on this page · raw message
kernel/sched/ext/sub.c new file +668 -0
668 lines not on this page · raw message
kernel/sched/ext/sub.h new file +51 -0
51 lines not on this page · raw message
Signature · 1 line
2.54.0
Andrea Righi
reviewerHi Tejun,
Quoted 8 lines · Tejun Heo
The sub-scheduler implementation has grown and will continue to expand. Move the sub-scheduler functions from
ext.cinto a newkernel/sched/ext/sub.c.sub.hholds the prototypes and the !CONFIG_EXT_SUB_SCHED no-op stubs.No functional change.
Signed-off-by Tejun Heo <[email protected]>---
...
Quoted 25 lines
kernel/sched/ext/sub.c new file +25
-0+/** +* scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler +* @cgroup_id: cgroup ID of the child scheduler to dispatch +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* Allows a parent scheduler to trigger dispatching on one of its direct +* child schedulers. The child scheduler runs its dispatch operation to +* move tasks from dispatch queues to the local runqueue. +* +* Returns: true on success, false if cgroup_id is invalid, not a direct +* child, or caller lacks dispatch permission. +*/ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ +struct rq *this_rq =this_rq (); +struct scx_sched *parent, *child; + +guard (rcu)(); + parent =scx_prog_sched (aux); +if (unlikely (!parent)) +return false; + + child =scx_find_sub_sched (cgroup_id); + +if (unlikely (!child))
kernel/sched/ext/sub.c line 663
Quoted 10 lines
kernel/sched/ext/sub.c new file +10
-0+return false; + +if (unlikely (scx_parent (child) != parent)) { +scx_error (parent,"trying to dispatch a distant sub-sched on cgroup %llu" , + cgroup_id); +return false; + } + +return scx_dispatch_sched (child, this_rq, this_rq->scx.sub_dispatch_prev, + true);
clangd isn't happy here, because we're missing the declaration of scx_dispatch_sched(). It needs to be always inline. We can move the whole function to internal.h, but that would require extra restructuring. Otherwise, can relax the always inline condition?
From a local build the stack allocation in bytes looks like this:
scx_bpf_sub_dispatch() scx_dispatch_sched() Per level 4 levels
-----------------------------------------------------------------
__always_inline 104 - 104 416
non-inline 72 72 144 576
-----------------------------------------------------------------delta +40 +160
I think the saving is quite still small compared with nested BPF frames. WDYT?
Thanks,
-Andrea
Tejun Heo
reviewerHello, Andrea.
Quoted 2 lines · Andrea Righi
clangd isn't happy here, because we're missing the declaration of
scx_dispatch_sched(). [...] can relax the always inline condition?
My slicing mistake. scx_dispatch_sched() gets promoted into sub.h a few patches later, and I posted the split ahead of that. I'll fold the promotion into the split and send v2 soon.
I'd rather keep __always_inline, though. scx_bpf_sub_dispatch() re-enters it as sub-scheduler dispatch nests, so inlining avoids a frame per nesting level.
Thanks.
--
tejun
Hi Tejun,
Quoted 7 lines · Tejun Heo
Hello,
The sub-scheduler implementation has grown and will keep growing. Move it out of
ext.cinto a newkernel/sched/ext/sub.c. The first three patches are mechanical prep (prefix file-local helpers, expose shared internals, inline a few trivial helpers) so the move itself stays pure code motion. No functional change.
Apart than the comment about scx_dispatch_sched() in PATCH 4/4, everything else looks good to me.
Thanks,
-Andrea
Quoted 21 lines
Based on sched_ext/for-7.3 (
5df6a4506d06) with sched_ext/for-7.2-fixes (b7d9c359e5cf) assumed merged.Tejun Heo (4):
sched_ext: Prefix file-localext.chelpers exposed by thesub.csplit
sched_ext: Expose theext.cinternals used by thesub.csplit
sched_ext: Inline smallext.chelpers shared across thesub.csplit
sched_ext: Split sub-scheduler implementation intosub.ckernel/sched/build_policy.c | 2 + kernel/sched/ext/ext.c | 1002 +++++-------------------------------------- kernel/sched/ext/internal.h | 136 +++++- kernel/sched/ext/sub.c | 668 +++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 51 +++ 5 files changed, 973 insertions(+), 886 deletions(-)Thanks.
--
tejun