sched_ext: Test scx_has_subs() inline before calling sub-sched hooks
Subsystems (2)
PATCH 1/1 sched_ext: Test scx_has_subs() inline before calling sub-sched hooks
scx_reenq_reject(), scx_process_sync_ecaps() and scx_resolve_local_dsq() return early if there are no sub-schedulers, but clang emits the full prologue before the static branch. Without sub-schedulers, we still pay for a call and for saving and restoring up to six registers, on every wakeup, dispatch and local DSQ insert.
Let's move the scx_has_subs() test into inline wrappers, so that these calls go away completely. The wrappers keep lockdep_assert_rq_held().
In a pinned perf bench sched pipe test on x86-64 with clang 22, this reduced kernel instructions by 52-54 per context switch with scx_simple, scx_lavd and scx_layered.
No functional change intended.
--- kernel/sched/ext/sub.c | 25 +++++++++---------------- kernel/sched/ext/sub.h | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 20 deletions(-)
kernel/sched/ext/sub.c +9 -16
@@ -675,7 +675,7 @@ void scx_rescue_init(struct rq *rq) }/** -* scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert +* __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted
@@ -695,12 +695,9 @@ void scx_rescue_init(struct rq *rq)* to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq (struct scx_sched *sch,struct rq *rq, -struct task_struct *p,u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq (struct scx_sched *sch,struct rq *rq, +struct task_struct *p,u64 *enq_flags) { -if (!scx_has_subs ()) -return &rq->scx.local_dsq; -s32 cid =__scx_cpu_to_cid (cpu_of (rq));struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;u64 needed =scx_caps_for_enq (*enq_flags);
@@ -783,14 +780,12 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)* scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. * Rejection can't happen for root. */ -void scx_reenq_reject (struct rq *rq) +void __scx_reenq_reject (struct rq *rq) {LIST_HEAD (tasks);struct task_struct *p, *n; -lockdep_assert_rq_held (rq); - -if (!scx_has_subs () ||list_empty (&rq->scx.reject_dsq.list)) +if (list_empty (&rq->scx.reject_dsq.list))return ;/*
@@ -926,7 +921,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)struct scx_sched_pcpu *pcpu =per_cpu_ptr (sch->pcpu, cpu);/* -* Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check +* Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */
@@ -951,7 +946,7 @@ static void discard_queued_syncs(struct rq *rq) }/** -* scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] +* __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch *
@@ -963,16 +958,14 @@ static void discard_queued_syncs(struct rq *rq)* learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps (struct rq *rq,struct task_struct *prev) +void __scx_process_sync_ecaps (struct rq *rq,struct task_struct *prev) {s32 cpu =cpu_of (rq);s32 cid, shard;struct llist_node *batch, *pos, *tmp;u64 lost_all =0 ; -lockdep_assert_rq_held (rq); - -if (!scx_has_subs () ||likely (llist_empty (&rq->scx.ecaps_to_sync))) +if (likely (llist_empty (&rq->scx.ecaps_to_sync)))return ;/*
kernel/sched/ext/sub.h +31 -4
@@ -28,16 +28,16 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);void scx_free_pshards (struct scx_sched *sch);s32 scx_alloc_pshards (struct scx_sched *sch);void scx_init_root_caps (struct scx_sched *sch); -void scx_process_sync_ecaps (struct rq *rq,struct task_struct *prev); +void __scx_process_sync_ecaps (struct rq *rq,struct task_struct *prev);void scx_unbypass_replay_ecaps (struct rq *rq,struct scx_sched *sch);void scx_online_ecaps (struct rq *rq);void scx_offline_ecaps (struct rq *rq);void scx_discard_ecaps_to_sync (s32 cpu,struct scx_sched_pcpu *pcpu);void scx_discard_stale_ecaps_syncs (void ); -struct scx_dispatch_q *scx_resolve_local_dsq (struct scx_sched *sch,struct rq *rq, -struct task_struct *p,u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq (struct scx_sched *sch,struct rq *rq, +struct task_struct *p,u64 *enq_flags);bool scx_task_reenq_on_cap_revoke (struct rq *rq,struct task_struct *p); -void scx_reenq_reject (struct rq *rq); +void __scx_reenq_reject (struct rq *rq);void scx_rescue_charge (struct rq *rq,s64 delta_exec);void scx_rescue_end (struct rq *rq);bool scx_rescue_keep (struct rq *rq,struct task_struct *p);
@@ -72,6 +72,33 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps (struct rq *rq,struct task_struct *prev) +{ +lockdep_assert_rq_held (rq); + +if (scx_has_subs ()) +__scx_process_sync_ecaps (rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq (struct scx_sched *sch, +struct rq *rq, +struct task_struct *p, +u64 *enq_flags) +{ +if (!scx_has_subs ()) +return &rq->scx.local_dsq; +return __scx_resolve_local_dsq (sch, rq, p, enq_flags); +} + +static inline void scx_reenq_reject (struct rq *rq) +{ +lockdep_assert_rq_held (rq); + +if (scx_has_subs ()) +__scx_reenq_reject (rq); +} +#else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre (struct scx_sched *pos,struct scx_sched *root) {return pos ?NULL : root; }
Signature · 1 line
2.53.0-Meta
Applied to sched_ext/for-7.4.
Thanks.
--
tejun