sched_ext: Add cid-form kfunc wrappers alongside cpu-form
Subsystems (2)
Tejun Heo
reviewerPATCH 11/17 sched_ext: Add cid-form kfunc wrappers alongside cpu-form
cpumask is awkward from BPF and unusable from arena; cid/cmask work in both. Sub-sched enqueue will need cmask. Without full cid coverage a scheduler has to mix cid and cpu forms, which is a subtle-bug factory. Close the gap with a cid-native interface.
Pair every cpu-form kfunc that takes a cpu id with a cid-form equivalent (kick, task placement, cpuperf query/set, per-cpu current task, nr-cpu-ids). Add two cid-natives with no cpu-form sibling:scx_bpf_this_cid() (cid of the running cpu, scx equivalent of bpf_get_smp_processor_id) and scx_bpf_nr_online_cids().
scx_bpf_cpu_rq is deprecated; no cid-form counterpart. NUMA node info is reachable via scx_bpf_cid_topo() on the BPF side.
Each cid-form wrapper is a thin cid -> cpu translation that delegates to the cpu path, registered in the same context sets so usage constraints match.
--- kernel/sched/ext.c | 184 +++++++++++++++++++++++ tools/sched_ext/include/scx/common.bpf.h | 9 ++ 2 files changed, 193 insertions(+)
kernel/sched/ext.c +184 -0
@@ -8751,6 +8751,28 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_auxscx_kick_cpu (sch, cpu, flags); } +/** +* scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid +* @cid: cid to kick +* @flags: %SCX_KICK_* flags +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* cid-addressed equivalent of scx_bpf_kick_cpu(). +*/ +__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +{ +struct scx_sched *sch; +s32 cpu; + +guard (rcu)(); + sch =scx_prog_sched (aux); +if (unlikely (!sch)) +return ; + cpu =scx_cid_to_cpu (sch, cid); +if (cpu >=0 ) +scx_kick_cpu (sch, cpu, flags); +} +/** * scx_bpf_dsq_nr_queued - Return the number of queued tasks * @dsq_id: id of the DSQ
@@ -9173,6 +9195,29 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux)return SCX_CPUPERF_ONE ; } +/** +* scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid +* @cid: cid of the CPU to query +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* cid-addressed equivalent of scx_bpf_cpuperf_cap(). +*/ +__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) +{ +struct scx_sched *sch; +s32 cpu; + +guard (rcu)(); + + sch =scx_prog_sched (aux); +if (unlikely (!sch)) +return SCX_CPUPERF_ONE ; + cpu =scx_cid_to_cpu (sch, cid); +if (cpu <0 ) +return SCX_CPUPERF_ONE ; +return arch_scale_cpu_capacity (cpu); +} +/** * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU * @cpu: CPU of interest
@@ -9201,6 +9246,29 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux)return SCX_CPUPERF_ONE ; } +/** +* scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid +* @cid: cid of the CPU to query +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* cid-addressed equivalent of scx_bpf_cpuperf_cur(). +*/ +__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) +{ +struct scx_sched *sch; +s32 cpu; + +guard (rcu)(); + + sch =scx_prog_sched (aux); +if (unlikely (!sch)) +return SCX_CPUPERF_ONE ; + cpu =scx_cid_to_cpu (sch, cid); +if (cpu <0 ) +return SCX_CPUPERF_ONE ; +return arch_scale_freq_capacity (cpu); +} +/** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest
@@ -9261,6 +9329,31 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au } } +/** +* scx_bpf_cidperf_set - Set the performance target of the CPU at @cid +* @cid: cid of the CPU to target +* @perf: target performance level [0, %SCX_CPUPERF_ONE] +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* cid-addressed equivalent of scx_bpf_cpuperf_set(). +*/ +__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, +const struct bpf_prog_aux *aux) +{ +struct scx_sched *sch; +s32 cpu; + +guard (rcu)(); + + sch =scx_prog_sched (aux); +if (unlikely (!sch)) +return ; + cpu =scx_cid_to_cpu (sch, cid); +if (cpu <0 ) +return ; +scx_bpf_cpuperf_set (cpu, perf, aux); +} +/** * scx_bpf_nr_node_ids - Return the number of possible node IDs *
@@ -9281,6 +9374,47 @@ __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)return nr_cpu_ids; } +/** +* scx_bpf_nr_cids - Return the size of the cid space +* +* Equals num_possible_cpus(). All valid cids are in [0, return value). +*/ +__bpf_kfunc u32 scx_bpf_nr_cids(void) +{ +return num_possible_cpus (); +} + +/** +* scx_bpf_nr_online_cids - Return current count of online CPUs in cid space +* +* Return num_online_cpus(). The standard model restarts the scheduler on +* hotplug, which lets schedulers treat [0, nr_online_cids) as the online +* range. Schedulers that prefer to handle hotplug without a restart should +* install a custom mapping via scx_bpf_cid_override() and track onlining +* through the ops.cid_online / ops.cid_offline callbacks. +*/ +__bpf_kfunc u32 scx_bpf_nr_online_cids(void) +{ +return num_online_cpus (); +} + +/** +* scx_bpf_this_cid - Return the cid of the CPU this program is running on +* +* cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. +* The current cpu is trivially valid, so this is just a table lookup. Return +* -EINVAL if called from a non-SCX program before any scheduler has ever +* been enabled (the cid table is still unallocated at that point). +*/ +__bpf_kfunc s32 scx_bpf_this_cid(void) +{ +s16 *tbl =READ_ONCE (scx_cpu_to_cid_tbl); + +if (!tbl) +return -EINVAL ; +return tbl[raw_smp_processor_id ()]; +} +/** * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask */
@@ -9329,6 +9463,23 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)return task_cpu (p); } +/** +* scx_bpf_task_cid - cid a task is currently associated with +* @p: task of interest +* +* cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a +* valid cpu, so this is just a table lookup. Return -EINVAL if called from +* a non-SCX program before any scheduler has ever been enabled. +*/ +__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) +{ +s16 *tbl =READ_ONCE (scx_cpu_to_cid_tbl); + +if (!tbl) +return -EINVAL ; +return tbl[task_cpu (p)]; +} +/** * scx_bpf_cpu_rq - Fetch the rq of a CPU * @cpu: CPU of the rq
@@ -9407,6 +9558,30 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_return rcu_dereference (cpu_rq (cpu)->curr); } +/** +* scx_bpf_cid_curr - Return the curr task on the CPU at @cid +* @cid: cid of interest +* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs +* +* cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU +* read lock (KF_RCU). +*/ +__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) +{ +struct scx_sched *sch; +s32 cpu; + +guard (rcu)(); + + sch =scx_prog_sched (aux); +if (unlikely (!sch)) +return NULL ; + cpu =scx_cid_to_cpu (sch, cid); +if (cpu <0 ) +return NULL ; +return rcu_dereference (cpu_rq (cpu)->curr); +} +/** * scx_bpf_tid_to_task - Look up a task by its scx tid * @tid: task ID previously read from p->scx.tid
@@ -9594,6 +9769,7 @@ BTF_KFUNCS_START(scx_kfunc_ids_any)BTF_ID_FLAGS (func, scx_bpf_task_set_slice,KF_IMPLICIT_ARGS |KF_RCU );BTF_ID_FLAGS (func, scx_bpf_task_set_dsq_vtime,KF_IMPLICIT_ARGS |KF_RCU ); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)BTF_ID_FLAGS (func, scx_bpf_dsq_peek,KF_IMPLICIT_ARGS |KF_RCU_PROTECTED |KF_RET_NULL )
@@ -9608,16 +9784,24 @@ BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cids) +BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_this_cid) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_now)BTF_ID_FLAGS (func, scx_bpf_events)
tools/sched_ext/include/scx/common.bpf.h +9 -0
@@ -105,6 +105,15 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __ s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; +void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; +s32 scx_bpf_this_cid(void) __ksym __weak; +struct task_struct *scx_bpf_cid_curr (s32 cid) __ksym__weak ; +u32 scx_bpf_nr_cids(void) __ksym __weak; +u32 scx_bpf_nr_online_cids(void) __ksym __weak; +u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; +u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; +void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;/* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
Signature · 1 line
2.53.0