sched_ext: cid/cmask interface prep
Subsystems (2)
Tejun Heo
reviewerPATCH v2 1/1 sched_ext: cid/cmask interface prep
Hello,
v2:
- New patch 2 reorders the BPF-side cmask helpers (cid.bpf.h) and their callers to (cid, mask) so the kernel and BPF interfaces match.
- Bound scx_cmask_for_each_cid() to the active range so a stale word past it can't yield a phantom cid (sashiko AI).
- scx_bpf_kick_cid() returns scx_cid_to_cpu()'s errno instead of -EINVAL (Andrea).
v1: https://lore.kernel.org/r/[email protected]
Preparatory changes to the cid/cmask interface that landed in for-7.2 this cycle - a single-cid helper-argument reorder on the kernel and BPF sides, two new helpers, and a kfunc return-type change. Posting ahead of the merge window, in particular the scx_bpf_kick_cid() return-type change in patch 4 - it changes a kfunc's signature and is cleaner to settle before 7.2 ships.
- Patch 1 reorders the single-cid cmask helpers to (cid, mask), matching the kernel's index-first bit and cpumask predicates.
- Patch 2 reorders the BPF-side counterparts in cid.bpf.h the same way.
- Patch 3 adds scx_cmask_test() and scx_cmask_for_each_cid().
- Patch 4 switches scx_bpf_kick_cid() from void to s32 so future cap enforcement can surface failures. The cid interface has no external users yet, so the ABI change is safe now.
These are extracted from the in-development sub-scheduler series. Nothing here depends on the rest of that work.
Base
----
sched_ext/for-7.2 (dad836524259)
kernel/sched/ext.c | 13 ++++--- kernel/sched/ext_cid.c | 2 +- kernel/sched/ext_cid.h | 53 +++++++++++++++++++++++++--- tools/sched_ext/include/scx/cid.bpf.h | 60 ++++++++++++++++---------------- tools/sched_ext/include/scx/common.bpf.h | 2 +- tools/sched_ext/scx_qmap.bpf.c | 12 +++---- 6 files changed, 94 insertions(+), 48 deletions(-)
Thanks.
--
tejun
Tejun Heo
reviewerPATCH 1/4 sched_ext: Order single-cid cmask helpers as (cid, mask)
__scx_cmask_set(), __scx_cmask_contains() and __scx_cmask_word() take the cmask first and the cid second. The kernel's bit and cpumask predicates put the index first: test_bit(nr, addr), cpumask_test_cpu(cpu, mask). Reorder the cmask helpers to (cid, mask) for consistency, ahead of new single-cid ops added next. Mask-level ops (and/or/andnot/copy/subset/intersects) keep (dst, src).
--- kernel/sched/ext_cid.c | 2 +- kernel/sched/ext_cid.h | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-)
kernel/sched/ext_cid.c +1 -1
@@ -267,7 +267,7 @@ void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst)s32 cid =__scx_cpu_to_cid (cpu);if (cid >=0 ) -__scx_cmask_set (dst, cid); +__scx_cmask_set (cid, dst); } }
kernel/sched/ext_cid.h +5 -5
@@ -147,13 +147,13 @@ static inline bool scx_is_cid_type(void)return static_branch_unlikely (&__scx_is_cid_type); } -static inline bool __scx_cmask_contains (const struct scx_cmask *m,u32 cid) +static inline bool __scx_cmask_contains (u32 cid,const struct scx_cmask *m) {return likely (cid >= m->base && cid < m->base + m->nr_cids); }/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ -static inline u64 *__scx_cmask_word (const struct scx_cmask *m,u32 cid) +static inline u64 *__scx_cmask_word (u32 cid,const struct scx_cmask *m) {return (u64 *)&m->bits[cid /64 - m->base /64 ]; }
@@ -218,11 +218,11 @@ static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) m->nr_cids = nr_cids; } -static inline void __scx_cmask_set (struct scx_cmask *m,u32 cid) +static inline void __scx_cmask_set (u32 cid,struct scx_cmask *m) { -if (!__scx_cmask_contains (m, cid)) +if (!__scx_cmask_contains (cid, m))return ; - *__scx_cmask_word (m, cid) |=BIT_U64 (cid &63 ); + *__scx_cmask_word (cid, m) |=BIT_U64 (cid &63 ); }#endif /* _KERNEL_SCHED_EXT_CID_H */
Signature · 1 line
2.54.0
Tejun Heo
reviewerPATCH 2/4 tools/sched_ext: Order single-cid cmask helpers as (cid, mask)
In reply to Tejun HeoThe BPF arena single-cid cmask helpers take the cmask first and the cid second. Reorder them to (cid, mask) to match the kernel-side helpers and the test_bit(nr, addr), cpumask_test_cpu(cpu, mask) convention. Range and iteration helpers keep (mask, start).
--- tools/sched_ext/include/scx/cid.bpf.h | 60 +++++++++++++-------------- tools/sched_ext/scx_qmap.bpf.c | 12 +++--- 2 files changed, 36 insertions(+), 36 deletions(-)
tools/sched_ext/include/scx/cid.bpf.h +30 -30
@@ -40,12 +40,12 @@*/ #define CMASK_NR_WORDS (nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) -static __always_inline bool __cmask_contains(const struct scx_cmask __arena *m, u32 cid) +static __always_inline bool __cmask_contains(u32 cid, const struct scx_cmask __arena *m) {return cid >= m->base && cid < m->base + m->nr_cids; } -static __always_inline u64 __arena *__cmask_word(const struct scx_cmask __arena *m, u32 cid) +static __always_inline u64 __arena *__cmask_word(u32 cid, const struct scx_cmask __arena *m) { return (u64 __arena *)&m->bits[cid / 64 - m->base / 64]; }
@@ -122,11 +122,11 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; } -static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 cid) +static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) { -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return false; -return *__cmask_word (m, cid) &BIT_U64 (cid &63 ); +return *__cmask_word (cid, m) &BIT_U64 (cid &63 ); }/*
@@ -140,15 +140,15 @@ static __always_inline bool cmask_test(const struct scx_cmask __arena *m, u32 ci*/ #define CMASK_CAS_TRIES (1U << 23) -static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid) +static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w;u64 bit, old, new;u32 i; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return ; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); bit =BIT_U64 (cid &63 ); bpf_for(i, 0, CMASK_CAS_TRIES) { old = *w;
@@ -161,15 +161,15 @@ static __always_inline void cmask_set(struct scx_cmask __arena *m, u32 cid)scx_bpf_error ("cmask_set CAS exhausted at cid %u" , cid); } -static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid) +static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w;u64 bit, old, new;u32 i; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return ; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); bit =BIT_U64 (cid &63 ); bpf_for(i, 0, CMASK_CAS_TRIES) { old = *w;
@@ -182,15 +182,15 @@ static __always_inline void cmask_clear(struct scx_cmask __arena *m, u32 cid)scx_bpf_error ("cmask_clear CAS exhausted at cid %u" , cid); } -static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) +static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w;u64 bit, old, new;u32 i; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return false; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); bit =BIT_U64 (cid &63 ); bpf_for(i, 0, CMASK_CAS_TRIES) { old = *w;
@@ -204,15 +204,15 @@ static __always_inline bool cmask_test_and_set(struct scx_cmask __arena *m, u32return false; } -static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) +static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w;u64 bit, old, new;u32 i; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return false; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); bit =BIT_U64 (cid &63 ); bpf_for(i, 0, CMASK_CAS_TRIES) { old = *w;
@@ -226,43 +226,43 @@ static __always_inline bool cmask_test_and_clear(struct scx_cmask __arena *m, u3return false; } -static __always_inline void __cmask_set(struct scx_cmask __arena *m, u32 cid) +static __always_inline void __cmask_set(u32 cid, struct scx_cmask __arena *m) { -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return ; - *__cmask_word (m, cid) |=BIT_U64 (cid &63 ); + *__cmask_word (cid, m) |=BIT_U64 (cid &63 ); } -static __always_inline void __cmask_clear(struct scx_cmask __arena *m, u32 cid) +static __always_inline void __cmask_clear(u32 cid, struct scx_cmask __arena *m) { -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return ; - *__cmask_word (m, cid) &= ~BIT_U64 (cid &63 ); + *__cmask_word (cid, m) &= ~BIT_U64 (cid &63 ); } -static __always_inline bool __cmask_test_and_set(struct scx_cmask __arena *m, u32 cid) +static __always_inline bool __cmask_test_and_set(u32 cid, struct scx_cmask __arena *m) {u64 bit =BIT_U64 (cid &63 ); u64 __arena *w;u64 prev; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return false; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); prev = *w & bit; *w |= bit;return prev; } -static __always_inline bool __cmask_test_and_clear(struct scx_cmask __arena *m, u32 cid) +static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __arena *m) {u64 bit =BIT_U64 (cid &63 ); u64 __arena *w;u64 prev; -if (!__cmask_contains (m, cid)) +if (!__cmask_contains (cid, m))return false; - w =__cmask_word (m, cid); + w =__cmask_word (cid, m); prev = *w & bit; *w &= ~bit;return prev;
@@ -671,7 +671,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m,continue ; cid =scx_bpf_cpu_to_cid (cpu);if (cid >=0 ) -__cmask_set (m, cid); +__cmask_set (cid, m); } }
tools/sched_ext/scx_qmap.bpf.c +6 -6
@@ -214,7 +214,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,if (!always_enq_immed && p->nr_cpus_allowed ==1 )return prev_cid; -if (cmask_test_and_clear (qa_idle_cids, prev_cid)) +if (cmask_test_and_clear (prev_cid, qa_idle_cids))return prev_cid; cid = prev_cid;
@@ -224,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,barrier_var (cid);if (cid >= nr_cids)return -1 ; -if (cmask_test_and_clear (qa_idle_cids, cid)) +if (cmask_test_and_clear (cid, qa_idle_cids))return cid; }return -1 ;
@@ -534,7 +534,7 @@ static bool dispatch_highpri(bool from_timer)if (!(taskc =lookup_task_ctx (p)))return false; -if (cmask_test (&taskc->cpus_allowed, this_cid)) +if (cmask_test (this_cid, &taskc->cpus_allowed)) cid = this_cid;else cid = cmask_next_set_wrap(&taskc->cpus_allowed,
@@ -656,7 +656,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)* document this class of issue -- other schedulers * seeing similar warnings can use this as a reference. */ -if (!cmask_test (&taskc->cpus_allowed, cid)) +if (!cmask_test (cid, &taskc->cpus_allowed))scx_bpf_kick_cid (scx_bpf_task_cid (p),0 ); batch--;
@@ -913,9 +913,9 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) {QMAP_TOUCH_ARENA ();if (idle) -cmask_set (qa_idle_cids, cid); +cmask_set (cid, qa_idle_cids);else -cmask_clear (qa_idle_cids, cid); +cmask_clear (cid, qa_idle_cids); } void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p,
Signature · 1 line
2.54.0
Tejun Heo
reviewerPATCH 3/4 sched_ext: Add scx_cmask_test() and scx_cmask_for_each_cid()
In reply to Tejun HeoAdd single-bit test and iterator over set cids in an scx_cmask.
v2: Bound scx_cmask_for_each_cid() to the active span. (sashiko AI)
--- kernel/sched/ext_cid.h | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+)
kernel/sched/ext_cid.h +43 -0
@@ -225,4 +225,47 @@ static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); } +/** +* scx_cmask_test - test whether @cid is set in @m +* @cid: cid to test +* @m: cmask to test +* +* Return %false if @cid is outside @m's active range. Otherwise return the +* bit's value. Read via READ_ONCE so callers can race set/clear writers. +*/ +static inline bool scx_cmask_test (u32 cid,const struct scx_cmask *m) +{ +if (!__scx_cmask_contains (cid, m)) +return false; +return READ_ONCE (*__scx_cmask_word (cid, m)) &BIT_U64 (cid &63 ); +} + +/* +* Words of bits[] the active range spans, 0 if empty. Tighter than the storage +* SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. +*/ +static inline u32 scx_cmask_nr_used_words (const struct scx_cmask *m) +{ +if (!m->nr_cids) +return 0 ; +return ((m->base &63 ) + m->nr_cids -1 ) /64 +1 ; +} + +/** +* scx_cmask_for_each_cid - iterate set cids in @m +* @cid: s32 loop var that receives each set cid in turn +* @m: cmask to iterate +* +* Visits set bits within @m's active range in ascending order. Scans only the +* words the active range spans, where head and tail padding is kept zero, so +* no per-cid range check is needed. +*/ +#define scx_cmask_for_each_cid (cid, m) \ + for (u64 __bs = (m)->base & ~63u, __wi = 0, \ + __nw = scx_cmask_nr_used_words(m); \ + __wi < __nw; __wi++) \ + for (u64 __w = READ_ONCE((m)->bits[__wi]); \ + __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ + __w &= __w - 1) +#endif /* _KERNEL_SCHED_EXT_CID_H */
Signature · 1 line
2.54.0
[email protected]
AI reviewRe: [PATCH 3/4] sched_ext: Add scx_cmask_test() and scx_cmask_for_each_cid()
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [High] The `scx_cmask_for_each_cid` macro expands into two nested `for` loops, breaking standard C…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] The scx_cmask_for_each_cid macro expands into two nested for loops, breaking standard C iteration semantics for break statements. If a caller uses a break statement to terminate the iteration early, it will only exit the inner loop (terminating the scan for the current 64-bit word). The outer loop will then increment, fetch the next word, and continue iterating. This creates a trap where developers intending to completely abort the loop will inadvertently only skip the rest of the current word and silently resume on the next one. --
commit e9a4307d205f9de4f9da9f425dd9329d36d47094
Author: Tejun Heo <[email protected]>
sched_ext: Add scx_cmask_test() and scx_cmask_for_each_cid()
Adds a single-bit test and an iterator over set cids in an scx_cmask. The iterator is bounded to the active span.
Quoted 4 lines
kernel/sched/ext_cid.h +0
-0
[ ... ]
Quoted 6 lines
kernel/sched/ext_cid.h +6
-0+/** +* scx_cmask_for_each_cid - iterate set cids in @m +* @cid: s32 loop var that receives each set cid in turn +* @m: cmask to iterate +* +* Visits set bits within @m's active range in ascending order. Scans only the
kernel/sched/ext_cid.h static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) · line 269
Quoted 10 lines
kernel/sched/ext_cid.h +10
-0+* words the active range spans, where head and tail padding is kept zero, so +* no per-cid range check is needed. +*/ +#define scx_cmask_for_each_cid (cid, m) \ + for (u64 __bs = (m)->base & ~63u, __wi = 0, \ + __nw = scx_cmask_nr_used_words(m); \ + __wi < __nw; __wi++) \ + for (u64 __w = READ_ONCE((m)->bits[__wi]); \ + __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ + __w &= __w - 1)
[Severity: High]
Since this macro expands into two nested for loops, what happens if a caller uses a break statement to terminate the iteration early?
It appears a break will only exit the inner loop, terminating the scan for the current 64-bit word. The outer loop will then increment, fetch the next word, and continue iterating.
Could this create a trap where a standard break statement inadvertently skips the rest of the current word but silently resumes yielding items on the next word, violating standard C iteration semantics?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=3
Tejun Heo
reviewerPATCH 4/4 sched_ext: Make scx_bpf_kick_cid() return s32
In reply to Tejun HeoSwitch scx_bpf_kick_cid() from void to s32 so future cap enforcement can surface failures. cid interface is introduced in this cycle and has no external users, so the ABI change is safe. Subsequent patches will add -EPERM returns when the calling sub-sched lacks the required cap on the target cid.
v2: Return scx_cid_to_cpu()'s errno instead of -EINVAL. (Andrea)
--- kernel/sched/ext.c | 13 ++++++++----- tools/sched_ext/include/scx/common.bpf.h | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-)
kernel/sched/ext.c +8 -5
@@ -9405,9 +9405,10 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux* @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * -* cid-addressed equivalent of scx_bpf_kick_cpu(). +* cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, +* -errno otherwise. */ -__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) {struct scx_sched *sch;s32 cpu;
@@ -9415,10 +9416,12 @@ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_auxguard (rcu)(); sch =scx_prog_sched (aux);if (unlikely (!sch)) -return ; +return -ENODEV ; cpu =scx_cid_to_cpu (sch, cid); -if (cpu >=0 ) -scx_kick_cpu (sch, cpu, flags); +if (cpu <0 ) +return cpu; +scx_kick_cpu (sch, cpu, flags); +return 0 ; }/**
tools/sched_ext/include/scx/common.bpf.h +1 -1
@@ -105,7 +105,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __ s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; -void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak;struct task_struct *scx_bpf_cid_curr (s32 cid) __ksym__weak ;
Signature · 1 line
2.54.0
Hi Tejun,
Quoted 26 lines · Tejun Heo
Hello,
v2:
- New patch 2 reorders the BPF-side cmask helpers (cid.bpf.h) and their callers to (cid, mask) so the kernel and BPF interfaces match.
- Boundscx_cmask_for_each_cid()to the active range so a stale word past it can't yield a phantom cid (sashiko AI).
-scx_bpf_kick_cid()returns scx_cid_to_cpu()'s errno instead of-EINVAL(Andrea).v1: https://lore.kernel.org/r/[email protected]
Preparatory changes to the cid/cmask interface that landed in for-7.2 this cycle - a single-cid helper-argument reorder on the kernel and BPF sides, two new helpers, and a kfunc return-type change. Posting ahead of the merge window, in particular the
scx_bpf_kick_cid()return-type change in patch 4 - it changes a kfunc's signature and is cleaner to settle before 7.2 ships.- Patch 1 reorders the single-cid cmask helpers to (cid, mask), matching the kernel's index-first bit and cpumask predicates.
- Patch 2 reorders the BPF-side counterparts incid.bpf.hthe same way.
- Patch 3 addsscx_cmask_test()andscx_cmask_for_each_cid().
- Patch 4 switchesscx_bpf_kick_cid()from void to s32 so future cap enforcement can surface failures. The cid interface has no external users yet, so the ABI change is safe now.
Quoted 2 lines
These are extracted from the in-development sub-scheduler series. Nothing here depends on the rest of that work.
Looks good to me.
Thanks,
-Andrea
Tejun Heo
reviewerHello,
Applied 1-4 to sched_ext/for-7.2 with Andrea's Reviewed-by added.
Thanks.
--
tejun