bpf: Avoid locks in bpf_timer and bpf_wq
Subsystems (4)
v3 cover bpf: Avoid locks in bpf_timer and bpf_wq
This series reworks implementation of BPF timer and workqueue APIs. The goal is to make both timers and wq non-blocking, enabling their use in NMI context.
Today this code relies on a bpf_spin_lock embedded in the map element to serialize:
* init of the async object,
* setting/changing the callback and bpf_prog
* starting/cancelling the timer/work
* tearing down when the map element is deleted or the map’s user ref is dropped
The basic design approach in this series:
* Use irq_work to offload all blocking work from NMI
* Introduce refcount to guarantee lifetime of the bpf_async_cb structs deferred to potentially multiple irq_work callbacks
* Keep objects under RCU protection to make sure they are not freed while kfuncs/helpers access them (We can't use refcnt for this, as refcnt itself is part of the bpf_async_cb struct)
---
Changes in v3:
- Major rework
- Introduce mpmc_cell, allowing concurrent writes and reads
- Implement irq_work deferring
- Adding selftests
- Introduces bpf_timer_cancel_async kfunc
- Link to v2: https://lore.kernel.org/r/[email protected]
Changes in v2:
- Move refcnt initialization and put (from cancel_and_free())
from patch 5 into the patch 4, so that patch 4 has more clear and full implementation and use of refcnt
- Link to v1: https://lore.kernel.org/r/[email protected]
---
Mykyta Yatsenko (10):
bpf: Refactor __bpf_async_set_callback()
bpf: Factor out timer deletion helper
bpf: Simplify bpf_timer_cancel()
bpf: Add lock-free cell for NMI-safe async operations
bpf: Enable bpf timer and workqueue use in NMI
bpf: Add verifier support for bpf_timer argument in kfuncs
bpf: Introduce bpf_timer_cancel_async() kfunc selftests/bpf: Refactor timer selftests selftests/bpf: Add stress test for timer async cancel selftests/bpf: Verify bpf_timer_cancel_async works
kernel/bpf/Makefile | 2 +- kernel/bpf/helpers.c | 409 +++++++++++++++---------- kernel/bpf/mpmc_cell.c | 62 ++++ kernel/bpf/mpmc_cell.h | 112 +++++++ kernel/bpf/verifier.c | 59 +++- tools/testing/selftests/bpf/prog_tests/timer.c | 92 +++++- tools/testing/selftests/bpf/progs/timer.c | 37 ++- 7 files changed, 588 insertions(+), 185 deletions(-)
---
base-commit: ea180ffbd27ce5abf2a06329fe1fc8d20dc9becf
change-id: 20251028-timer_nolock-457f5b9daace
Best regards,
Signature · 1 line
Mykyta Yatsenko <[email protected]>
RFC PATCH v3 1/10 bpf: Refactor __bpf_async_set_callback()
From: Mykyta Yatsenko <[email protected]>
Refactor __bpf_async_set_callback() getting rid of locks. The idea of the algorithm is to store both callback_fn and prog in struct bpf_async_cb and verify that both pointers are stored, if any pointer does not match (because of the concurrent update), retry until complete match. On each iteration, increment refcnt of the prog that is going to be set and decrement the one that is evicted, ensuring that get/put are balanced, as each iteration has both inc/dec.
--- kernel/bpf/helpers.c | 61 ++++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-)
kernel/bpf/helpers.c +21 -40
@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { };static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type) +struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb; -if (in_nmi ()) -return -EOPNOTSUPP ; -__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } -if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - } - prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog); - goto out; +if (!cb) +return -EPERM ; + +do { +if (prog) { + prog =bpf_prog_inc_not_zero (prog); +if (IS_ERR (prog)) +return PTR_ERR (prog); } + + prev =xchg (&cb->prog, prog); +rcu_assign_pointer (cb->callback_fn, callback_fn); +if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev); - cb->prog = prog; - } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; } BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern *, timer, void *, callback_fn, struct bpf_prog_aux *, aux) { -return __bpf_async_set_callback (timer, callback_fn, aux,0 ,BPF_ASYNC_TYPE_TIMER ); +return __bpf_async_set_callback (timer, callback_fn, aux->prog); }static const struct bpf_func_proto bpf_timer_set_callback_proto = {
@@ -3131,7 +3112,7 @@ __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,if (flags)return -EINVAL ; -return __bpf_async_set_callback (async, callback_fn, aux, flags,BPF_ASYNC_TYPE_WQ ); +return __bpf_async_set_callback (async, callback_fn, aux->prog); } __bpf_kfunc void bpf_preempt_disable(void)
Signature · 1 line
2.52.0
[email protected]
AI reviewRe: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
A small concern: the error code changed from -EINVAL to -EPERM. The verifier comment at kernel/bpf/verifier.c mentions that non-initialized timers "will return -EINVAL". Is the error code change…
Quoted 18 lines
kernel/bpf/helpers.c +3
-5@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { };static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type) +struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb;
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1364
Quoted 10 lines
kernel/bpf/helpers.c +2
-8-if (in_nmi ()) -return -EOPNOTSUPP ; -__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } +if (!cb) +return -EPERM ;
A small concern: the error code changed from -EINVAL to -EPERM. The verifier comment at kernel/bpf/verifier.c mentions that non-initialized timers "will return -EINVAL". Is the error code change intentional, and should the verifier comment be updated?
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1381
Quoted 9 lines
kernel/bpf/helpers.c +0
-9-if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - }
The usercnt check is removed here. The check still exists in __bpf_async_init(), so a timer cannot be initialized when usercnt is zero. But can there be issues if set_callback is called after usercnt drops to zero on an already-initialized timer? The old code would fail with -EPERM, while the new code proceeds.
Quoted 8 lines
kernel/bpf/helpers.c +0
-8- prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog);
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1374
Quoted 10 lines
kernel/bpf/helpers.c +8
-1- goto out; + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog); + if (IS_ERR(prog)) + return PTR_ERR(prog); } + + prev = xchg(&cb->prog, prog); + rcu_assign_pointer(cb->callback_fn, callback_fn);
Is there a potential race between these two operations? Consider two concurrent callers (with programs P and Q) racing:
1. Thread A: xchg stores P into cb->prog
2. Thread B: xchg stores Q into cb->prog, gets P back
3. Thread B: rcu_assign_pointer stores FB (Q's callback)
4. Thread B: bpf_prog_put(P), B's check passes, B exits
5. Thread A: rcu_assign_pointer stores FA (P's callback), overwrites FB
6. Thread A: check fails (cb->prog is Q, not P), retries
After step 5, cb->prog holds Q (with Q's reference) but cb->callback_fn points to FA (from P). If P's refcount drops to zero elsewhere and P is freed, would the timer callback FA become a dangling pointer?
The bpf_timer_cb() function reads cb->callback_fn and calls it directly without verifying it corresponds to cb->prog. The cb->prog field holds the reference keeping the program alive, but if the callback pointer is from a different program that may have been freed, this could be a problem.
Quoted 5 lines
kernel/bpf/helpers.c +1
-2+if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev); - cb->prog = prog;
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1382
Quoted 10 lines
kernel/bpf/helpers.c +4
-5- } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; }
[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
Quoted 19 lines · On Wed, Jan 7, 2026 at 10:22 AM <[email protected]> wrote:
Quoted 19 lines
kernel/bpf/helpers.c +3
-5@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { };static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type) +struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb;
Quoted 16 lines
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1364
Quoted 10 lines
kernel/bpf/helpers.c +2
-8-if (in_nmi ()) -return -EOPNOTSUPP ; -__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } +if (!cb) +return -EPERM ;A small concern: the error code changed from
-EINVALto-EPERM. The verifier comment atkernel/bpf/verifier.cmentions that non-initialized timers "will return-EINVAL". Is the error code change intentional, and should the verifier comment be updated?
good AI, I asked the same
Quoted 10 lines
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1381
Quoted 9 lines
kernel/bpf/helpers.c +0
-9-if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - }
Quoted 6 lines
The usercnt check is removed here. The check still exists in
__bpf_async_init(), so a timer cannot be initialized when usercnt is zero. But can there be issues ifset_callbackis called after usercnt drops to zero on an already-initialized timer? The old code would fail with-EPERM, while the new code proceeds.
Wasn't this discussed earlier, Mykyta? Do you remember the conclusion?
Quoted 29 lines
Quoted 8 lines
kernel/bpf/helpers.c +0
-8- prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog);kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1374
Quoted 10 lines
kernel/bpf/helpers.c +8
-1- goto out; + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog); + if (IS_ERR(prog)) + return PTR_ERR(prog); } + + prev = xchg(&cb->prog, prog); + rcu_assign_pointer(cb->callback_fn, callback_fn);Is there a potential race between these two operations? Consider two concurrent callers (with programs P and Q) racing:
1. Thread A: xchg stores P into
cb->prog
2. Thread B: xchg stores Q intocb->prog, gets P back
3. Thread B:rcu_assign_pointerstores FB (Q's callback)
4. Thread B:bpf_prog_put(P), B's check passes, B exits
5. Thread A:rcu_assign_pointerstores FA (P's callback), overwrites FB
6. Thread A: check fails (cb->progis Q, not P), retries
Quoted 3 lines
After step 5,
cb->progholds Q (with Q's reference) butcb->callback_fnpoints to FA (from P). If P's refcount drops to zero elsewhere and P is freed, would the timer callback FA become a dangling pointer?
AI is not completely wrong here, IMO. No, there is no use-after-free just yet because program is RCU protected, so we don't have dangling pointer just yet.
But. That bpf_prog_inc_not_zero() on retry will fail if P's program refcount dropped to zero already. And then once we exit, we'll have Q+FA combo, which is not good.
So I think we need to do one bpf_prog_inc_not_zero() outside of the loop once and keep that extra ref until we exit. And then instead of bpf_prog_inc_not_zero(prog) inside the loop, we do unconditional bpf_prog_inc(), which we *know* has to succeed. Less error handling inside the loop as well.
Thoughts?
Quoted 30 lines
The
bpf_timer_cb()function readscb->callback_fnand calls it directly without verifying it corresponds tocb->prog. Thecb->progfield holds the reference keeping the program alive, but if the callback pointer is from a different program that may have been freed, this could be a problem.Quoted 5 lines
kernel/bpf/helpers.c +1
-2+if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev); - cb->prog = prog;kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1382
Quoted 10 lines
kernel/bpf/helpers.c +4
-5- } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; }[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
Quoted 92 lines · Andrii Nakryiko
Quoted 18 lines · On Wed, Jan 7, 2026 at 10:22 AM <[email protected]> wrote:
Quoted 18 lines
kernel/bpf/helpers.c +3
-5@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { };static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type) +struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb;Quoted 15 lines
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1364
Quoted 10 lines
kernel/bpf/helpers.c +2
-8-if (in_nmi ()) -return -EOPNOTSUPP ; -__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } +if (!cb) +return -EPERM ;A small concern: the error code changed from
-EINVALto-EPERM. The verifier comment atkernel/bpf/verifier.cmentions that non-initialized timers "will return-EINVAL". Is the error code change intentional, and should the verifier comment be updated?good AI, I asked the same
Quoted 9 lines
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1381
Quoted 9 lines
kernel/bpf/helpers.c +0
-9-if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - }Quoted 6 lines
The usercnt check is removed here. The check still exists in
__bpf_async_init(), so a timer cannot be initialized when usercnt is zero. But can there be issues ifset_callbackis called after usercnt drops to zero on an already-initialized timer? The old code would fail with-EPERM, while the new code proceeds.Wasn't this discussed earlier, Mykyta? Do you remember the conclusion?
Quoted 28 lines
Quoted 8 lines
kernel/bpf/helpers.c +0
-8- prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog);kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1374
Quoted 10 lines
kernel/bpf/helpers.c +8
-1- goto out; + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog); + if (IS_ERR(prog)) + return PTR_ERR(prog); } + + prev = xchg(&cb->prog, prog); + rcu_assign_pointer(cb->callback_fn, callback_fn);Is there a potential race between these two operations? Consider two concurrent callers (with programs P and Q) racing:
1. Thread A: xchg stores P into
cb->prog
2. Thread B: xchg stores Q intocb->prog, gets P back
3. Thread B:rcu_assign_pointerstores FB (Q's callback)
4. Thread B:bpf_prog_put(P), B's check passes, B exits
5. Thread A:rcu_assign_pointerstores FA (P's callback), overwrites FB
6. Thread A: check fails (cb->progis Q, not P), retriesQuoted 3 lines
After step 5,
cb->progholds Q (with Q's reference) butcb->callback_fnpoints to FA (from P). If P's refcount drops to zero elsewhere and P is freed, would the timer callback FA become a dangling pointer?AI is not completely wrong here, IMO. No, there is no use-after-free just yet because program is RCU protected, so we don't have dangling pointer just yet.
But. That
bpf_prog_inc_not_zero()on retry will fail if P's program refcount dropped to zero already. And then once we exit, we'll have Q+FA combo, which is not good.
Quoted 7 lines
So I think we need to do one
bpf_prog_inc_not_zero()outside of the loop once and keep that extra ref until we exit. And then instead ofbpf_prog_inc_not_zero(prog)inside the loop, we do unconditionalbpf_prog_inc(), which we *know* has to succeed. Less error handling inside the loop as well.Thoughts?
Sounds right, alternatively, we may try to reset both prog and callback to NULL if prog's refcnt can't be incremented.
Quoted 29 lines
Quoted 29 lines
The
bpf_timer_cb()function readscb->callback_fnand calls it directly without verifying it corresponds tocb->prog. Thecb->progfield holds the reference keeping the program alive, but if the callback pointer is from a different program that may have been freed, this could be a problem.Quoted 5 lines
kernel/bpf/helpers.c +1
-2+if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev); - cb->prog = prog;kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1382
Quoted 10 lines
kernel/bpf/helpers.c +4
-5- } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; }[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
In reply to Andrii NakryikoQuoted 45 lines · Andrii Nakryiko
Quoted 18 lines · On Wed, Jan 7, 2026 at 10:22 AM <[email protected]> wrote:
Quoted 18 lines
kernel/bpf/helpers.c +3
-5@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { };static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type) +struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb;Quoted 15 lines
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1364
Quoted 10 lines
kernel/bpf/helpers.c +2
-8-if (in_nmi ()) -return -EOPNOTSUPP ; -__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } +if (!cb) +return -EPERM ;A small concern: the error code changed from
-EINVALto-EPERM. The verifier comment atkernel/bpf/verifier.cmentions that non-initialized timers "will return-EINVAL". Is the error code change intentional, and should the verifier comment be updated?good AI, I asked the same
Quoted 9 lines
Quoted 9 lines
kernel/bpf/helpers.c +0
-9-if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - }
Quoted 7 lines
Quoted 6 lines
The usercnt check is removed here. The check still exists in
__bpf_async_init(), so a timer cannot be initialized when usercnt is zero. But can there be issues ifset_callbackis called after usercnt drops to zero on an already-initialized timer? The old code would fail with-EPERM, while the new code proceeds.Wasn't this discussed earlier, Mykyta? Do you remember the conclusion?
I think what I missed is to put rcu lock in this function, because the
last prog reference is put in the rcu callback (the change this patchset introduces). This way we make sure that whatever is set here will be put if map's usercount goes to 0.
Quoted 77 lines
Quoted 29 lines
Quoted 9 lines
kernel/bpf/helpers.c +0
-8- prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog);kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1374
Quoted 10 lines
kernel/bpf/helpers.c +8
-1- goto out; + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog); + if (IS_ERR(prog)) + return PTR_ERR(prog); } + + prev = xchg(&cb->prog, prog); + rcu_assign_pointer(cb->callback_fn, callback_fn);Is there a potential race between these two operations? Consider two concurrent callers (with programs P and Q) racing:
1. Thread A: xchg stores P into
cb->prog
2. Thread B: xchg stores Q intocb->prog, gets P back
3. Thread B:rcu_assign_pointerstores FB (Q's callback)
4. Thread B:bpf_prog_put(P), B's check passes, B exits
5. Thread A:rcu_assign_pointerstores FA (P's callback), overwrites FB
6. Thread A: check fails (cb->progis Q, not P), retriesQuoted 3 lines
After step 5,
cb->progholds Q (with Q's reference) butcb->callback_fnpoints to FA (from P). If P's refcount drops to zero elsewhere and P is freed, would the timer callback FA become a dangling pointer?AI is not completely wrong here, IMO. No, there is no use-after-free just yet because program is RCU protected, so we don't have dangling pointer just yet.
But. That
bpf_prog_inc_not_zero()on retry will fail if P's program refcount dropped to zero already. And then once we exit, we'll have Q+FA combo, which is not good.So I think we need to do one
bpf_prog_inc_not_zero()outside of the loop once and keep that extra ref until we exit. And then instead ofbpf_prog_inc_not_zero(prog)inside the loop, we do unconditionalbpf_prog_inc(), which we *know* has to succeed. Less error handling inside the loop as well.Thoughts?
Quoted 28 lines
The
bpf_timer_cb()function readscb->callback_fnand calls it directly without verifying it corresponds tocb->prog. Thecb->progfield holds the reference keeping the program alive, but if the callback pointer is from a different program that may have been freed, this could be a problem.Quoted 5 lines
kernel/bpf/helpers.c +1
-2+if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev); - cb->prog = prog;kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1382
Quoted 10 lines
kernel/bpf/helpers.c +4
-5- } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; }[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
In reply to Mykyta YatsenkoQuoted 11 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor
__bpf_async_set_callback()getting rid of locks. The idea of the algorithm is to store bothcallback_fnand prog in structbpf_async_cband verify that both pointers are stored, if any pointer does not match (because of the concurrent update), retry until complete match. On each iteration, increment refcnt of the prog that is going to be set and decrement the one that is evicted, ensuring that get/put are balanced, as each iteration has both inc/dec.
Quoted 5 lines
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 61 ++++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-)
LGTM
[...]
Kumar Kartikeya Dwivedi
maintainerRe: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
In reply to Mykyta YatsenkoQuoted 26 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor
__bpf_async_set_callback()getting rid of locks. The idea of the algorithm is to store bothcallback_fnand prog in structbpf_async_cband verify that both pointers are stored, if any pointer does not match (because of the concurrent update), retry until complete match. On each iteration, increment refcnt of the prog that is going to be set and decrement the one that is evicted, ensuring that get/put are balanced, as each iteration has both inc/dec.Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 61 ++++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-)kernel/bpf/helpers.c +0
-2@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { }; static int __bpf_async_set_callback(struct bpf_async_kern *async, void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type)
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1366
Quoted 10 lines
kernel/bpf/helpers.c +3
-5+ struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb; -if (in_nmi ()) -return -EOPNOTSUPP ;
I'm just nitpicking, but it may be cleaner to drop this hunk in the commit which marks both NMI-safe, and keep this commit topical to making this function lockless and ensuring nothing breaks.
Quoted 30 lines
kernel/bpf/helpers.c +6
-24-__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } -if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - } - prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog); - goto out; +if (!cb) +return -EPERM ; + + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog);
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1377
Quoted 10 lines
kernel/bpf/helpers.c +6
-1+if (IS_ERR (prog)) +return PTR_ERR (prog); } + + prev =xchg (&cb->prog, prog); +rcu_assign_pointer (cb->callback_fn, callback_fn); +if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev);
I was wondering whether bpf_prog_put needs adjustment when done in NMI context. At this point I forgot what my conclusion was when looking at it for task_work, but there are two things that may require care:
1) whether in_hardirq() || irqs_disabled() is enough to detect NMI context too. This seems to be fine on x86, but I am not sure it will be the same across architectures.
2) whether schedule_work() can deadlock when invoked from NMI while its locks are held in some lower context.
Once this function is opened up to be called in NMI context you can have a reentrant async_set_callback on the same CPU. This will require both dec_and_test() to drop to 0 on the same CPU somehow. Thus, we may need to defer to irq_work for bpf_prog_put. queue_work() et. al. have local_irq_save() so hardirq/task reentrancy should be prevented for them (they are already supposed to work fine in such cases), even after we lose our own irqsave through the spin lock.
Quoted 33 lines
kernel/bpf/helpers.c +6
-9- cb->prog = prog; - } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; } BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern *, timer, void *, callback_fn, struct bpf_prog_aux *, aux) { -return __bpf_async_set_callback (timer, callback_fn, aux,0 ,BPF_ASYNC_TYPE_TIMER ); +return __bpf_async_set_callback (timer, callback_fn, aux->prog); }static const struct bpf_func_proto bpf_timer_set_callback_proto = {@@ -3131,7 +3112,7 @@ __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,if (flags)return -EINVAL ; -return __bpf_async_set_callback (async, callback_fn, aux, flags,BPF_ASYNC_TYPE_WQ ); +return __bpf_async_set_callback (async, callback_fn, aux->prog); } __bpf_kfunc void bpf_preempt_disable(void) --2.52.0
Kumar Kartikeya Dwivedi
maintainerRe: [PATCH RFC v3 01/10] bpf: Refactor __bpf_async_set_callback()
Quoted 93 lines · Kumar Kartikeya Dwivedi
Quoted 26 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor
__bpf_async_set_callback()getting rid of locks. The idea of the algorithm is to store bothcallback_fnand prog in structbpf_async_cband verify that both pointers are stored, if any pointer does not match (because of the concurrent update), retry until complete match. On each iteration, increment refcnt of the prog that is going to be set and decrement the one that is evicted, ensuring that get/put are balanced, as each iteration has both inc/dec.Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 61 ++++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-)kernel/bpf/helpers.c +0
-2@@ -1355,55 +1355,36 @@ static const struct bpf_func_proto bpf_timer_init_proto = { }; static int __bpf_async_set_callback(struct bpf_async_kern *async, void *callback_fn, -struct bpf_prog_aux *aux,unsigned int flags, -enum bpf_async_type type)kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1366
Quoted 10 lines
kernel/bpf/helpers.c +3
-5+ struct bpf_prog *prog) { -struct bpf_prog *prev, *prog = aux->prog; -struct bpf_async_cb *cb; -int ret =0 ; +struct bpf_prog *prev; +struct bpf_async_cb *cb = async->cb; -if (in_nmi ()) -return -EOPNOTSUPP ;I'm just nitpicking, but it may be cleaner to drop this hunk in the commit which marks both NMI-safe, and keep this commit topical to making this function lockless and ensuring nothing breaks.
Quoted 30 lines
kernel/bpf/helpers.c +6
-24-__bpf_spin_lock_irqsave (&async->lock); - cb = async->cb; -if (!cb) { - ret = -EINVAL ; - goto out; - } -if (!atomic64_read (&cb->map->usercnt)) { -/* maps with timers must be either held by user space -* or pinned in bpffs. Otherwise timer might still be -* running even when bpf prog is detached and user space -* is gone, since map_release_uref won't ever be called. -*/ - ret = -EPERM ; - goto out; - } - prev = cb->prog; -if (prev != prog) { -/* Bump prog refcnt once. Every bpf_timer_set_callback() -* can pick different callback_fn-s within the same prog. -*/ - prog =bpf_prog_inc_not_zero (prog); -if (IS_ERR (prog)) { - ret =PTR_ERR (prog); - goto out; +if (!cb) +return -EPERM ; + + do { + if (prog) { + prog = bpf_prog_inc_not_zero(prog);kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1377
Quoted 10 lines
kernel/bpf/helpers.c +6
-1+if (IS_ERR (prog)) +return PTR_ERR (prog); } + + prev =xchg (&cb->prog, prog); +rcu_assign_pointer (cb->callback_fn, callback_fn); +if (prev) -/* Drop prev prog refcnt when swapping with new prog */ bpf_prog_put (prev);I was wondering whether
bpf_prog_putneeds adjustment when done in NMI context. At this point I forgot what my conclusion was when looking at it fortask_work, but there are two things that may require care:
1) whetherin_hardirq()||irqs_disabled()is enough to detect NMI context too. This seems to be fine on x86, but I am not sure it will be the same across architectures.
2) whetherschedule_work()can deadlock when invoked from NMI while its locks are held in some lower context.
Quoted 9 lines
Once this function is opened up to be called in NMI context you can have a reentrant
async_set_callbackon the same CPU. This will require bothdec_and_test()to drop to 0 on the same CPU somehow. Thus, we may need to defer toirq_workforbpf_prog_put.queue_work()et. al. havelocal_irq_save()so hardirq/task reentrancy should be prevented for them (they are already supposed to work fine in such cases), even after we lose our own irqsave through the spin lock.
I saw that you handled this for other cases in the later patch, but I guess it's missing for async_set_callback.
Quoted 34 lines
Quoted 34 lines
kernel/bpf/helpers.c +6
-9- cb->prog = prog; - } -rcu_assign_pointer (cb->callback_fn, callback_fn); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + + }while (READ_ONCE (cb->prog) != prog ||READ_ONCE (cb->callback_fn) != callback_fn); + +return 0 ; } BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern *, timer, void *, callback_fn, struct bpf_prog_aux *, aux) { -return __bpf_async_set_callback (timer, callback_fn, aux,0 ,BPF_ASYNC_TYPE_TIMER ); +return __bpf_async_set_callback (timer, callback_fn, aux->prog); }static const struct bpf_func_proto bpf_timer_set_callback_proto = {@@ -3131,7 +3112,7 @@ __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,if (flags)return -EINVAL ; -return __bpf_async_set_callback (async, callback_fn, aux, flags,BPF_ASYNC_TYPE_WQ ); +return __bpf_async_set_callback (async, callback_fn, aux->prog); } __bpf_kfunc void bpf_preempt_disable(void) --2.52.0
RFC PATCH v3 2/10 bpf: Factor out timer deletion helper
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Move the timer deletion logic into a dedicated bpf_timer_delete() helper so it can be reused by later patches.
--- kernel/bpf/helpers.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-)
kernel/bpf/helpers.c +18 -11
@@ -1539,18 +1539,10 @@ static struct bpf_async_cb *__bpf_async_cancel_and_free(struct bpf_async_kern *areturn cb; } -/* This function is called by map_delete/update_elem for individual element and -* by ops->map_release_uref when the user space reference to a map reaches zero. -*/ -void bpf_timer_cancel_and_free (void *val) +static void bpf_timer_delete (struct bpf_hrtimer *t) { -struct bpf_hrtimer *t; - - t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free (val); - -if (!t) -return ; -/* We check that bpf_map_delete/update_elem() was called from timer +/* +* We check that bpf_map_delete/update_elem() was called from timer * callback_fn. In such case we don't call hrtimer_cancel() (since it * will deadlock) and don't call hrtimer_try_to_cancel() (since it will * just return -1). Though callback_fn is still running on this cpu it's
@@ -1599,6 +1591,21 @@ void bpf_timer_cancel_and_free(void *val) } } +/* +* This function is called by map_delete/update_elem for individual element and +* by ops->map_release_uref when the user space reference to a map reaches zero. +*/ +void bpf_timer_cancel_and_free (void *val) +{ +struct bpf_hrtimer *t; + + t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free (val); +if (!t) +return ; + +bpf_timer_delete (t); +} +/* This function is called by map_delete/update_elem for individual element and * by ops->map_release_uref when the user space reference to a map reaches zero. */
Signature · 1 line
2.52.0
Re: [PATCH RFC v3 02/10] bpf: Factor out timer deletion helper
Quoted 9 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Move the timer deletion logic into a dedicated
bpf_timer_delete()helper so it can be reused by later patches.Acked-by Eduard Zingerman <[email protected]>Signed-off-by Mykyta Yatsenko <[email protected]>---
RFC PATCH v3 3/10 bpf: Simplify bpf_timer_cancel()
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Remove lock from the bpf_timer_cancel() helper. The lock does not protect from concurrent modification of the bpf_async_cb data fields as those are modified in the callback without locking.
Use guard(rcu)() instead of pair of explicit lock()/unlock().
--- kernel/bpf/helpers.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-)
kernel/bpf/helpers.c +14 -17
@@ -1446,7 +1446,7 @@ static void drop_prog_refcnt(struct bpf_async_cb *async) } } -BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) +BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, async) {struct bpf_hrtimer *t, *cur_t;bool inc = false;
@@ -1454,13 +1454,12 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)if (in_nmi ())return -EOPNOTSUPP ; -rcu_read_lock (); -__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer; +if (!t) +return -EINVAL ; cur_t =this_cpu_read (hrtimer_running);if (cur_t == t) {
@@ -1468,8 +1467,7 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)* its own timer the hrtimer_cancel() will deadlock * since it waits for callback_fn to finish. */ - ret = -EDEADLK ; - goto out; +return -EDEADLK ; }/* Only account in-flight cancellations when invoked from a timer
@@ -1492,20 +1490,19 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)* cancelling and waiting for it synchronously, since it might * do the same. Bail! */ - ret = -EDEADLK ; - goto out; +atomic_dec (&t->cancelling); +return -EDEADLK ; } + drop: -drop_prog_refcnt (&t->cb); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); +__bpf_async_set_callback (async,NULL ,NULL );/* Cancel the timer and wait for associated callback to finish * if it was running. */ - ret = ret ?:hrtimer_cancel (&t->timer); + ret =hrtimer_cancel (&t->timer); +if (inc)atomic_dec (&t->cancelling); -rcu_read_unlock ();return ret; }
Signature · 1 line
2.52.0
[email protected]
AI reviewRe: [PATCH RFC v3 03/10] bpf: Simplify bpf_timer_cancel()
With the lock removed here, can bpf_timer_cancel() race with __bpf_async_cancel_and_free()? The concern is that both paths can attempt to drop the prog reference:
Quoted 4 lines
kernel/bpf/helpers.c +0
-0
[ ... ]
Quoted 17 lines
kernel/bpf/helpers.c +6
-7@@ -1454,13 +1454,12 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)if (in_nmi ())return -EOPNOTSUPP ; -rcu_read_lock (); -__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer; +if (!t) +return -EINVAL ;
[ ... ]
Quoted 5 lines
kernel/bpf/helpers.c +0
-1@@ -1492,20 +1490,19 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)* cancelling and waiting for it synchronously, since it might * do the same. Bail! */ - ret = -EDEADLK ;
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1498
Quoted 10 lines
kernel/bpf/helpers.c +4
-4- goto out; +atomic_dec (&t->cancelling); +return -EDEADLK ; } + drop: -drop_prog_refcnt (&t->cb); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); +__bpf_async_set_callback (async,NULL ,NULL );
With the lock removed here, can bpf_timer_cancel() race with __bpf_async_cancel_and_free()? The concern is that both paths can attempt to drop the prog reference:
bpf_timer_cancel():
__bpf_async_set_callback(async, NULL, NULL)
-> prev = xchg(&cb->prog, NULL)
-> bpf_prog_put(prev) __bpf_async_cancel_and_free():
-> drop_prog_refcnt(cb)
-> prog = cb->prog (plain read)
-> bpf_prog_put(prog)The xchg in __bpf_async_set_callback() is atomic, but drop_prog_refcnt() uses a plain read of cb->prog. If drop_prog_refcnt() reads cb->prog before the xchg sets it to NULL, both paths would call bpf_prog_put() on the same program, leading to a double put of the reference count.
Note: This appears to be addressed later in the series by commit 0035985a0a23 ("bpf: Enable bpf timer and workqueue use in NMI") which moves bpf_prog_put() to the RCU callback, but the race window exists in this intermediate commit.
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1507
Quoted 11 lines
kernel/bpf/helpers.c +2
-2/* Cancel the timer and wait for associated callback to finish * if it was running. */ - ret = ret ?:hrtimer_cancel (&t->timer); + ret =hrtimer_cancel (&t->timer); +if (inc)atomic_dec (&t->cancelling); -rcu_read_unlock ();return ret; }
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 03/10] bpf: Simplify bpf_timer_cancel()
Quoted 10 lines · On Wed, Jan 7, 2026 at 10:22 AM <[email protected]> wrote:
Quoted 5 lines
kernel/bpf/helpers.c +0
-0[ ... ]
Quoted 2 lines
kernel/bpf/helpers.c +0
-0@@ -1454,13 +1454,12 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)
Quoted 13 lines
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1460
Quoted 13 lines
kernel/bpf/helpers.c +4
-7if (in_nmi ())return -EOPNOTSUPP ; -rcu_read_lock (); -__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer;
this should be READ_ONCE() now, no?
Quoted 40 lines
Quoted 2 lines
kernel/bpf/helpers.c +2
-0+if (!t) +return -EINVAL ;[ ... ]
Quoted 5 lines
kernel/bpf/helpers.c +0
-1@@ -1492,20 +1490,19 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)* cancelling and waiting for it synchronously, since it might * do the same. Bail! */ - ret = -EDEADLK ;kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1498
Quoted 10 lines
kernel/bpf/helpers.c +4
-4- goto out; +atomic_dec (&t->cancelling); +return -EDEADLK ; } + drop: -drop_prog_refcnt (&t->cb); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); +__bpf_async_set_callback (async,NULL ,NULL );With the lock removed here, can
bpf_timer_cancel()race with__bpf_async_cancel_and_free()? The concern is that both paths can attempt to drop the prog reference:
bpf_timer_cancel():__bpf_async_set_callback(async, NULL, NULL) -> prev = xchg(&cb->prog, NULL) -> bpf_prog_put(prev)
__bpf_async_cancel_and_free():-> drop_prog_refcnt(cb) -> prog = cb->prog (plain read) -> bpf_prog_put(prog)The xchg in
__bpf_async_set_callback()is atomic, butdrop_prog_refcnt()uses a plain read ofcb->prog. Ifdrop_prog_refcnt()readscb->progbefore the xchg sets it to NULL, both paths would callbpf_prog_put()on the same program, leading to a double put of the reference count.
Quoted 4 lines
Note: This appears to be addressed later in the series by commit
0035985a0a23("bpf: Enable bpf timer and workqueue use in NMI") which movesbpf_prog_put()to the RCU callback, but the race window exists in this intermediate commit.
fair point, do we need to do this early on?
Quoted 19 lines
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1507
Quoted 12 lines
kernel/bpf/helpers.c +2
-2/* Cancel the timer and wait for associated callback to finish * if it was running. */ - ret = ret ?:hrtimer_cancel (&t->timer); + ret =hrtimer_cancel (&t->timer); +if (inc)atomic_dec (&t->cancelling); -rcu_read_unlock ();return ret; }---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Kumar Kartikeya Dwivedi
maintainerRe: [PATCH RFC v3 03/10] bpf: Simplify bpf_timer_cancel()
Quoted 68 lines · Andrii Nakryiko
Quoted 10 lines · On Wed, Jan 7, 2026 at 10:22 AM <[email protected]> wrote:
Quoted 5 lines
kernel/bpf/helpers.c +0
-0[ ... ]
Quoted 2 lines
kernel/bpf/helpers.c +0
-0@@ -1454,13 +1454,12 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)Quoted 13 lines
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1460
Quoted 13 lines
kernel/bpf/helpers.c +4
-7if (in_nmi ())return -EOPNOTSUPP ; -rcu_read_lock (); -__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer;this should be
READ_ONCE()now, no?Quoted 40 lines
Quoted 2 lines
kernel/bpf/helpers.c +2
-0+if (!t) +return -EINVAL ;[ ... ]
Quoted 5 lines
kernel/bpf/helpers.c +0
-1@@ -1492,20 +1490,19 @@ BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer)* cancelling and waiting for it synchronously, since it might * do the same. Bail! */ - ret = -EDEADLK ;kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1498
Quoted 10 lines
kernel/bpf/helpers.c +4
-4- goto out; +atomic_dec (&t->cancelling); +return -EDEADLK ; } + drop: -drop_prog_refcnt (&t->cb); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); +__bpf_async_set_callback (async,NULL ,NULL );With the lock removed here, can
bpf_timer_cancel()race with__bpf_async_cancel_and_free()? The concern is that both paths can attempt to drop the prog reference:
bpf_timer_cancel():__bpf_async_set_callback(async, NULL, NULL) -> prev = xchg(&cb->prog, NULL) -> bpf_prog_put(prev)
__bpf_async_cancel_and_free():-> drop_prog_refcnt(cb) -> prog = cb->prog (plain read) -> bpf_prog_put(prog)The xchg in
__bpf_async_set_callback()is atomic, butdrop_prog_refcnt()uses a plain read ofcb->prog. Ifdrop_prog_refcnt()readscb->progbefore the xchg sets it to NULL, both paths would callbpf_prog_put()on the same program, leading to a double put of the reference count.
Quoted 7 lines
Quoted 4 lines
Note: This appears to be addressed later in the series by commit
0035985a0a23("bpf: Enable bpf timer and workqueue use in NMI") which movesbpf_prog_put()to the RCU callback, but the race window exists in this intermediate commit.fair point, do we need to do this early on?
It might also make sense to just fold this patch into the later one.
Quoted 20 lines
Quoted 20 lines
kernel/bpf/helpers.c BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, timer) · line 1507
Quoted 13 lines
kernel/bpf/helpers.c +2
-2/* Cancel the timer and wait for associated callback to finish * if it was running. */ - ret = ret ?:hrtimer_cancel (&t->timer); + ret =hrtimer_cancel (&t->timer); +if (inc)atomic_dec (&t->cancelling); -rcu_read_unlock ();return ret; }---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
RFC PATCH v3 4/10 bpf: Add lock-free cell for NMI-safe async operations
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Introduce mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.
Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)
The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cell expects users to pre-allocate double buffers.
Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.
This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like hrtimer_start()) to irq_work effectively allowing BPF programs to initiate timers and workqueues from NMI context.
--- kernel/bpf/Makefile | 2 +- kernel/bpf/mpmc_cell.c | 62 +++++++++++++++++++++++++++ kernel/bpf/mpmc_cell.h | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-)
kernel/bpf/Makefile +1 -1
@@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o mpmc_cell.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
kernel/bpf/mpmc_cell.c new file +62 -0
62 lines not on this page · raw message
kernel/bpf/mpmc_cell.h new file +112 -0
112 lines not on this page · raw message
Signature · 1 line
2.52.0
[email protected]
AI reviewRe: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Should this check be "if (cell)" instead of "if (!IS_ERR(cell))"?
Quoted 6 lines
kernel/bpf/mpmc_cell.h new file +0
-0@@ -0,0 +1,112 @@
[ ... ]
Quoted 8 lines
+ * Typical usage:
+ *+ * // Writer (from NMI or any context): + * cell = bpf_mpmc_cell_write_begin(ctl); + * if (!IS_ERR(cell)) {+ *
memcpy(cell, data, size);
+ *bpf_mpmc_cell_write_commit(ctl);
+ * }
Should this check be "if (cell)" instead of "if (!IS_ERR(cell))"?
bpf_mpmc_cell_write_begin() returns NULL when another writer is active, not an ERR_PTR. Since IS_ERR(NULL) returns false, "!IS_ERR(cell)" will evaluate to true when cell is NULL, causing the memcpy to execute with a NULL pointer.
The function documentation correctly states "Return: Pointer to the write cell, or NULL if another writer is active", but the example code here uses the wrong check pattern.
[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Kumar Kartikeya Dwivedi
maintainerRe: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
In reply to Mykyta YatsenkoQuoted 28 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.
Quoted 6 lines
This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.
Quoted 2 lines
[...]
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.
Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.
yes, multiple concurrent writers support would is the main difference between seqcount_latch_t and my implementation. I'll switch to seqcount_latch_t and external synchronization for writers.
Quoted 2 lines
Quoted 2 lines
[...]
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 42 lines · Mykyta Yatsenko
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.
Quoted 3 lines
yes, multiple concurrent writers support would is the main difference between
seqcount_latch_tand my implementation. I'll switch toseqcount_latch_tand external synchronization for writers.
One advantage of custom primitive is that we don't need a second atomic counter for writers. Here we combine the reader latch counter (it's just scaled 2x for custom implementation) and "writer is active" bit (even/odd counter).
With potentially millions of timer activations per second for some extreme cases, would performance be enough reason to have custom "seqlock latch"? (I'm not sure myself, trying to get opinions)
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
[...]
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 46 lines · Andrii Nakryiko
Quoted 41 lines · Mykyta Yatsenko
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.Quoted 3 lines
yes, multiple concurrent writers support would is the main difference between
seqcount_latch_tand my implementation. I'll switch toseqcount_latch_tand external synchronization for writers.
Quoted 9 lines
One advantage of custom primitive is that we don't need a second atomic counter for writers. Here we combine the reader latch counter (it's just scaled 2x for custom implementation) and "writer is active" bit (even/odd counter).
With potentially millions of timer activations per second for some extreme cases, would performance be enough reason to have custom "seqlock latch"? (I'm not sure myself, trying to get opinions)
Actually seqcount_latch_t variant may be faster (correct me if I'm wrong), because mpmc_cell requires 2 lock prefixed instructions to enter the write critical section and seqcount_latch_t just 1.
mpmc_cell:
if (1&atomic_fetch_or_acquire(1, &ctl->seq)) // first lock prefixed insn return;
...
atomic_fetch_add_release(1, &ctl->seq); // second lock prefixed insn
seqcount_latch_t based:
if (atomic_cmpxchg(&ctl->lock, 0, 1)) // first lock prefixed insn return;
write_seqcount_latch_begin(&ctl->seq); // inc with barriers
...
write_seqcount_latch(&ctl->seq); // inc with barriers atomic_set(&ctl->lock, 0); // plain mov on x86_64
Does it look right?
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
[...]
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 79 lines · Mykyta Yatsenko
Quoted 46 lines · Andrii Nakryiko
Quoted 41 lines · Mykyta Yatsenko
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.Quoted 3 lines
yes, multiple concurrent writers support would is the main difference between
seqcount_latch_tand my implementation. I'll switch toseqcount_latch_tand external synchronization for writers.Quoted 9 lines
One advantage of custom primitive is that we don't need a second atomic counter for writers. Here we combine the reader latch counter (it's just scaled 2x for custom implementation) and "writer is active" bit (even/odd counter).
With potentially millions of timer activations per second for some extreme cases, would performance be enough reason to have custom "seqlock latch"? (I'm not sure myself, trying to get opinions)
Actually
seqcount_latch_tvariant may be faster (correct me if I'm wrong), becausempmc_cellrequires 2 lock prefixed instructions to enter the write critical section andseqcount_latch_tjust 1.
mpmc_cell:if (1&atomic_fetch_or_acquire(1,
&ctl->seq)) // first lock prefixed insn return;
...atomic_fetch_add_release(1, &ctl->seq); // second lockprefixed insn
seqcount_latch_tbased:if (atomic_cmpxchg(&ctl->lock, 0, 1)) // first lock prefixed insn return; write_seqcount_latch_begin(&ctl->seq); // inc with barriers ... write_seqcount_latch(&ctl->seq); // inc with barriers atomic_set(&ctl->lock, 0); // plain mov on x86_64
Quoted 1 line
Does it look right?
So I think you are overpivoting on atomic vs non-atomic differences
here: when uncontended, atomic is almost as fast as non-atomic (difference is irrelevant). But under contention, those four writes due to separate latch is four cache line bounces (potentially) between competing CPUs, while with custom sequence logic it's just two.
I'm not dead set on one approach or the other, but I don't think that latch adds that match value. But if Kumar or others insist on latch, I don't mind, in the end both will work.
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
[...]
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 90 lines · Andrii Nakryiko
Quoted 79 lines · Mykyta Yatsenko
Quoted 46 lines · Andrii Nakryiko
Quoted 41 lines · Mykyta Yatsenko
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.Quoted 3 lines
yes, multiple concurrent writers support would is the main difference between
seqcount_latch_tand my implementation. I'll switch toseqcount_latch_tand external synchronization for writers.Quoted 9 lines
One advantage of custom primitive is that we don't need a second atomic counter for writers. Here we combine the reader latch counter (it's just scaled 2x for custom implementation) and "writer is active" bit (even/odd counter).
With potentially millions of timer activations per second for some extreme cases, would performance be enough reason to have custom "seqlock latch"? (I'm not sure myself, trying to get opinions)
Actually
seqcount_latch_tvariant may be faster (correct me if I'm wrong), becausempmc_cellrequires 2 lock prefixed instructions to enter the write critical section andseqcount_latch_tjust 1.
mpmc_cell:if (1&atomic_fetch_or_acquire(1,
&ctl->seq)) // first lock prefixed insn return;
...atomic_fetch_add_release(1, &ctl->seq); // second lockprefixed insn
seqcount_latch_tbased:if (atomic_cmpxchg(&ctl->lock, 0, 1)) // first lock prefixed insn return; write_seqcount_latch_begin(&ctl->seq); // inc with barriers ... write_seqcount_latch(&ctl->seq); // inc with barriers atomic_set(&ctl->lock, 0); // plain mov on x86_64Quoted 1 line
Does it look right?
So I think you are overpivoting on atomic vs non-atomic differences
here: when uncontended, atomic is almost as fast as non-atomic (difference is irrelevant). But under contention, those four writes due to separate latch is four cache line bounces (potentially) between competing CPUs, while with custom sequence logic it's just two.
Quoted 3 lines
I'm not dead set on one approach or the other, but I don't think that latch adds that match value. But if Kumar or others insist on latch, I don't mind, in the end both will work.
I think it's worth writing a micro benchmark for it. Intuitively I feel seqcount_latch_t approach will be faster and it has all the kcsan annotation around it, while this new primitive still needs work to teach kcsan about it.
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
Quoted 97 lines · Alexei Starovoitov
Quoted 90 lines · Andrii Nakryiko
Quoted 79 lines · Mykyta Yatsenko
Quoted 46 lines · Andrii Nakryiko
Quoted 41 lines · Mykyta Yatsenko
Quoted 32 lines · Kumar Kartikeya Dwivedi
Quoted 31 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.Quoted 8 lines
Quoted 2 lines
Signed-off-by Mykyta Yatsenko <[email protected]>---
We already have a dual-versioned concurrency control primitive in the kernel (
seqcount_latch_t). I would just use that instead of reinventing it here. I don't see much of a difference except writer serialization, which can be done on top of it. If it was already considered and discarded for some reason, please add that reason to the commit message.Quoted 3 lines
yes, multiple concurrent writers support would is the main difference between
seqcount_latch_tand my implementation. I'll switch toseqcount_latch_tand external synchronization for writers.Quoted 9 lines
One advantage of custom primitive is that we don't need a second atomic counter for writers. Here we combine the reader latch counter (it's just scaled 2x for custom implementation) and "writer is active" bit (even/odd counter).
With potentially millions of timer activations per second for some extreme cases, would performance be enough reason to have custom "seqlock latch"? (I'm not sure myself, trying to get opinions)
Actually
seqcount_latch_tvariant may be faster (correct me if I'm wrong), becausempmc_cellrequires 2 lock prefixed instructions to enter the write critical section andseqcount_latch_tjust 1.
mpmc_cell:if (1&atomic_fetch_or_acquire(1,
&ctl->seq)) // first lock prefixed insn return;
...atomic_fetch_add_release(1, &ctl->seq); // second lockprefixed insn
seqcount_latch_tbased:if (atomic_cmpxchg(&ctl->lock, 0, 1)) // first lock prefixed insn return; write_seqcount_latch_begin(&ctl->seq); // inc with barriers ... write_seqcount_latch(&ctl->seq); // inc with barriers atomic_set(&ctl->lock, 0); // plain mov on x86_64Quoted 1 line
Does it look right?
So I think you are overpivoting on atomic vs non-atomic differences
here: when uncontended, atomic is almost as fast as non-atomic (difference is irrelevant). But under contention, those four writes due to separate latch is four cache line bounces (potentially) between competing CPUs, while with custom sequence logic it's just two.Quoted 3 lines
I'm not dead set on one approach or the other, but I don't think that latch adds that match value. But if Kumar or others insist on latch, I don't mind, in the end both will work.
Quoted 5 lines
I think it's worth writing a micro benchmark for it. Intuitively I feel
seqcount_latch_tapproach will be faster and it has all the kcsan annotation around it, while this new primitive still needs work to teach kcsan about it.
Let's just stick to latch, no need to have more unnecessary work. As I mentioned to Mykyta offline, latch's advantage (regardless of performance implications) is that someone already thought and validated all the memory barriers, so let's take advantage of that and concentrate on other important aspects here.
Re: [PATCH RFC v3 04/10] bpf: Add lock-free cell for NMI-safe async operations
In reply to Mykyta YatsenkoQuoted 32 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Introduce
mpmc_cell, a lock-free cell primitive designed to support concurrent writes to struct in NMI context (only one writer advances), allowing readers to consume consistent snapshot.Implementation details:
Double buffering allows writers run concurrently with readers (read from one cell, write to another)The implementation uses a sequence-number-based protocol to enable exclusive writes.
* Bit 0 of seq indicates an active writer
* Bits 1+ form a generation counter
* (seq & 2) >> 1 selects the read cell, write cell is opposite
* Writers atomically set bit 0, write to the inactive cell, then increment seq to publish
* Readers snapshot seq, read from the active cell, then validate that seq hasn't changed
mpmc_cellexpects users to pre-allocate double buffers.Key properties:
* Writers never block (fail if lost the race to another writer)
* Readers never block writers (double buffering), but may require retries if write updates the snapshot concurrently.This will be used by BPF timer and workqueue helpers to defer NMI-unsafe operations (like
hrtimer_start()) toirq_workeffectively allowing BPF programs to initiate timers and workqueues from NMI context.
Quoted 7 lines
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/Makefile | 2 +- kernel/bpf/mpmc_cell.c | 62 +++++++++++++++++++++++++++ kernel/bpf/mpmc_cell.h | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-)
LGTM overall (though I didn't think much about all the acquire/release semantics)
Quoted 19 lines
kernel/bpf/Makefile +1
-1@@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o mpmc_cell.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.okernel/bpf/mpmc_cell.c new file +0
-0
kernel/bpf/mpmc_cell.c line 8
Quoted 8 lines
kernel/bpf/mpmc_cell.c new file +8
-0@@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include "mpmc_cell.h" + +static u32 read_cell_idx (struct bpf_mpmc_cell_ctl *ctl,u32 seq) +{ +return (seq &2 ) >>1 ;
unconventional, why not (seq >> 1) & 1?
kernel/bpf/mpmc_cell.c line 21
Quoted 13 lines
kernel/bpf/mpmc_cell.c new file +13
-0+} + +void bpf_mpmc_cell_init (struct bpf_mpmc_cell_ctl *ctl,void *cell1,void *cell2) +{ +atomic_set (&ctl->seq,0 ); + ctl->cell[0 ] = cell1; + ctl->cell[1 ] = cell2; +} + +void *bpf_mpmc_cell_read_begin (struct bpf_mpmc_cell_ctl *ctl,u32 *seq) +{ + *seq =atomic_read_acquire (&ctl->seq); +/* Mask out acive writer bit */
typo: active
kernel/bpf/mpmc_cell.c line 22
Quoted 1 line
kernel/bpf/mpmc_cell.c new file +1
-0+ *seq &= ~1 ;
instead of this bit manipulation, why not return logical epoch to user, i.e., (seq >> 1)
kernel/bpf/mpmc_cell.c line 35
Quoted 13 lines
kernel/bpf/mpmc_cell.c new file +13
-0+ +return ctl->cell[read_cell_idx (ctl, *seq)]; +} + +int bpf_mpmc_cell_read_end (struct bpf_mpmc_cell_ctl *ctl,u32 seq) +{ +u32 new_seq; + +/* Ensure cell reads complete before checking seq */ +smp_rmb (); + + new_seq =atomic_read_acquire (&ctl->seq); + new_seq &= ~1 ;/* Ignore active write bit */
as above, new_seq = seq >> 1 (this is "epoch", lower bit is for internal writer coordination, that's implementation detail, not part of external contract)
Quoted 27 lines
kernel/bpf/mpmc_cell.c new file +27
-0+/* Check if seq changed between begin and end, if it did, new snapshot is available */ +if (new_seq != seq) +return -EAGAIN ; + +return 0 ; +} + +void *bpf_mpmc_cell_write_begin (struct bpf_mpmc_cell_ctl *ctl) +{ +u32 seq; + +/* +* Try to set the lowest bit, on success, writer owns cell exclusively, +* other writers fail +*/ + seq =atomic_fetch_or_acquire (1 , &ctl->seq); +if (seq &1 )/* Check if another writer is active */ +return NULL ; + +/* Write to opposite to read buffer */ +return ctl->cell[read_cell_idx (ctl, seq) ^1 ]; +} + +void bpf_mpmc_cell_write_commit (struct bpf_mpmc_cell_ctl *ctl) +{ +atomic_fetch_add_release (1 , &ctl->seq); +}
[...]
RFC PATCH v3 5/10 bpf: Enable bpf timer and workqueue use in NMI
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to irq_work.
Previously, bpf_timer_start(), and bpf_wq_start()
could not be called from NMI context because they acquired bpf_spin_lock and called hrtimer/schedule_work APIs directly. This patch removes these limitations.
Key changes:
* Remove bpf_spin_lock from struct bpf_async_kern. Replace locked operations with atomic cmpxchg() for initialization and xchg() for cancel and free.
* Add per-async irq_work to defer NMI-unsafe operations (hrtimer_start, hrtimer_try_to_cancel, schedule_work) from NMI to softirq context.
* Use the lock-free mpmc_cell (added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to the irq_work handler.
* Add reference counting to bpf_async_cb to ensure the object stays alive until all scheduled irq_work completes and the timer/work callback finishes.
* Move bpf_prog_put() to RCU callback to handle races between set_callback() and cancel_and_free().
This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
--- kernel/bpf/helpers.c | 288 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 191 insertions(+), 97 deletions(-)
kernel/bpf/helpers.c +191 -97
@@ -29,6 +29,7 @@#include <linux/task_work.h> #include <linux/irq_work.h> #include <linux/buildid.h> +#include <mpmc_cell.h> #include "../../lib/kstrtox.h"
@@ -1095,6 +1096,23 @@ static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)return (void *)value -round_up (map->key_size,8 ); } +enum bpf_async_type { +BPF_ASYNC_TYPE_TIMER =0 , +BPF_ASYNC_TYPE_WQ , +}; + +enum bpf_async_op { +BPF_ASYNC_START , +BPF_ASYNC_CANCEL , +BPF_ASYNC_CANCEL_AND_FREE , +}; + +struct bpf_async_cmd { +u64 nsec; +u32 mode; +u32 op; +}; +struct bpf_async_cb {struct bpf_map *map;struct bpf_prog *prog;
@@ -1105,6 +1123,12 @@ struct bpf_async_cb {struct work_struct delete_work; };u64 flags; +struct irq_work worker; +struct bpf_mpmc_cell_ctl ctl; +struct bpf_async_cmd cmd[2 ]; +atomic_t last_seq; +refcount_t refcnt; +enum bpf_async_type type; };/* BPF map elements can contain 'struct bpf_timer'.
@@ -1142,18 +1166,8 @@ struct bpf_async_kern {struct bpf_hrtimer *timer;struct bpf_work *work; }; -/* bpf_spin_lock is used here instead of spinlock_t to make -* sure that it always fits into space reserved by struct bpf_timer -* regardless of LOCKDEP and spinlock debug flags. -*/ -struct bpf_spin_lock lock; } __attribute__((aligned(8))); -enum bpf_async_type { -BPF_ASYNC_TYPE_TIMER =0 , -BPF_ASYNC_TYPE_WQ , -}; - static DEFINE_PER_CPU(struct bpf_hrtimer *, hrtimer_running);static enum hrtimer_restart bpf_timer_cb (struct hrtimer *hrtimer)
@@ -1219,6 +1233,13 @@ static void bpf_async_cb_rcu_free(struct rcu_head *rcu) { struct bpf_async_cb *cb = container_of(rcu, struct bpf_async_cb, rcu); +/* +* Drop the last reference to prog only after RCU GP, as set_callback() +* may race with cancel_and_free() +*/ +if (cb->prog) +bpf_prog_put (cb->prog); +kfree_nolock (cb); }
@@ -1246,18 +1267,17 @@ static void bpf_timer_delete_work(struct work_struct *work)call_rcu (&t->cb.rcu, bpf_async_cb_rcu_free); } +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async); +static void bpf_async_irq_worker (struct irq_work *work); +static int __bpf_async_init (struct bpf_async_kern *async,struct bpf_map *map,u64 flags,enum bpf_async_type type) { -struct bpf_async_cb *cb; +struct bpf_async_cb *cb, *old_cb;struct bpf_hrtimer *t;struct bpf_work *w;clockid_t clockid;size_t size; -int ret =0 ; - -if (in_nmi ()) -return -EOPNOTSUPP ;switch (type) {case BPF_ASYNC_TYPE_TIMER :
@@ -1270,18 +1290,13 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, ureturn -EINVAL ; } -__bpf_spin_lock_irqsave (&async->lock); t = async->timer; -if (t) { - ret = -EBUSY ; - goto out; - } +if (t) +return -EBUSY ; cb =bpf_map_kmalloc_nolock (map, size,0 , map->numa_node); -if (!cb) { - ret = -ENOMEM ; - goto out; - } +if (!cb) +return -ENOMEM ;switch (type) {case BPF_ASYNC_TYPE_TIMER :
@@ -1304,9 +1319,19 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u cb->map = map; cb->prog =NULL ; cb->flags = flags; + cb->worker =IRQ_WORK_INIT (bpf_async_irq_worker); +bpf_mpmc_cell_init (&cb->ctl, &cb->cmd[0 ], &cb->cmd[1 ]); +refcount_set (&cb->refcnt,1 );/* map's reference */ +atomic_set (&cb->last_seq,0 ); + cb->type = type;rcu_assign_pointer (cb->callback_fn,NULL ); -WRITE_ONCE (async->cb, cb); + old_cb =cmpxchg (&async->cb,NULL , cb); +if (old_cb) { +/* Lost the race to initialize this bpf_async_kern, drop the allocated object */ +kfree_nolock (cb); +return -EBUSY ; + }/* Guarantee the order between async->cb and map->usercnt. So * when there are concurrent uref release and bpf timer init, either * bpf_timer_cancel_and_free() called by uref release reads a no-NULL
@@ -1317,13 +1342,11 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u/* maps with timers must be either held by user space * or pinned in bpffs. */ -WRITE_ONCE (async->cb,NULL ); -kfree_nolock (cb); - ret = -EPERM ; +__bpf_async_cancel_and_free (async); +return -EPERM ; } -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + +return 0 ; } BPF_CALL_3(bpf_timer_init, struct bpf_async_kern *, timer, struct bpf_map *, map,
@@ -1354,6 +1377,61 @@ static const struct bpf_func_proto bpf_timer_init_proto = { .arg3_type = ARG_ANYTHING, }; +static int bpf_async_schedule_op (struct bpf_async_cb *cb,u32 op,u64 nsec,u32 timer_mode) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; + + cmd =bpf_mpmc_cell_write_begin (ctl); +if (!cmd) +return -EBUSY ; + + cmd->nsec = nsec; + cmd->mode = timer_mode; + cmd->op = op; + +bpf_mpmc_cell_write_commit (ctl); + +if (!refcount_inc_not_zero (&cb->refcnt)) +return -EBUSY ; + +irq_work_queue (&cb->worker); + +return 0 ; +} + +static int bpf_async_read_op (struct bpf_async_cb *cb,enum bpf_async_op *op, +u64 *nsec,u32 *flags) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; +u32 seq, last_seq; + +do { + last_seq =atomic_read_acquire (&cb->last_seq); + cmd =bpf_mpmc_cell_read_begin (ctl, &seq); + +/* Return -EBUSY if current seq is consumed by another reader */ +if (seq == last_seq) +return -EBUSY ; + + *nsec = cmd->nsec; + *flags = cmd->mode; + *op = cmd->op; + +/* +* Retry read on one of the two conditions: +* 1. Some writer produced new snapshot while we were reading. Our snapshot may have been +* modified, and not consistent. +* 2. Another reader consumed some snapshot. We need to validate that this snapshot is not +* consumed. This prevents duplicate op processing. +*/ + }while (bpf_mpmc_cell_read_end (ctl, seq) == -EAGAIN || +atomic_cmpxchg_release (&cb->last_seq, last_seq, seq) != last_seq); + +return 0 ; +} +static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn,struct bpf_prog *prog) {
@@ -1395,22 +1473,19 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .arg2_type = ARG_PTR_TO_FUNC, }; -BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flags) +BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, async, u64, nsecs, u64, flags) {struct bpf_hrtimer *t; -int ret =0 ; -enum hrtimer_mode mode; +u32 mode; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags & ~(BPF_F_TIMER_ABS |BPF_F_TIMER_CPU_PIN ))return -EINVAL ; -__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t || !t->cb.prog) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer; +if (!t || !t->cb.prog) +return -EINVAL ;if (flags &BPF_F_TIMER_ABS ) mode =HRTIMER_MODE_ABS_SOFT ;
@@ -1420,10 +1495,7 @@ BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flaif (flags &BPF_F_TIMER_CPU_PIN ) mode |=HRTIMER_MODE_PINNED ; -hrtimer_start (&t->timer,ns_to_ktime (nsecs), mode); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); -return ret; +return bpf_async_schedule_op (&t->cb,BPF_ASYNC_START , nsecs, mode); }static const struct bpf_func_proto bpf_timer_start_proto = {
@@ -1435,17 +1507,6 @@ static const struct bpf_func_proto bpf_timer_start_proto = { .arg3_type = ARG_ANYTHING, }; -static void drop_prog_refcnt (struct bpf_async_cb *async) -{ -struct bpf_prog *prog = async->prog; - -if (prog) { -bpf_prog_put (prog); - async->prog =NULL ; -rcu_assign_pointer (async->callback_fn,NULL ); - } -} - BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, async) {struct bpf_hrtimer *t, *cur_t;
@@ -1513,27 +1574,16 @@ static const struct bpf_func_proto bpf_timer_cancel_proto = { .arg1_type = ARG_PTR_TO_TIMER, }; -static struct bpf_async_cb *__bpf_async_cancel_and_free (struct bpf_async_kern *async) +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async) {struct bpf_async_cb *cb; -/* Performance optimization: read async->cb without lock first. */ -if (!READ_ONCE (async->cb)) -return NULL ; - -__bpf_spin_lock_irqsave (&async->lock); -/* re-read it under lock */ - cb = async->cb; + cb =xchg (&async->cb,NULL );if (!cb) - goto out; -drop_prog_refcnt (cb); -/* The subsequent bpf_timer_start/cancel() helpers won't be able to use -* this timer, since it won't be initialized. -*/ -WRITE_ONCE (async->cb,NULL ); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return cb; +return ; + +/* Consume map's refcnt */ +irq_work_queue (&cb->worker); }static void bpf_timer_delete (struct bpf_hrtimer *t)
@@ -1588,19 +1638,76 @@ static void bpf_timer_delete(struct bpf_hrtimer *t) } } +static void bpf_async_process_op (struct bpf_async_cb *cb,u32 op, +u64 timer_nsec,u32 timer_mode) +{ +switch (cb->type) { +case BPF_ASYNC_TYPE_TIMER : { + struct bpf_hrtimer *t = container_of(cb, struct bpf_hrtimer, cb); + +switch (op) { +case BPF_ASYNC_START : +hrtimer_start (&t->timer,ns_to_ktime (timer_nsec), timer_mode); +break ; +case BPF_ASYNC_CANCEL : +hrtimer_try_to_cancel (&t->timer); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +bpf_timer_delete (t); +break ; +default : +break ; + } +break ; + } +case BPF_ASYNC_TYPE_WQ : { + struct bpf_work *w = container_of(cb, struct bpf_work, cb); + +switch (op) { +case BPF_ASYNC_START : +schedule_work (&w->work); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +/* +* Trigger cancel of the sleepable work, but *do not* wait for +* it to finish. +* kfree will be called once the work has finished. +*/ +schedule_work (&w->delete_work); +break ; +default : +break ; + } +break ; + } + } +} + +static void bpf_async_irq_worker (struct irq_work *work) +{ + struct bpf_async_cb *cb = container_of(work, struct bpf_async_cb, worker); +u32 op, timer_mode; +u64 nsec; +int err; + + err =bpf_async_read_op (cb, &op, &nsec, &timer_mode); +if (err) + goto out; + +bpf_async_process_op (cb, op, nsec, timer_mode); + +out: +if (refcount_dec_and_test (&cb->refcnt)) +bpf_async_process_op (cb,BPF_ASYNC_CANCEL_AND_FREE ,0 ,0 ); +} +/* * This function is called by map_delete/update_elem for individual element and * by ops->map_release_uref when the user space reference to a map reaches zero. */ void bpf_timer_cancel_and_free (void *val) { -struct bpf_hrtimer *t; - - t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free (val); -if (!t) -return ; - -bpf_timer_delete (t); +__bpf_async_cancel_and_free (val); }/* This function is called by map_delete/update_elem for individual element and
@@ -1608,19 +1715,7 @@ void bpf_timer_cancel_and_free(void *val)*/ void bpf_wq_cancel_and_free (void *val) { -struct bpf_work *work; - -BTF_TYPE_EMIT (struct bpf_wq ); - - work = (struct bpf_work *)__bpf_async_cancel_and_free (val); -if (!work) -return ; -/* Trigger cancel of the sleepable work, but *do not* wait for -* it to finish if it was running as we might not be in a -* sleepable context. -* kfree will be called once the work has finished. -*/ -schedule_work (&work->delete_work); +__bpf_async_cancel_and_free (val); } BPF_CALL_2(bpf_kptr_xchg, void *, dst, void *, ptr)
@@ -3093,15 +3188,14 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)struct bpf_async_kern *async = (struct bpf_async_kern *)wq;struct bpf_work *w; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags)return -EINVAL ; w =READ_ONCE (async->work);if (!w || !READ_ONCE (w->cb.prog))return -EINVAL ; -schedule_work (&w->work); +bpf_async_schedule_op (&w->cb,BPF_ASYNC_START ,0 ,0 ); +return 0 ; }
Signature · 1 line
2.52.0
[email protected]
AI reviewRe: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
Should this return the result of bpf_async_schedule_op() instead of unconditionally returning 0? The new bpf_async_schedule_op() can fail with -EBUSY (when the mpmc cell write fails or when…
Quoted 4 lines
kernel/bpf/helpers.c +0
-0
[ ... ]
Quoted 22 lines
kernel/bpf/helpers.c +22
-0+static int bpf_async_schedule_op (struct bpf_async_cb *cb,u32 op,u64 nsec,u32 timer_mode) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; + + cmd =bpf_mpmc_cell_write_begin (ctl); +if (!cmd) +return -EBUSY ; + + cmd->nsec = nsec; + cmd->mode = timer_mode; + cmd->op = op; + +bpf_mpmc_cell_write_commit (ctl); + +if (!refcount_inc_not_zero (&cb->refcnt)) +return -EBUSY ; + +irq_work_queue (&cb->worker); + +return 0 ; +}
[ ... ]
Quoted 18 lines
kernel/bpf/helpers.c +7
-5-BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flags) +BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, async, u64, nsecs, u64, flags) {struct bpf_hrtimer *t; -int ret =0 ; -enum hrtimer_mode mode; +u32 mode; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags & ~(BPF_F_TIMER_ABS |BPF_F_TIMER_CPU_PIN ))return -EINVAL ; +guard (rcu)(); + + t = async->timer; +if (!t || !t->cb.prog) +return -EINVAL ;
[ ... ]
Quoted 6 lines
kernel/bpf/helpers.c +1
-4-hrtimer_start (&t->timer,ns_to_ktime (nsecs), mode); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); -return ret; +return bpf_async_schedule_op (&t->cb,BPF_ASYNC_START , nsecs, mode); }
[ ... ]
Quoted 7 lines
kernel/bpf/helpers.c +0
-2@@ -3093,15 +3188,14 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)struct bpf_async_kern *async = (struct bpf_async_kern *)wq;struct bpf_work *w; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags)
kernel/bpf/helpers.c __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) · line 3200
Quoted 10 lines
kernel/bpf/helpers.c +2
-1return -EINVAL ; w =READ_ONCE (async->work);if (!w || !READ_ONCE (w->cb.prog))return -EINVAL ; -schedule_work (&w->work); +bpf_async_schedule_op (&w->cb,BPF_ASYNC_START ,0 ,0 ); +return 0 ; }
Should this return the result of bpf_async_schedule_op() instead of unconditionally returning 0? The new bpf_async_schedule_op() can fail with -EBUSY (when the mpmc cell write fails or when refcount_inc_not_zero fails), but this error is silently dropped.
For comparison, bpf_timer_start() does return the result of bpf_async_schedule_op(). This inconsistency could cause bpf_wq_start() callers to believe work was successfully scheduled when it was not.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/20791345842
Re: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
In reply to Mykyta YatsenkoQuoted 72 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to
irq_work.Previously,
bpf_timer_start(), andbpf_wq_start()
could not be called from NMI context because they acquiredbpf_spin_lockand called hrtimer/schedule_work APIs directly. This patch removes these limitations.Key changes:
* Removebpf_spin_lockfrom structbpf_async_kern. Replace locked operations with atomiccmpxchg()for initialization andxchg()for cancel and free.
* Add per-asyncirq_workto defer NMI-unsafe operations (hrtimer_start,hrtimer_try_to_cancel,schedule_work) from NMI to softirq context.
* Use the lock-freempmc_cell(added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to theirq_workhandler.
* Add reference counting tobpf_async_cbto ensure the object stays alive until all scheduledirq_workcompletes and the timer/work callback finishes.
* Movebpf_prog_put()to RCU callback to handle races betweenset_callback()andcancel_and_free().This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 288 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 191 insertions(+), 97 deletions(-)kernel/bpf/helpers.c +18
-0@@ -29,6 +29,7 @@#include <linux/task_work.h> #include <linux/irq_work.h> #include <linux/buildid.h> +#include <mpmc_cell.h> #include "../../lib/kstrtox.h" @@ -1095,6 +1096,23 @@ static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)return (void *)value -round_up (map->key_size,8 ); } +enum bpf_async_type { +BPF_ASYNC_TYPE_TIMER =0 , +BPF_ASYNC_TYPE_WQ , +}; + +enum bpf_async_op { +BPF_ASYNC_START , +BPF_ASYNC_CANCEL , +BPF_ASYNC_CANCEL_AND_FREE , +}; + +struct bpf_async_cmd { +u64 nsec; +u32 mode; +u32 op; +}; +struct bpf_async_cb {struct bpf_map *map;struct bpf_prog *prog;
kernel/bpf/helpers.c struct bpf_async_cb { · line 1127
Quoted 5 lines
kernel/bpf/helpers.c +2
-0@@ -1105,6 +1123,12 @@ struct bpf_async_cb {struct work_struct delete_work; };u64 flags; +struct irq_work worker; +struct bpf_mpmc_cell_ctl ctl;
nit: I'd call it more meaningful "cmd_cell"
Quoted 8 lines
kernel/bpf/helpers.c +4
-0+struct bpf_async_cmd cmd[2 ]; +atomic_t last_seq; +refcount_t refcnt; +enum bpf_async_type type; };/* BPF map elements can contain 'struct bpf_timer'.
kernel/bpf/helpers.c struct bpf_async_kern { · line 1149
Quoted 8 lines
kernel/bpf/helpers.c +0
-5@@ -1142,18 +1166,8 @@ struct bpf_async_kern {struct bpf_hrtimer *timer;struct bpf_work *work; }; -/* bpf_spin_lock is used here instead of spinlock_t to make -* sure that it always fits into space reserved by struct bpf_timer -* regardless of LOCKDEP and spinlock debug flags. -*/ -struct bpf_spin_lock lock;
we have to leave dummy placeholder instead of preserve bpf_async_kern size for ABI compatibility
Quoted 48 lines
kernel/bpf/helpers.c +11
-10} __attribute__((aligned(8))); -enum bpf_async_type { -BPF_ASYNC_TYPE_TIMER =0 , -BPF_ASYNC_TYPE_WQ , -}; - static DEFINE_PER_CPU(struct bpf_hrtimer *, hrtimer_running);static enum hrtimer_restart bpf_timer_cb (struct hrtimer *hrtimer)@@ -1219,6 +1233,13 @@ static void bpf_async_cb_rcu_free(struct rcu_head *rcu) { struct bpf_async_cb *cb = container_of(rcu, struct bpf_async_cb, rcu); +/* +* Drop the last reference to prog only after RCU GP, as set_callback() +* may race with cancel_and_free() +*/ +if (cb->prog) +bpf_prog_put (cb->prog); +kfree_nolock (cb); }@@ -1246,18 +1267,17 @@ static void bpf_timer_delete_work(struct work_struct *work)call_rcu (&t->cb.rcu, bpf_async_cb_rcu_free); } +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async); +static void bpf_async_irq_worker (struct irq_work *work); +static int __bpf_async_init (struct bpf_async_kern *async,struct bpf_map *map,u64 flags,enum bpf_async_type type) { -struct bpf_async_cb *cb; +struct bpf_async_cb *cb, *old_cb;struct bpf_hrtimer *t;struct bpf_work *w;clockid_t clockid;size_t size; -int ret =0 ; - -if (in_nmi ()) -return -EOPNOTSUPP ;switch (type) {case BPF_ASYNC_TYPE_TIMER :
kernel/bpf/helpers.c static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u · line 1293
Quoted 5 lines
kernel/bpf/helpers.c +0
-1@@ -1270,18 +1290,13 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, ureturn -EINVAL ; } -__bpf_spin_lock_irqsave (&async->lock); t = async->timer;
READ_ONCE()?
Quoted 69 lines
kernel/bpf/helpers.c +28
-15-if (t) { - ret = -EBUSY ; - goto out; - } +if (t) +return -EBUSY ; cb =bpf_map_kmalloc_nolock (map, size,0 , map->numa_node); -if (!cb) { - ret = -ENOMEM ; - goto out; - } +if (!cb) +return -ENOMEM ;switch (type) {case BPF_ASYNC_TYPE_TIMER :@@ -1304,9 +1319,19 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u cb->map = map; cb->prog =NULL ; cb->flags = flags; + cb->worker =IRQ_WORK_INIT (bpf_async_irq_worker); +bpf_mpmc_cell_init (&cb->ctl, &cb->cmd[0 ], &cb->cmd[1 ]); +refcount_set (&cb->refcnt,1 );/* map's reference */ +atomic_set (&cb->last_seq,0 ); + cb->type = type;rcu_assign_pointer (cb->callback_fn,NULL ); -WRITE_ONCE (async->cb, cb); + old_cb =cmpxchg (&async->cb,NULL , cb); +if (old_cb) { +/* Lost the race to initialize this bpf_async_kern, drop the allocated object */ +kfree_nolock (cb); +return -EBUSY ; + }/* Guarantee the order between async->cb and map->usercnt. So * when there are concurrent uref release and bpf timer init, either * bpf_timer_cancel_and_free() called by uref release reads a no-NULL @@ -1317,13 +1342,11 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u/* maps with timers must be either held by user space * or pinned in bpffs. */ -WRITE_ONCE (async->cb,NULL ); -kfree_nolock (cb); - ret = -EPERM ; +__bpf_async_cancel_and_free (async); +return -EPERM ; } -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + +return 0 ; } BPF_CALL_3(bpf_timer_init, struct bpf_async_kern *, timer, struct bpf_map *, map,@@ -1354,6 +1377,61 @@ static const struct bpf_func_proto bpf_timer_init_proto = { .arg3_type = ARG_ANYTHING, }; +static int bpf_async_schedule_op (struct bpf_async_cb *cb,u32 op,u64 nsec,u32 timer_mode) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; + + cmd =bpf_mpmc_cell_write_begin (ctl); +if (!cmd) +return -EBUSY ; +
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1398
Quoted 10 lines
kernel/bpf/helpers.c +10
-0+ cmd->nsec = nsec; + cmd->mode = timer_mode; + cmd->op = op; + +bpf_mpmc_cell_write_commit (ctl); + +if (!refcount_inc_not_zero (&cb->refcnt)) +return -EBUSY ; + +irq_work_queue (&cb->worker);
if not in NMI and not irq-disabled mode, we should be able to call bpf_async_irq_worker() directly here and execute action synchronously without irq_work execution. Add TODO so we don't forget to implement that before patch set lands?
Quoted 21 lines
kernel/bpf/helpers.c +21
-0+ +return 0 ; +} + +static int bpf_async_read_op (struct bpf_async_cb *cb,enum bpf_async_op *op, +u64 *nsec,u32 *flags) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; +u32 seq, last_seq; + + do { + last_seq = atomic_read_acquire(&cb->last_seq); + cmd = bpf_mpmc_cell_read_begin(ctl, &seq); + + /* Return -EBUSY if current seq is consumed by another reader */ + if (seq == last_seq) + return -EBUSY; + + *nsec = cmd->nsec; + *flags = cmd->mode;
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1429
Quoted 10 lines
kernel/bpf/helpers.c +10
-0+ *op = cmd->op; + +/* +* Retry read on one of the two conditions: +* 1. Some writer produced new snapshot while we were reading. Our snapshot may have been +* modified, and not consistent. +* 2. Another reader consumed some snapshot. We need to validate that this snapshot is not +* consumed. This prevents duplicate op processing. +*/ + } while (bpf_mpmc_cell_read_end(ctl, seq) == -EAGAIN ||
can read_end return any other error? If yes, how do we handle? If not, why hard-code -EAGAIN here, maybe just < 0 check?
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1430
Quoted 1 line
kernel/bpf/helpers.c +1
-0+ atomic_cmpxchg_release(&cb->last_seq, last_seq, seq) != last_seq);
nit: repeat condition is complicated enough (and requires multi-line weirdly indented comment), so I'd do:
while (true) {
....
if (bpf_mpmc_cell_read_end(ctl, seq) < 0)
continue;
if (atomic_cmpxchg() == last_seq) /* success*/
break;(or invert cmpxchg + continue, and then stand-alone break)
Quoted 23 lines
kernel/bpf/helpers.c +6
-5+ +return 0 ; +} +static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn,struct bpf_prog *prog) {@@ -1395,22 +1473,19 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .arg2_type = ARG_PTR_TO_FUNC, }; -BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flags) +BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, async, u64, nsecs, u64, flags) {struct bpf_hrtimer *t; -int ret =0 ; -enum hrtimer_mode mode; +u32 mode; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags & ~(BPF_F_TIMER_ABS |BPF_F_TIMER_CPU_PIN ))return -EINVAL ;
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_set_callback_proto = { · line 1486
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t || !t->cb.prog) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer;
READ_ONCE()
Quoted 56 lines
kernel/bpf/helpers.c +5
-26+if (!t || !t->cb.prog) +return -EINVAL ;if (flags &BPF_F_TIMER_ABS ) mode =HRTIMER_MODE_ABS_SOFT ;@@ -1420,10 +1495,7 @@ BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flaif (flags &BPF_F_TIMER_CPU_PIN ) mode |=HRTIMER_MODE_PINNED ; -hrtimer_start (&t->timer,ns_to_ktime (nsecs), mode); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); -return ret; +return bpf_async_schedule_op (&t->cb,BPF_ASYNC_START , nsecs, mode); }static const struct bpf_func_proto bpf_timer_start_proto = {@@ -1435,17 +1507,6 @@ static const struct bpf_func_proto bpf_timer_start_proto = { .arg3_type = ARG_ANYTHING, }; -static void drop_prog_refcnt (struct bpf_async_cb *async) -{ -struct bpf_prog *prog = async->prog; - -if (prog) { -bpf_prog_put (prog); - async->prog =NULL ; -rcu_assign_pointer (async->callback_fn,NULL ); - } -} - BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, async) {struct bpf_hrtimer *t, *cur_t;@@ -1513,27 +1574,16 @@ static const struct bpf_func_proto bpf_timer_cancel_proto = { .arg1_type = ARG_PTR_TO_TIMER, }; -static struct bpf_async_cb *__bpf_async_cancel_and_free (struct bpf_async_kern *async) +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async) {struct bpf_async_cb *cb; -/* Performance optimization: read async->cb without lock first. */ -if (!READ_ONCE (async->cb)) -return NULL ; - -__bpf_spin_lock_irqsave (&async->lock); -/* re-read it under lock */ - cb = async->cb; + cb =xchg (&async->cb,NULL );if (!cb) - goto out; -drop_prog_refcnt (cb); -/* The subsequent bpf_timer_start/cancel() helpers won't be able to use
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_cancel_proto = { · line 1586
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-* this timer, since it won't be initialized. -*/ -WRITE_ONCE (async->cb,NULL ); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return cb; +return ; + +/* Consume map's refcnt */ +irq_work_queue (&cb->worker);
hm... this is subtle (and maybe broken?) irq_work_queue() can be ignored here, if there is already another one scheduled, so I think your clever idea with CANCEL_AND_FREE being done based on this refcount drop is flawed...
CANCEL_AND_FREE has to succeed, so it's out-of-bounds signal that shouldn't be going through that command cell, yes. But can't we just have a simple one-way bool that will be set to true here (+ memory barriers, maybe), and then irq_work_queue() scheduled. If there is irq work is scheduled, it will inevitable will see this flag (even if it's not our callback), and if not, then irq_work_queue() will successfully schedule callback which will also clean up.
also, same about TODO for irq_work_queue() avoidance
Quoted 130 lines
kernel/bpf/helpers.c +67
-24}static void bpf_timer_delete (struct bpf_hrtimer *t)@@ -1588,19 +1638,76 @@ static void bpf_timer_delete(struct bpf_hrtimer *t) } } +static void bpf_async_process_op (struct bpf_async_cb *cb,u32 op, +u64 timer_nsec,u32 timer_mode) +{ +switch (cb->type) { +case BPF_ASYNC_TYPE_TIMER : { + struct bpf_hrtimer *t = container_of(cb, struct bpf_hrtimer, cb); + +switch (op) { +case BPF_ASYNC_START : +hrtimer_start (&t->timer,ns_to_ktime (timer_nsec), timer_mode); +break ; +case BPF_ASYNC_CANCEL : +hrtimer_try_to_cancel (&t->timer); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +bpf_timer_delete (t); +break ; +default : +break ; + } +break ; + } +case BPF_ASYNC_TYPE_WQ : { + struct bpf_work *w = container_of(cb, struct bpf_work, cb); + +switch (op) { +case BPF_ASYNC_START : +schedule_work (&w->work); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +/* +* Trigger cancel of the sleepable work, but *do not* wait for +* it to finish. +* kfree will be called once the work has finished. +*/ +schedule_work (&w->delete_work); +break ; +default : +break ; + } +break ; + } + } +} + +static void bpf_async_irq_worker (struct irq_work *work) +{ + struct bpf_async_cb *cb = container_of(work, struct bpf_async_cb, worker); +u32 op, timer_mode; +u64 nsec; +int err; + + err =bpf_async_read_op (cb, &op, &nsec, &timer_mode); +if (err) + goto out; + +bpf_async_process_op (cb, op, nsec, timer_mode); + +out: +if (refcount_dec_and_test (&cb->refcnt)) +bpf_async_process_op (cb,BPF_ASYNC_CANCEL_AND_FREE ,0 ,0 ); +} +/* * This function is called by map_delete/update_elem for individual element and * by ops->map_release_uref when the user space reference to a map reaches zero. */ void bpf_timer_cancel_and_free (void *val) { -struct bpf_hrtimer *t; - - t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free (val); -if (!t) -return ; - -bpf_timer_delete (t); +__bpf_async_cancel_and_free (val); }/* This function is called by map_delete/update_elem for individual element and @@ -1608,19 +1715,7 @@ void bpf_timer_cancel_and_free(void *val)*/ void bpf_wq_cancel_and_free (void *val) { -struct bpf_work *work; - -BTF_TYPE_EMIT (struct bpf_wq ); - - work = (struct bpf_work *)__bpf_async_cancel_and_free (val); -if (!work) -return ; -/* Trigger cancel of the sleepable work, but *do not* wait for -* it to finish if it was running as we might not be in a -* sleepable context. -* kfree will be called once the work has finished. -*/ -schedule_work (&work->delete_work); +__bpf_async_cancel_and_free (val); } BPF_CALL_2(bpf_kptr_xchg, void *, dst, void *, ptr)@@ -3093,15 +3188,14 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)struct bpf_async_kern *async = (struct bpf_async_kern *)wq;struct bpf_work *w; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags)return -EINVAL ; w =READ_ONCE (async->work);if (!w || !READ_ONCE (w->cb.prog))return -EINVAL ; -schedule_work (&w->work); +bpf_async_schedule_op (&w->cb,BPF_ASYNC_START ,0 ,0 ); +return 0 ; } --2.52.0
Re: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
Quoted 393 lines · Andrii Nakryiko
Quoted 71 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to
irq_work.Previously,
bpf_timer_start(), andbpf_wq_start()
could not be called from NMI context because they acquiredbpf_spin_lockand called hrtimer/schedule_work APIs directly. This patch removes these limitations.Key changes:
* Removebpf_spin_lockfrom structbpf_async_kern. Replace locked operations with atomiccmpxchg()for initialization andxchg()for cancel and free.
* Add per-asyncirq_workto defer NMI-unsafe operations (hrtimer_start,hrtimer_try_to_cancel,schedule_work) from NMI to softirq context.
* Use the lock-freempmc_cell(added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to theirq_workhandler.
* Add reference counting tobpf_async_cbto ensure the object stays alive until all scheduledirq_workcompletes and the timer/work callback finishes.
* Movebpf_prog_put()to RCU callback to handle races betweenset_callback()andcancel_and_free().This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 288 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 191 insertions(+), 97 deletions(-)kernel/bpf/helpers.c +18
-0@@ -29,6 +29,7 @@#include <linux/task_work.h> #include <linux/irq_work.h> #include <linux/buildid.h> +#include <mpmc_cell.h> #include "../../lib/kstrtox.h" @@ -1095,6 +1096,23 @@ static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)return (void *)value -round_up (map->key_size,8 ); } +enum bpf_async_type { +BPF_ASYNC_TYPE_TIMER =0 , +BPF_ASYNC_TYPE_WQ , +}; + +enum bpf_async_op { +BPF_ASYNC_START , +BPF_ASYNC_CANCEL , +BPF_ASYNC_CANCEL_AND_FREE , +}; + +struct bpf_async_cmd { +u64 nsec; +u32 mode; +u32 op; +}; +struct bpf_async_cb {struct bpf_map *map;struct bpf_prog *prog;kernel/bpf/helpers.c struct bpf_async_cb { · line 1127
Quoted 5 lines
kernel/bpf/helpers.c +2
-0@@ -1105,6 +1123,12 @@ struct bpf_async_cb {struct work_struct delete_work; };u64 flags; +struct irq_work worker; +struct bpf_mpmc_cell_ctl ctl;nit: I'd call it more meaningful "
cmd_cell"Quoted 8 lines
kernel/bpf/helpers.c +4
-0+struct bpf_async_cmd cmd[2 ]; +atomic_t last_seq; +refcount_t refcnt; +enum bpf_async_type type; };/* BPF map elements can contain 'struct bpf_timer'. kernel/bpf/helpers.c struct bpf_async_kern { · line 1149
Quoted 8 lines
kernel/bpf/helpers.c +0
-5@@ -1142,18 +1166,8 @@ struct bpf_async_kern {struct bpf_hrtimer *timer;struct bpf_work *work; }; -/* bpf_spin_lock is used here instead of spinlock_t to make -* sure that it always fits into space reserved by struct bpf_timer -* regardless of LOCKDEP and spinlock debug flags. -*/ -struct bpf_spin_lock lock;we have to leave dummy placeholder instead of preserve
bpf_async_kernsize for ABI compatibilityQuoted 48 lines
kernel/bpf/helpers.c +11
-10} __attribute__((aligned(8))); -enum bpf_async_type { -BPF_ASYNC_TYPE_TIMER =0 , -BPF_ASYNC_TYPE_WQ , -}; - static DEFINE_PER_CPU(struct bpf_hrtimer *, hrtimer_running);static enum hrtimer_restart bpf_timer_cb (struct hrtimer *hrtimer)@@ -1219,6 +1233,13 @@ static void bpf_async_cb_rcu_free(struct rcu_head *rcu) { struct bpf_async_cb *cb = container_of(rcu, struct bpf_async_cb, rcu); +/* +* Drop the last reference to prog only after RCU GP, as set_callback() +* may race with cancel_and_free() +*/ +if (cb->prog) +bpf_prog_put (cb->prog); +kfree_nolock (cb); }@@ -1246,18 +1267,17 @@ static void bpf_timer_delete_work(struct work_struct *work)call_rcu (&t->cb.rcu, bpf_async_cb_rcu_free); } +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async); +static void bpf_async_irq_worker (struct irq_work *work); +static int __bpf_async_init (struct bpf_async_kern *async,struct bpf_map *map,u64 flags,enum bpf_async_type type) { -struct bpf_async_cb *cb; +struct bpf_async_cb *cb, *old_cb;struct bpf_hrtimer *t;struct bpf_work *w;clockid_t clockid;size_t size; -int ret =0 ; - -if (in_nmi ()) -return -EOPNOTSUPP ;switch (type) {case BPF_ASYNC_TYPE_TIMER :kernel/bpf/helpers.c static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u · line 1293
Quoted 5 lines
kernel/bpf/helpers.c +0
-1@@ -1270,18 +1290,13 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, ureturn -EINVAL ; } -__bpf_spin_lock_irqsave (&async->lock); t = async->timer;
READ_ONCE()?Quoted 69 lines
kernel/bpf/helpers.c +28
-15-if (t) { - ret = -EBUSY ; - goto out; - } +if (t) +return -EBUSY ; cb =bpf_map_kmalloc_nolock (map, size,0 , map->numa_node); -if (!cb) { - ret = -ENOMEM ; - goto out; - } +if (!cb) +return -ENOMEM ;switch (type) {case BPF_ASYNC_TYPE_TIMER :@@ -1304,9 +1319,19 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u cb->map = map; cb->prog =NULL ; cb->flags = flags; + cb->worker =IRQ_WORK_INIT (bpf_async_irq_worker); +bpf_mpmc_cell_init (&cb->ctl, &cb->cmd[0 ], &cb->cmd[1 ]); +refcount_set (&cb->refcnt,1 );/* map's reference */ +atomic_set (&cb->last_seq,0 ); + cb->type = type;rcu_assign_pointer (cb->callback_fn,NULL ); -WRITE_ONCE (async->cb, cb); + old_cb =cmpxchg (&async->cb,NULL , cb); +if (old_cb) { +/* Lost the race to initialize this bpf_async_kern, drop the allocated object */ +kfree_nolock (cb); +return -EBUSY ; + }/* Guarantee the order between async->cb and map->usercnt. So * when there are concurrent uref release and bpf timer init, either * bpf_timer_cancel_and_free() called by uref release reads a no-NULL @@ -1317,13 +1342,11 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u/* maps with timers must be either held by user space * or pinned in bpffs. */ -WRITE_ONCE (async->cb,NULL ); -kfree_nolock (cb); - ret = -EPERM ; +__bpf_async_cancel_and_free (async); +return -EPERM ; } -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return ret; + +return 0 ; } BPF_CALL_3(bpf_timer_init, struct bpf_async_kern *, timer, struct bpf_map *, map,@@ -1354,6 +1377,61 @@ static const struct bpf_func_proto bpf_timer_init_proto = { .arg3_type = ARG_ANYTHING, }; +static int bpf_async_schedule_op (struct bpf_async_cb *cb,u32 op,u64 nsec,u32 timer_mode) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; + + cmd =bpf_mpmc_cell_write_begin (ctl); +if (!cmd) +return -EBUSY ; +kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1398
Quoted 10 lines
kernel/bpf/helpers.c +10
-0+ cmd->nsec = nsec; + cmd->mode = timer_mode; + cmd->op = op; + +bpf_mpmc_cell_write_commit (ctl); + +if (!refcount_inc_not_zero (&cb->refcnt)) +return -EBUSY ; + +irq_work_queue (&cb->worker);if not in NMI and not irq-disabled mode, we should be able to call
bpf_async_irq_worker()directly here and execute action synchronously withoutirq_workexecution. Add TODO so we don't forget to implement that before patch set lands?Quoted 21 lines
kernel/bpf/helpers.c +21
-0+ +return 0 ; +} + +static int bpf_async_read_op (struct bpf_async_cb *cb,enum bpf_async_op *op, +u64 *nsec,u32 *flags) +{ +struct bpf_mpmc_cell_ctl *ctl = &cb->ctl; +struct bpf_async_cmd *cmd; +u32 seq, last_seq; + + do { + last_seq = atomic_read_acquire(&cb->last_seq); + cmd = bpf_mpmc_cell_read_begin(ctl, &seq); + + /* Return -EBUSY if current seq is consumed by another reader */ + if (seq == last_seq) + return -EBUSY; + + *nsec = cmd->nsec; + *flags = cmd->mode;kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1429
Quoted 10 lines
kernel/bpf/helpers.c +10
-0+ *op = cmd->op; + +/* +* Retry read on one of the two conditions: +* 1. Some writer produced new snapshot while we were reading. Our snapshot may have been +* modified, and not consistent. +* 2. Another reader consumed some snapshot. We need to validate that this snapshot is not +* consumed. This prevents duplicate op processing. +*/ + } while (bpf_mpmc_cell_read_end(ctl, seq) == -EAGAIN ||can
read_endreturn any other error? If yes, how do we handle? If not, why hard-code-EAGAINhere, maybe just< 0check?kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_init_proto = { · line 1430
Quoted 1 line
kernel/bpf/helpers.c +1
-0+ atomic_cmpxchg_release(&cb->last_seq, last_seq, seq) != last_seq);nit: repeat condition is complicated enough (and requires multi-line weirdly indented comment), so I'd do:
while (true) { .... if (bpf_mpmc_cell_read_end(ctl, seq) < 0) continue; if (atomic_cmpxchg() == last_seq) /* success*/ break;(or invert cmpxchg + continue, and then stand-alone break)
Quoted 23 lines
kernel/bpf/helpers.c +6
-5+ +return 0 ; +} +static int __bpf_async_set_callback (struct bpf_async_kern *async,void *callback_fn,struct bpf_prog *prog) {@@ -1395,22 +1473,19 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .arg2_type = ARG_PTR_TO_FUNC, }; -BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flags) +BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, async, u64, nsecs, u64, flags) {struct bpf_hrtimer *t; -int ret =0 ; -enum hrtimer_mode mode; +u32 mode; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags & ~(BPF_F_TIMER_ABS |BPF_F_TIMER_CPU_PIN ))return -EINVAL ;kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_set_callback_proto = { · line 1486
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-__bpf_spin_lock_irqsave (&timer->lock); - t = timer->timer; -if (!t || !t->cb.prog) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer;
READ_ONCE()Quoted 56 lines
kernel/bpf/helpers.c +5
-26+if (!t || !t->cb.prog) +return -EINVAL ;if (flags &BPF_F_TIMER_ABS ) mode =HRTIMER_MODE_ABS_SOFT ;@@ -1420,10 +1495,7 @@ BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flaif (flags &BPF_F_TIMER_CPU_PIN ) mode |=HRTIMER_MODE_PINNED ; -hrtimer_start (&t->timer,ns_to_ktime (nsecs), mode); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); -return ret; +return bpf_async_schedule_op (&t->cb,BPF_ASYNC_START , nsecs, mode); }static const struct bpf_func_proto bpf_timer_start_proto = {@@ -1435,17 +1507,6 @@ static const struct bpf_func_proto bpf_timer_start_proto = { .arg3_type = ARG_ANYTHING, }; -static void drop_prog_refcnt (struct bpf_async_cb *async) -{ -struct bpf_prog *prog = async->prog; - -if (prog) { -bpf_prog_put (prog); - async->prog =NULL ; -rcu_assign_pointer (async->callback_fn,NULL ); - } -} - BPF_CALL_1(bpf_timer_cancel, struct bpf_async_kern *, async) {struct bpf_hrtimer *t, *cur_t;@@ -1513,27 +1574,16 @@ static const struct bpf_func_proto bpf_timer_cancel_proto = { .arg1_type = ARG_PTR_TO_TIMER, }; -static struct bpf_async_cb *__bpf_async_cancel_and_free (struct bpf_async_kern *async) +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async) {struct bpf_async_cb *cb; -/* Performance optimization: read async->cb without lock first. */ -if (!READ_ONCE (async->cb)) -return NULL ; - -__bpf_spin_lock_irqsave (&async->lock); -/* re-read it under lock */ - cb = async->cb; + cb =xchg (&async->cb,NULL );if (!cb) - goto out; -drop_prog_refcnt (cb); -/* The subsequent bpf_timer_start/cancel() helpers won't be able to use kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_cancel_proto = { · line 1586
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-* this timer, since it won't be initialized. -*/ -WRITE_ONCE (async->cb,NULL ); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return cb; +return ; + +/* Consume map's refcnt */ +irq_work_queue (&cb->worker);hm... this is subtle (and maybe broken?)
irq_work_queue()can be ignored here, if there is already another one scheduled, so I think your clever idea withCANCEL_AND_FREEbeing done based on this refcount drop is flawed...
Quoted 7 lines
CANCEL_AND_FREEhas to succeed, so it's out-of-bounds signal that shouldn't be going through that command cell, yes. But can't we just have a simple one-way bool that will be set to true here (+ memory barriers, maybe), and thenirq_work_queue()scheduled. If there is irq work is scheduled, it will inevitable will see this flag (even if it's not our callback), and if not, thenirq_work_queue()will successfully schedule callback which will also clean up.
Thanks for pointing to this issue.
I don't think we can solve it with a simple bool, because cleanup
should only happen once, and only after refcnt is 0, otherwise we need to go back to states to indicate cleanup initiation and cleanup entering (to implement mutual exclusion of the irq_work callbacks trying to run cleanup).
We can still solve this with the refcnt: Drop reference, if refcnt is not 0, we successfully released map reference and one of the scheduled irq_work callbacks will cleanup, otherwise we took the last reference, all we need is to schedule new irq_work (which can't fail)
if (!refcount_dec_and_test(&cb->refcnt))
return;
/* We took the last reference, need to schedule cleanup */ refcount_set(&cb->refcnt, 1); irq_work_queue(&cb->worker);
Quoted 134 lines
also, same about TODO for
irq_work_queue()avoidanceQuoted 130 lines
kernel/bpf/helpers.c +67
-24}static void bpf_timer_delete (struct bpf_hrtimer *t)@@ -1588,19 +1638,76 @@ static void bpf_timer_delete(struct bpf_hrtimer *t) } } +static void bpf_async_process_op (struct bpf_async_cb *cb,u32 op, +u64 timer_nsec,u32 timer_mode) +{ +switch (cb->type) { +case BPF_ASYNC_TYPE_TIMER : { + struct bpf_hrtimer *t = container_of(cb, struct bpf_hrtimer, cb); + +switch (op) { +case BPF_ASYNC_START : +hrtimer_start (&t->timer,ns_to_ktime (timer_nsec), timer_mode); +break ; +case BPF_ASYNC_CANCEL : +hrtimer_try_to_cancel (&t->timer); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +bpf_timer_delete (t); +break ; +default : +break ; + } +break ; + } +case BPF_ASYNC_TYPE_WQ : { + struct bpf_work *w = container_of(cb, struct bpf_work, cb); + +switch (op) { +case BPF_ASYNC_START : +schedule_work (&w->work); +break ; +case BPF_ASYNC_CANCEL_AND_FREE : +/* +* Trigger cancel of the sleepable work, but *do not* wait for +* it to finish. +* kfree will be called once the work has finished. +*/ +schedule_work (&w->delete_work); +break ; +default : +break ; + } +break ; + } + } +} + +static void bpf_async_irq_worker (struct irq_work *work) +{ + struct bpf_async_cb *cb = container_of(work, struct bpf_async_cb, worker); +u32 op, timer_mode; +u64 nsec; +int err; + + err =bpf_async_read_op (cb, &op, &nsec, &timer_mode); +if (err) + goto out; + +bpf_async_process_op (cb, op, nsec, timer_mode); + +out: +if (refcount_dec_and_test (&cb->refcnt)) +bpf_async_process_op (cb,BPF_ASYNC_CANCEL_AND_FREE ,0 ,0 ); +} +/* * This function is called by map_delete/update_elem for individual element and * by ops->map_release_uref when the user space reference to a map reaches zero. */ void bpf_timer_cancel_and_free (void *val) { -struct bpf_hrtimer *t; - - t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free (val); -if (!t) -return ; - -bpf_timer_delete (t); +__bpf_async_cancel_and_free (val); }/* This function is called by map_delete/update_elem for individual element and @@ -1608,19 +1715,7 @@ void bpf_timer_cancel_and_free(void *val)*/ void bpf_wq_cancel_and_free (void *val) { -struct bpf_work *work; - -BTF_TYPE_EMIT (struct bpf_wq ); - - work = (struct bpf_work *)__bpf_async_cancel_and_free (val); -if (!work) -return ; -/* Trigger cancel of the sleepable work, but *do not* wait for -* it to finish if it was running as we might not be in a -* sleepable context. -* kfree will be called once the work has finished. -*/ -schedule_work (&work->delete_work); +__bpf_async_cancel_and_free (val); } BPF_CALL_2(bpf_kptr_xchg, void *, dst, void *, ptr)@@ -3093,15 +3188,14 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)struct bpf_async_kern *async = (struct bpf_async_kern *)wq;struct bpf_work *w; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags)return -EINVAL ; w =READ_ONCE (async->work);if (!w || !READ_ONCE (w->cb.prog))return -EINVAL ; -schedule_work (&w->work); +bpf_async_schedule_op (&w->cb,BPF_ASYNC_START ,0 ,0 ); +return 0 ; } --2.52.0
Re: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
Quoted 30 lines · Mykyta Yatsenko
Quoted 28 lines · Andrii Nakryiko
Quoted 26 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to
irq_work.Previously,
bpf_timer_start(), andbpf_wq_start()
could not be called from NMI context because they acquiredbpf_spin_lockand called hrtimer/schedule_work APIs directly. This patch removes these limitations.Key changes:
* Removebpf_spin_lockfrom structbpf_async_kern. Replace locked operations with atomiccmpxchg()for initialization andxchg()for cancel and free.
* Add per-asyncirq_workto defer NMI-unsafe operations (hrtimer_start,hrtimer_try_to_cancel,schedule_work) from NMI to softirq context.
* Use the lock-freempmc_cell(added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to theirq_workhandler.
* Add reference counting tobpf_async_cbto ensure the object stays alive until all scheduledirq_workcompletes and the timer/work callback finishes.
* Movebpf_prog_put()to RCU callback to handle races betweenset_callback()andcancel_and_free().
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 288 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 191 insertions(+), 97 deletions(-)
please trim irrelevant parts to shorten distractions in email
[...]
Quoted 39 lines
Quoted 32 lines
Quoted 17 lines
kernel/bpf/helpers.c +2
-11-static struct bpf_async_cb *__bpf_async_cancel_and_free (struct bpf_async_kern *async) +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async) {struct bpf_async_cb *cb; -/* Performance optimization: read async->cb without lock first. */ -if (!READ_ONCE (async->cb)) -return NULL ; - -__bpf_spin_lock_irqsave (&async->lock); -/* re-read it under lock */ - cb = async->cb; + cb =xchg (&async->cb,NULL );if (!cb) - goto out; -drop_prog_refcnt (cb); -/* The subsequent bpf_timer_start/cancel() helpers won't be able to use kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_cancel_proto = { · line 1586
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-* this timer, since it won't be initialized. -*/ -WRITE_ONCE (async->cb,NULL ); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return cb; +return ; + +/* Consume map's refcnt */ +irq_work_queue (&cb->worker);hm... this is subtle (and maybe broken?)
irq_work_queue()can be ignored here, if there is already another one scheduled, so I think your clever idea withCANCEL_AND_FREEbeing done based on this refcount drop is flawed...Quoted 7 lines
CANCEL_AND_FREEhas to succeed, so it's out-of-bounds signal that shouldn't be going through that command cell, yes. But can't we just have a simple one-way bool that will be set to true here (+ memory barriers, maybe), and thenirq_work_queue()scheduled. If there is irq work is scheduled, it will inevitable will see this flag (even if it's not our callback), and if not, thenirq_work_queue()will successfully schedule callback which will also clean up.
Quoted 7 lines
Thanks for pointing to this issue.
I don't think we can solve it with a simple bool, because cleanup should only happen once, and only after refcnt is 0, otherwise we need to go
back to states to indicate cleanup initiation and cleanup entering (to implement
mutual exclusion of theirq_workcallbacks trying to run cleanup).
I meant bool as a command indicator of sorts, which is just part of the solution. We do have reader's mutual exclusion with last_seq, don't we? And the idea was that once the winning reader notices that shutdown was requested, they can perform it and make sure that subsequent readers (if there are any scheduled) would do nothing. E.g., by setting last_seq to some special large value to indicate "we are done, no more commands will be accepted".
But let me go and read the latest version and refresh what's going on there in my mind.
Quoted 14 lines
We can still solve this with the refcnt: Drop reference, if refcnt is not 0, we successfully released map reference and one of the scheduled
irq_workcallbacks
will cleanup, otherwise we took the last reference, all we need is to schedule newirq_work
(which can't fail)if (!refcount_dec_and_test(&cb->refcnt)) return; /* We took the last reference, need to schedule cleanup */ refcount_set(&cb->refcnt, 1); irq_work_queue(&cb->worker);
[...]
Re: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
Quoted 89 lines · Andrii Nakryiko
Quoted 29 lines · Mykyta Yatsenko
Quoted 28 lines · Andrii Nakryiko
Quoted 26 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to
irq_work.Previously,
bpf_timer_start(), andbpf_wq_start()
could not be called from NMI context because they acquiredbpf_spin_lockand called hrtimer/schedule_work APIs directly. This patch removes these limitations.Key changes:
* Removebpf_spin_lockfrom structbpf_async_kern. Replace locked operations with atomiccmpxchg()for initialization andxchg()for cancel and free.
* Add per-asyncirq_workto defer NMI-unsafe operations (hrtimer_start,hrtimer_try_to_cancel,schedule_work) from NMI to softirq context.
* Use the lock-freempmc_cell(added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to theirq_workhandler.
* Add reference counting tobpf_async_cbto ensure the object stays alive until all scheduledirq_workcompletes and the timer/work callback finishes.
* Movebpf_prog_put()to RCU callback to handle races betweenset_callback()andcancel_and_free().Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 288 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 191 insertions(+), 97 deletions(-)please trim irrelevant parts to shorten distractions in email
[...]
Quoted 39 lines
Quoted 32 lines
Quoted 17 lines
kernel/bpf/helpers.c +2
-11-static struct bpf_async_cb *__bpf_async_cancel_and_free (struct bpf_async_kern *async) +static void __bpf_async_cancel_and_free (struct bpf_async_kern *async) {struct bpf_async_cb *cb; -/* Performance optimization: read async->cb without lock first. */ -if (!READ_ONCE (async->cb)) -return NULL ; - -__bpf_spin_lock_irqsave (&async->lock); -/* re-read it under lock */ - cb = async->cb; + cb =xchg (&async->cb,NULL );if (!cb) - goto out; -drop_prog_refcnt (cb); -/* The subsequent bpf_timer_start/cancel() helpers won't be able to use kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_cancel_proto = { · line 1586
Quoted 10 lines
kernel/bpf/helpers.c +4
-6-* this timer, since it won't be initialized. -*/ -WRITE_ONCE (async->cb,NULL ); -out: -__bpf_spin_unlock_irqrestore (&async->lock); -return cb; +return ; + +/* Consume map's refcnt */ +irq_work_queue (&cb->worker);hm... this is subtle (and maybe broken?)
irq_work_queue()can be ignored here, if there is already another one scheduled, so I think your clever idea withCANCEL_AND_FREEbeing done based on this refcount drop is flawed...Quoted 7 lines
CANCEL_AND_FREEhas to succeed, so it's out-of-bounds signal that shouldn't be going through that command cell, yes. But can't we just have a simple one-way bool that will be set to true here (+ memory barriers, maybe), and thenirq_work_queue()scheduled. If there is irq work is scheduled, it will inevitable will see this flag (even if it's not our callback), and if not, thenirq_work_queue()will successfully schedule callback which will also clean up.Quoted 7 lines
Thanks for pointing to this issue.
I don't think we can solve it with a simple bool, because cleanup should only happen once, and only after refcnt is 0, otherwise we need to go
back to states to indicate cleanup initiation and cleanup entering (to implement
mutual exclusion of theirq_workcallbacks trying to run cleanup).
Quoted 7 lines
I meant bool as a command indicator of sorts, which is just part of the solution. We do have reader's mutual exclusion with
last_seq, don't we? And the idea was that once the winning reader notices that shutdown was requested, they can perform it and make sure that subsequent readers (if there are any scheduled) would do nothing. E.g., by settinglast_seqto some special large value to indicate "we are done, no more commands will be accepted".
Winning reader that runs cleanup can't tell other pending readers to not do anything, because after cleanup we can't guarantee that the bpf_async_cb is not freed, other readers have to access it, that's why we have refcnt that is the mechanism to make sure the last reader does cleanup.
Quoted 19 lines
But let me go and read the latest version and refresh what's going on there in my mind.
Quoted 14 lines
We can still solve this with the refcnt: Drop reference, if refcnt is not 0, we successfully released map reference and one of the scheduled
irq_workcallbacks
will cleanup, otherwise we took the last reference, all we need is to schedule newirq_work
(which can't fail)if (!refcount_dec_and_test(&cb->refcnt)) return; /* We took the last reference, need to schedule cleanup */ refcount_set(&cb->refcnt, 1); irq_work_queue(&cb->worker);[...]
Kumar Kartikeya Dwivedi
maintainerRe: [PATCH RFC v3 05/10] bpf: Enable bpf timer and workqueue use in NMI
In reply to Mykyta YatsenkoQuoted 47 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Refactor bpf timer and workqueue helpers to allow calling them from NMI context by making all operations lock-free and deferring NMI-unsafe work to
irq_work.Previously,
bpf_timer_start(), andbpf_wq_start()
could not be called from NMI context because they acquiredbpf_spin_lockand called hrtimer/schedule_work APIs directly. This patch removes these limitations.Key changes:
* Removebpf_spin_lockfrom structbpf_async_kern. Replace locked operations with atomiccmpxchg()for initialization andxchg()for cancel and free.
* Add per-asyncirq_workto defer NMI-unsafe operations (hrtimer_start,hrtimer_try_to_cancel,schedule_work) from NMI to softirq context.
* Use the lock-freempmc_cell(added in the previous commit) to pass operation commands (start/cancel/free) along with their parameters (nsec, mode) from NMI-safe callers to theirq_workhandler.
* Add reference counting tobpf_async_cbto ensure the object stays alive until all scheduledirq_workcompletes and the timer/work callback finishes.
* Movebpf_prog_put()to RCU callback to handle races betweenset_callback()andcancel_and_free().This enables BPF programs attached to NMI-context hooks (perf events) to use timers and workqueues for deferred processing.
Signed-off-by Mykyta Yatsenko <[email protected]>---
[...]kernel/bpf/helpers.c +2
-6-BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flags) +BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, async, u64, nsecs, u64, flags) {struct bpf_hrtimer *t; -int ret =0 ; -enum hrtimer_mode mode; +u32 mode; -if (in_nmi ()) -return -EOPNOTSUPP ;if (flags & ~(BPF_F_TIMER_ABS |BPF_F_TIMER_CPU_PIN ))return -EINVAL ; -__bpf_spin_lock_irqsave (&timer->lock);
kernel/bpf/helpers.c static const struct bpf_func_proto bpf_timer_set_callback_proto = { · line 1487
Quoted 10 lines
kernel/bpf/helpers.c +5
-5- t = timer->timer; -if (!t || !t->cb.prog) { - ret = -EINVAL ; - goto out; - } + +guard (rcu)(); + + t = async->timer; +if (!t || !t->cb.prog)
Read of t->cb.prog also needs READ_ONCE().
Quoted 16 lines
kernel/bpf/helpers.c +2
-4+return -EINVAL ;if (flags &BPF_F_TIMER_ABS ) mode =HRTIMER_MODE_ABS_SOFT ;@@ -1420,10 +1495,7 @@ BPF_CALL_3(bpf_timer_start, struct bpf_async_kern *, timer, u64, nsecs, u64, flaif (flags &BPF_F_TIMER_CPU_PIN ) mode |=HRTIMER_MODE_PINNED ; -hrtimer_start (&t->timer,ns_to_ktime (nsecs), mode); -out: -__bpf_spin_unlock_irqrestore (&timer->lock); -return ret; +return bpf_async_schedule_op (&t->cb,BPF_ASYNC_START , nsecs, mode); }
RFC PATCH v3 6/10 bpf: Add verifier support for bpf_timer argument in kfuncs
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Extend the verifier to recognize struct bpf_timer as a valid kfunc argument type. Previously, bpf_timer was only supported in BPF helpers.
This prepares for adding timer-related kfuncs in subsequent patches.
--- kernel/bpf/verifier.c | 59 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 6 deletions(-)
kernel/bpf/verifier.c +53 -6
@@ -8569,17 +8569,15 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno, }static int process_timer_func (struct bpf_verifier_env *env,int regno, -struct bpf_call_arg_meta *meta) +struct bpf_map *map) { -struct bpf_reg_state *regs =cur_regs (env), *reg = ®s[regno]; -struct bpf_map *map = reg->map_ptr;int err; err =check_map_field_pointer (env, regno,BPF_TIMER );if (err)return err; -if (meta->map_ptr) { +if (map) {verifier_bug (env,"Two map pointers in a timer helper" );return -EFAULT ; }
@@ -8587,8 +8585,36 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno,verbose (env,"bpf_timer cannot be used for PREEMPT_RT.\n" );return -EOPNOTSUPP ; } +return 0 ; +} + +static int process_timer_helper (struct bpf_verifier_env *env,int regno, +struct bpf_call_arg_meta *meta) +{ +struct bpf_reg_state *regs =cur_regs (env), *reg = ®s[regno]; +int err; + + err =process_timer_func (env, regno, meta->map_ptr); +if (err) +return err; + meta->map_uid = reg->map_uid; - meta->map_ptr = map; + meta->map_ptr = reg->map_ptr; +return 0 ; +} + +static int process_timer_kfunc (struct bpf_verifier_env *env,int regno, +struct bpf_kfunc_call_arg_meta *meta) +{ +struct bpf_reg_state *regs =cur_regs (env), *reg = ®s[regno]; +int err; + + err =process_timer_func (env, regno, meta->map.ptr); +if (err) +return err; + + meta->map.uid = reg->map_uid; + meta->map.ptr = reg->map_ptr;return 0 ; }
@@ -9911,7 +9937,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, }break ;case ARG_PTR_TO_TIMER : - err =process_timer_func (env, regno, meta); + err =process_timer_helper (env, regno, meta);if (err)return err;break ;
@@ -12164,6 +12190,7 @@ enum {KF_ARG_WORKQUEUE_ID ,KF_ARG_RES_SPIN_LOCK_ID ,KF_ARG_TASK_WORK_ID , +KF_ARG_TIMER_ID , };BTF_ID_LIST (kf_arg_btf_ids)
@@ -12175,6 +12202,7 @@ BTF_ID(struct, bpf_rb_node) BTF_ID(struct, bpf_wq) BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) +BTF_ID(struct, bpf_timer)static bool __is_kfunc_ptr_arg_type (const struct btf *btf,const struct btf_param *arg,int type)
@@ -12218,6 +12246,11 @@ static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_parreturn __is_kfunc_ptr_arg_type (btf, arg,KF_ARG_RB_NODE_ID ); } +static bool is_kfunc_arg_timer (const struct btf *btf,const struct btf_param *arg) +{ +return __is_kfunc_ptr_arg_type (btf, arg,KF_ARG_TIMER_ID ); +} +static bool is_kfunc_arg_wq (const struct btf *btf,const struct btf_param *arg) {return __is_kfunc_ptr_arg_type (btf, arg,KF_ARG_WORKQUEUE_ID );
@@ -12312,6 +12345,7 @@ enum kfunc_ptr_arg_type {KF_ARG_PTR_TO_NULL ,KF_ARG_PTR_TO_CONST_STR ,KF_ARG_PTR_TO_MAP , +KF_ARG_PTR_TO_TIMER ,KF_ARG_PTR_TO_WORKQUEUE ,KF_ARG_PTR_TO_IRQ_FLAG , KF_ARG_PTR_TO_RES_SPIN_LOCK,
@@ -12555,6 +12589,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,if (is_kfunc_arg_wq (meta->btf, &args[argno]))return KF_ARG_PTR_TO_WORKQUEUE ; +if (is_kfunc_arg_timer (meta->btf, &args[argno])) +return KF_ARG_PTR_TO_TIMER ; +if (is_kfunc_arg_task_work (meta->btf, &args[argno]))return KF_ARG_PTR_TO_TASK_WORK ;
@@ -13334,6 +13371,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_case KF_ARG_PTR_TO_REFCOUNTED_KPTR :case KF_ARG_PTR_TO_CONST_STR :case KF_ARG_PTR_TO_WORKQUEUE : +case KF_ARG_PTR_TO_TIMER :case KF_ARG_PTR_TO_TASK_WORK :case KF_ARG_PTR_TO_IRQ_FLAG :case KF_ARG_PTR_TO_RES_SPIN_LOCK :
@@ -13633,6 +13671,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_if (ret <0 )return ret;break ; +case KF_ARG_PTR_TO_TIMER : +if (reg->type !=PTR_TO_MAP_VALUE ) { +verbose (env,"arg#%d doesn't point to a map value\n" , i); +return -EINVAL ; + } + ret =process_timer_kfunc (env, regno, meta); +if (ret <0 ) +return ret; +break ;case KF_ARG_PTR_TO_TASK_WORK :if (reg->type !=PTR_TO_MAP_VALUE ) {verbose (env,"arg#%d doesn't point to a map value\n" , i);
Signature · 1 line
2.52.0
Re: [PATCH RFC v3 06/10] bpf: Add verifier support for bpf_timer argument in kfuncs
Quoted 6 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Extend the verifier to recognize struct
bpf_timeras a valid kfunc argument type. Previously,bpf_timerwas only supported in BPF helpers.
Quoted 7 lines
This prepares for adding timer-related kfuncs in subsequent patches.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/verifier.c | 59 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 6 deletions(-)
Looks reasonable.
[...]
RFC PATCH v3 7/10 bpf: Introduce bpf_timer_cancel_async() kfunc
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
introducing bpf timer cancel kfunc that attempts canceling timer asynchronously, hence, supports working in NMI context.
--- kernel/bpf/helpers.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
kernel/bpf/helpers.c +14 -0
@@ -4439,6 +4439,19 @@ __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)return 0 ; } +__bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer *timer) +{ +struct bpf_async_cb *cb; +struct bpf_async_kern *async = (void *)timer; + +guard (rcu)(); + cb = async->cb; +if (!cb) +return -EINVAL ; + +return bpf_async_schedule_op (cb,BPF_ASYNC_CANCEL ,0 ,0 ); +} +__bpf_kfunc_end_defs ();static void bpf_task_work_cancel_scheduled (struct irq_work *irq_work)
@@ -4620,6 +4633,7 @@ BTF_ID_FLAGS(func, bpf_task_work_schedule_signal_impl) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume_impl) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard) +BTF_ID_FLAGS(func, bpf_timer_cancel_async)BTF_KFUNCS_END (common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {
Signature · 1 line
2.52.0
Re: [PATCH RFC v3 07/10] bpf: Introduce bpf_timer_cancel_async() kfunc
Quoted 16 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
introducing bpf timer cancel kfunc that attempts canceling timer asynchronously, hence, supports working in NMI context.
Signed-off-by Mykyta Yatsenko <[email protected]>--- kernel/bpf/helpers.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)kernel/bpf/helpers.c +0
-0
kernel/bpf/helpers.c __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr) · line 4448
Quoted 10 lines
kernel/bpf/helpers.c +7
-0@@ -4439,6 +4439,19 @@ __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)return 0 ; } +__bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer *timer) +{ +struct bpf_async_cb *cb; +struct bpf_async_kern *async = (void *)timer; + +guard (rcu)(); + cb = async->cb;
READ_ONCE(), this cb can be xchg'ed into NULL, and we can't allow compiler to re-read the pointer
Quoted 21 lines
kernel/bpf/helpers.c +7
-1+if (!cb) +return -EINVAL ; + +return bpf_async_schedule_op (cb,BPF_ASYNC_CANCEL ,0 ,0 ); +} +__bpf_kfunc_end_defs ();static void bpf_task_work_cancel_scheduled (struct irq_work *irq_work)@@ -4620,6 +4633,7 @@ BTF_ID_FLAGS(func, bpf_task_work_schedule_signal_impl) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume_impl) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard) +BTF_ID_FLAGS(func, bpf_timer_cancel_async)BTF_KFUNCS_END (common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = { --2.52.0
RFC PATCH v3 8/10 selftests/bpf: Refactor timer selftests
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Refactor timer selftests, extracting stress test into a separate test. This makes it easier to debug test failures and allows to extend.
--- tools/testing/selftests/bpf/prog_tests/timer.c | 55 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-)
tools/testing/selftests/bpf/prog_tests/timer.c +36 -19
@@ -22,13 +22,35 @@ static void *spin_lock_thread(void *arg)pthread_exit (arg); } -static int timer (struct timer *timer_skel) + +static int timer_stress (struct timer *timer_skel) { -int i, err, prog_fd; +int i, err =1 , prog_fd;LIBBPF_OPTS (bpf_test_run_opts, topts);pthread_t thread_id[NUM_THR ];void *ret; + prog_fd =bpf_program__fd (timer_skel->progs.race); +for (i =0 ; i <NUM_THR ; i++) { + err =pthread_create (&thread_id[i],NULL , + &spin_lock_thread, &prog_fd); +if (!ASSERT_OK (err,"pthread_create" )) +break ; + } + +while (i) { + err =pthread_join (thread_id[--i], &ret); +if (ASSERT_OK (err,"pthread_join" )) +ASSERT_EQ (ret, (void *)&prog_fd,"pthread_join" ); + } +return err; +} + +static int timer (struct timer *timer_skel) +{ +int err, prog_fd; +LIBBPF_OPTS (bpf_test_run_opts, topts); + err =timer__attach (timer_skel);if (!ASSERT_OK (err,"timer_attach" ))return err;
@@ -63,25 +85,10 @@ static int timer(struct timer *timer_skel)/* check that code paths completed */ ASSERT_EQ (timer_skel->bss->ok,1 |2 |4 ,"ok" ); - prog_fd =bpf_program__fd (timer_skel->progs.race); -for (i =0 ; i <NUM_THR ; i++) { - err =pthread_create (&thread_id[i],NULL , - &spin_lock_thread, &prog_fd); -if (!ASSERT_OK (err,"pthread_create" )) -break ; - } - -while (i) { - err =pthread_join (thread_id[--i], &ret); -if (ASSERT_OK (err,"pthread_join" )) -ASSERT_EQ (ret, (void *)&prog_fd,"pthread_join" ); - } -return 0 ; } -/* TODO: use pid filtering */ -void serial_test_timer (void ) +static void test_timer (int (*timer_test_fn)(struct timer *timer_skel)) {struct timer *timer_skel =NULL ;int err;
@@ -94,13 +101,23 @@ void serial_test_timer(void)if (!ASSERT_OK_PTR (timer_skel,"timer_skel_load" ))return ; - err =timer (timer_skel); + err =timer_test_fn (timer_skel);ASSERT_OK (err,"timer" );timer__destroy (timer_skel); +} + +void serial_test_timer (void ) +{ +test_timer (timer);RUN_TESTS (timer_failure); } +void serial_test_timer_stress (void ) +{ +test_timer (timer_stress); +} +void test_timer_interrupt (void ) {struct timer_interrupt *skel =NULL ;
Signature · 1 line
2.52.0
RFC PATCH v3 9/10 selftests/bpf: Add stress test for timer async cancel
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Extend BPF timer selftest to run stress test for async cancel.
--- tools/testing/selftests/bpf/prog_tests/timer.c | 18 +++++++++++++++++- tools/testing/selftests/bpf/progs/timer.c | 14 +++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-)
tools/testing/selftests/bpf/prog_tests/timer.c +17 -1
@@ -23,13 +23,14 @@ static void *spin_lock_thread(void *arg) } -static int timer_stress (struct timer *timer_skel) +static int timer_stress_runner (struct timer *timer_skel,bool async_cancel) {int i, err =1 , prog_fd;LIBBPF_OPTS (bpf_test_run_opts, topts);pthread_t thread_id[NUM_THR ];void *ret; + timer_skel->bss->async_cancel = async_cancel; prog_fd =bpf_program__fd (timer_skel->progs.race);for (i =0 ; i <NUM_THR ; i++) { err = pthread_create(&thread_id[i], NULL,
@@ -46,6 +47,16 @@ static int timer_stress(struct timer *timer_skel)return err; } +static int timer_stress (struct timer *timer_skel) +{ +return timer_stress_runner (timer_skel, false); +} + +static int timer_stress_async_cancel (struct timer *timer_skel) +{ +return timer_stress_runner (timer_skel, true); +} +static int timer (struct timer *timer_skel) {int err, prog_fd;
@@ -118,6 +129,11 @@ void serial_test_timer_stress(void)test_timer (timer_stress); } +void serial_test_timer_stress_async_cancel (void ) +{ +test_timer (timer_stress_async_cancel); +} +void test_timer_interrupt (void ) {struct timer_interrupt *skel =NULL ;
tools/testing/selftests/bpf/progs/timer.c +11 -3
@@ -1,13 +1,17 @@// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2021 Facebook */ -#include <linux/bpf.h> -#include <time.h> + +#include <vmlinux.h> #include <stdbool.h> #include <errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +#define CLOCK_MONOTONIC 1 +#define CLOCK_BOOTTIME 7 + char _license[] SEC("license") = "GPL"; +struct hmap_elem {int counter;struct bpf_timer timer;
@@ -63,6 +67,7 @@ __u64 callback_check = 52;__u64 callback2_check =52 ;__u64 pinned_callback_check;__s32 pinned_cpu; +bool async_cancel =0 ;#define ARRAY 1#define HTAB 2
@@ -419,7 +424,10 @@ int race(void *ctx)bpf_timer_set_callback (timer, race_timer_callback);bpf_timer_start (timer,0 ,0 ); -bpf_timer_cancel (timer); +if (async_cancel) +bpf_timer_cancel_async (timer); +else +bpf_timer_cancel (timer);return 0 ; }
Signature · 1 line
2.52.0
RFC PATCH v3 10/10 selftests/bpf: Verify bpf_timer_cancel_async works
In reply to Mykyta YatsenkoFrom: Mykyta Yatsenko <[email protected]>
Add test that verifies that bpf_timer_cancel_async works: can cancel callback successfully.
--- tools/testing/selftests/bpf/prog_tests/timer.c | 25 +++++++++++++++++++++++++ tools/testing/selftests/bpf/progs/timer.c | 23 +++++++++++++++++++++++ 2 files changed, 48 insertions(+)
tools/testing/selftests/bpf/prog_tests/timer.c +25 -0
@@ -99,6 +99,26 @@ static int timer(struct timer *timer_skel)return 0 ; } +static int timer_cancel_async (struct timer *timer_skel) +{ +int err, prog_fd; +LIBBPF_OPTS (bpf_test_run_opts, topts); + + prog_fd =bpf_program__fd (timer_skel->progs.test_async_cancel_succeed); + err =bpf_prog_test_run_opts (prog_fd, &topts); +ASSERT_OK (err,"test_run" ); +ASSERT_EQ (topts.retval,0 ,"test_run" ); + +usleep (500 ); +/* check that there were no errors in timer execution */ +ASSERT_EQ (timer_skel->bss->err,0 ,"err" ); + +/* check that code paths completed */ +ASSERT_EQ (timer_skel->bss->ok,1 |2 |4 ,"ok" ); + +return 0 ; +} +static void test_timer (int (*timer_test_fn)(struct timer *timer_skel)) {struct timer *timer_skel =NULL ;
@@ -134,6 +154,11 @@ void serial_test_timer_stress_async_cancel(void)test_timer (timer_stress_async_cancel); } +void serial_test_timer_async_cancel (void ) +{ +test_timer (timer_cancel_async); +} +void test_timer_interrupt (void ) {struct timer_interrupt *skel =NULL ;
tools/testing/selftests/bpf/progs/timer.c +23 -0
@@ -169,6 +169,29 @@ int BPF_PROG2(test1, int, a)return 0 ; } +static int timer_error (void *map,int *key,struct bpf_timer *timer) +{ + err =42 ; +return 0 ; +} + +SEC("syscall") +int test_async_cancel_succeed (void *ctx) +{ +struct bpf_timer *arr_timer; +int array_key =ARRAY ; + + arr_timer =bpf_map_lookup_elem (&array, &array_key); +if (!arr_timer) +return 0 ; +bpf_timer_init (arr_timer, &array,CLOCK_MONOTONIC ); +bpf_timer_set_callback (arr_timer, timer_error); +bpf_timer_start (arr_timer,100000 /* 100us */ ,0 ); +bpf_timer_cancel_async (arr_timer); + ok =7 ; +return 0 ; +} +/* callback for prealloc and non-prealloca hashtab timers */ static int timer_cb2 (void *map,int *key,struct hmap_elem *val) {
Signature · 1 line
2.52.0
Re: [PATCH RFC v3 10/10] selftests/bpf: Verify bpf_timer_cancel_async works
Quoted 71 lines · Mykyta Yatsenko
From: Mykyta Yatsenko <[email protected]>
Add test that verifies that
bpf_timer_cancel_asyncworks: can cancel callback successfully.Signed-off-by Mykyta Yatsenko <[email protected]>--- tools/testing/selftests/bpf/prog_tests/timer.c | 25 +++++++++++++++++++++++++ tools/testing/selftests/bpf/progs/timer.c | 23 +++++++++++++++++++++++ 2 files changed, 48 insertions(+)tools/testing/selftests/bpf/prog_tests/timer.c +25
-0@@ -99,6 +99,26 @@ static int timer(struct timer *timer_skel)return 0 ; } +static int timer_cancel_async (struct timer *timer_skel) +{ +int err, prog_fd; +LIBBPF_OPTS (bpf_test_run_opts, topts); + + prog_fd =bpf_program__fd (timer_skel->progs.test_async_cancel_succeed); + err =bpf_prog_test_run_opts (prog_fd, &topts); +ASSERT_OK (err,"test_run" ); +ASSERT_EQ (topts.retval,0 ,"test_run" ); + +usleep (500 ); +/* check that there were no errors in timer execution */ +ASSERT_EQ (timer_skel->bss->err,0 ,"err" ); + +/* check that code paths completed */ +ASSERT_EQ (timer_skel->bss->ok,1 |2 |4 ,"ok" ); + +return 0 ; +} +static void test_timer (int (*timer_test_fn)(struct timer *timer_skel)) {struct timer *timer_skel =NULL ;@@ -134,6 +154,11 @@ void serial_test_timer_stress_async_cancel(void)test_timer (timer_stress_async_cancel); } +void serial_test_timer_async_cancel (void ) +{ +test_timer (timer_cancel_async); +} +void test_timer_interrupt (void ) {struct timer_interrupt *skel =NULL ;tools/testing/selftests/bpf/progs/timer.c +8
-0@@ -169,6 +169,29 @@ int BPF_PROG2(test1, int, a)return 0 ; } +static int timer_error (void *map,int *key,struct bpf_timer *timer) +{ + err =42 ; +return 0 ; +} + +SEC("syscall") +int test_async_cancel_succeed (void *ctx)
tools/testing/selftests/bpf/progs/timer.c int BPF_PROG2(test1, int, a) · line 189
Quoted 10 lines
tools/testing/selftests/bpf/progs/timer.c +10
-0+{ +struct bpf_timer *arr_timer; +int array_key =ARRAY ; + + arr_timer =bpf_map_lookup_elem (&array, &array_key); +if (!arr_timer) +return 0 ; +bpf_timer_init (arr_timer, &array,CLOCK_MONOTONIC ); +bpf_timer_set_callback (arr_timer, timer_error); +bpf_timer_start (arr_timer,100000 /* 100us */ ,0 );
flaky timing-dependent test... not sure how to write a better one, but this one is not great :(
Quoted 12 lines
tools/testing/selftests/bpf/progs/timer.c +5
-1+bpf_timer_cancel_async (arr_timer); + ok =7 ; +return 0 ; +} +/* callback for prealloc and non-prealloca hashtab timers */ static int timer_cb2 (void *map,int *key,struct hmap_elem *val) { --2.52.0