Fix sleepable context tracking for async callbacks

Rename the storage_get_func_atomic flag to a more generic non_sleepable flag that tracks whether a helper or kfunc may be called from a non-sleepable context. This makes the flag more broadly applicable beyond just storage_get helpers. See [0] for more context.

The flag is now set unconditionally for all helpers and kfuncs when:
- RCU critical section is active.
- Preemption is disabled.
- IRQs are disabled.
- In a non-sleepable context within a sleepable program (e.g., timer callbacks), which is indicated by !in_sleepable().

Previously, the flag was only set for storage_get helpers in these contexts. With this change, it can be used by any code that needs to differentiate between sleepable and non-sleepable contexts at the per-instruction level.

The existing usage in do_misc_fixups() for storage_get helpers is preserved by checking is_storage_get_function() before using the flag.

[0]: https://lore.kernel.org/bpf/CAP01T76cbaNi4p-y8E0sjE2NXSra2S=Uja8G4hSQDu_SbXxREQ@mail.gmail.com

Cc Mykyta Yatsenko <[email protected]>

Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>

---

Rename the storage_get_func_atomic flag to a more generic non_sleepable flag that tracks whether a helper or kfunc may be called from a non-sleepable context. This makes the flag more broadly applicable beyond just storage_get helpers. See [0] for more context.

The flag is now set unconditionally for all helpers and kfuncs when:
- RCU critical section is active.
- Preemption is disabled.
- IRQs are disabled.
- In a non-sleepable context within a sleepable program (e.g., timer callbacks), which is indicated by !in_sleepable().

Previously, the flag was only set for storage_get helpers in these contexts. With this change, it can be used by any code that needs to differentiate between sleepable and non-sleepable contexts at the per-instruction level.

The existing usage in do_misc_fixups() for storage_get helpers is preserved by checking is_storage_get_function() before using the flag.

   [0]: https://lore.kernel.org/bpf/CAP01T76cbaNi4p-y8E0sjE2NXSra2S=Uja8G4hSQDu_SbXxREQ@mail.gmail.com

Cc Mykyta Yatsenko <[email protected]>

Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>

---
  include/linux/bpf_verifier.h |  2 +-
  kernel/bpf/verifier.c        | 33 +++++++++++++++++----------------
  2 files changed, 18 insertions(+), 17 deletions(-)

Add tests to verify that async callback's sleepable attribute is correctly determined by the callback type, not the arming program's context, reflecting its true execution context.

Introduce verifier_async_cb_context.c with tests for all three async callback primitives: bpf_timer, bpf_wq, and bpf_task_work. Each primitive is tested when armed from both sleepable (lsm.s/file_open) and non-sleepable (fentry) programs.

Test coverage:
- bpf_timer callbacks: Verify they are never sleepable, even when armed from sleepable programs. Both tests should fail when attempting to use sleepable helper bpf_copy_from_user() in the callback.

- bpf_wq callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

- bpf_task_work callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

Acked-by Eduard Zingerman <[email protected]>

Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>

---
 .../selftests/bpf/prog_tests/verifier.c       |   2 +
 .../bpf/progs/verifier_async_cb_context.c     | 181 ++++++++++++++++++
 2 files changed, 183 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/verifier_async_cb_context.c

tools/testing/selftests/bpf/prog_tests/verifier.c +2 -0

index 28e81161e6fc..c0e8ffdaa484 100644
@@ -7,6 +7,7 @@
 #include "verifier_arena.skel.h"
 #include "verifier_arena_large.skel.h"
 #include "verifier_array_access.skel.h"
+#include "verifier_async_cb_context.skel.h"
 #include "verifier_basic_stack.skel.h"
 #include "verifier_bitfield_write.skel.h"
 #include "verifier_bounds.skel.h"

@@ -280,6 +281,7 @@ void test_verifier_array_access(void)
                      verifier_array_access__elf_bytes,
                      init_array_access_maps);
 }
+void test_verifier_async_cb_context(void)    { RUN(verifier_async_cb_context); }
 
 static int init_value_ptr_arith_maps(struct bpf_object *obj)
 {

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c new file +53 -0

new file mode 100644
index 000000000000..96ff6749168b
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+
+char _license[] SEC("license") = "GPL";
+
+/* Timer tests */
+
+struct timer_elem {
+       struct bpf_timer t;
+};
+
+struct {
+       __uint(type, BPF_MAP_TYPE_ARRAY);
+       __uint(max_entries, 1);
+       __type(key, int);
+       __type(value, struct timer_elem);
+} timer_map SEC(".maps");
+
+static int timer_cb(void *map, int *key, struct bpf_timer *timer)
+{
+       u32 data;
+       /* Timer callbacks are never sleepable, even from non-sleepable programs */
+       bpf_copy_from_user(&data, sizeof(data), NULL);
+       return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_non_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;
+       int key = 0;
+
+       val = bpf_map_lookup_elem(&timer_map, &key);
+       if (!val)
+               return 0;
+
+       bpf_timer_init(&val->t, &timer_map, 0);
+       bpf_timer_set_callback(&val->t, timer_cb);
+       return 0;
+}
+
+SEC("lsm.s/file_open")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;

Quoted 110 lines · Kumar Kartikeya Dwivedi

Add tests to verify that async callback's sleepable attribute is correctly determined by the callback type, not the arming program's context, reflecting its true execution context.

Introduce verifier_async_cb_context.c with tests for all three async callback primitives: bpf_timer, bpf_wq, and bpf_task_work. Each primitive is tested when armed from both sleepable (lsm.s/file_open) and non-sleepable (fentry) programs.

Test coverage:
- bpf_timer callbacks: Verify they are never sleepable, even when armed from sleepable programs. Both tests should fail when attempting to use sleepable helper bpf_copy_from_user() in the callback.

- bpf_wq callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

- bpf_task_work callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

Acked-by Eduard Zingerman <[email protected]>

Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>

---
 .../selftests/bpf/prog_tests/verifier.c       |   2 +
 .../bpf/progs/verifier_async_cb_context.c     | 181 ++++++++++++++++++
 2 files changed, 183 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/verifier_async_cb_context.c

tools/testing/selftests/bpf/prog_tests/verifier.c +2 -0

index 28e81161e6fc..c0e8ffdaa484 100644
@@ -7,6 +7,7 @@
 #include "verifier_arena.skel.h"
 #include "verifier_arena_large.skel.h"
 #include "verifier_array_access.skel.h"
+#include "verifier_async_cb_context.skel.h"
 #include "verifier_basic_stack.skel.h"
 #include "verifier_bitfield_write.skel.h"
 #include "verifier_bounds.skel.h"

@@ -280,6 +281,7 @@ void test_verifier_array_access(void)
                      verifier_array_access__elf_bytes,
                      init_array_access_maps);
 }
+void test_verifier_async_cb_context(void)    { RUN(verifier_async_cb_context); }
 
 static int init_value_ptr_arith_maps(struct bpf_object *obj)
 {

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c new file +53 -0

new file mode 100644
index 000000000000..96ff6749168b
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+
+char _license[] SEC("license") = "GPL";
+
+/* Timer tests */
+
+struct timer_elem {
+       struct bpf_timer t;
+};
+
+struct {
+       __uint(type, BPF_MAP_TYPE_ARRAY);
+       __uint(max_entries, 1);
+       __type(key, int);
+       __type(value, struct timer_elem);
+} timer_map SEC(".maps");
+
+static int timer_cb(void *map, int *key, struct bpf_timer *timer)
+{
+       u32 data;
+       /* Timer callbacks are never sleepable, even from non-sleepable programs */
+       bpf_copy_from_user(&data, sizeof(data), NULL);
+       return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_non_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;
+       int key = 0;
+
+       val = bpf_map_lookup_elem(&timer_map, &key);
+       if (!val)
+               return 0;
+
+       bpf_timer_init(&val->t, &timer_map, 0);
+       bpf_timer_set_callback(&val->t, timer_cb);
+       return 0;
+}
+
+SEC("lsm.s/file_open")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c line 63

Quoted 10 lines

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c new file +10 -0

+       int key = 0;
+
+       val = bpf_map_lookup_elem(&timer_map, &key);
+       if (!val)
+               return 0;
+
+       bpf_timer_init(&val->t, &timer_map, 0);
+       bpf_timer_set_callback(&val->t, timer_cb);
+       return 0;
+}

can you please also add (as a follow up is fine) this case:

static noinline int mixed_mode_subprog(void)

{
    /* use one of is_storage_get_function() helpers here */
}

static int timer_cb(void *map, int *key, struct bpf_timer *timer)

{
    mixed_mode_subprog();
    return 0;
}

SEC("lsm.s/file_open") /* sleepable entry program */

int sleepable(void *ctx) {
    ...
    bpf_timer_set_callback(&val->t, timer_cb);

    /* !!! important to call it here */
    mixed_mode_subprog();
}

Idea being that we have a subprog that is called in both sleepable and non-sleepable modes. It should be rejected.

Let's do the same the other way, when the entry program is non-sleepable but an async callback is sleepable, ok?

Note also that noinline part is important, we need to make sure it doesn't get inline anywhere. If necessary, we can do __weak __hidden to make this happen.

[...]

Quoted 148 lines · Andrii Nakryiko

Quoted 110 lines · Kumar Kartikeya Dwivedi

Add tests to verify that async callback's sleepable attribute is correctly determined by the callback type, not the arming program's context, reflecting its true execution context.

Introduce verifier_async_cb_context.c with tests for all three async callback primitives: bpf_timer, bpf_wq, and bpf_task_work. Each primitive is tested when armed from both sleepable (lsm.s/file_open) and non-sleepable (fentry) programs.

Test coverage:
- bpf_timer callbacks: Verify they are never sleepable, even when armed from sleepable programs. Both tests should fail when attempting to use sleepable helper bpf_copy_from_user() in the callback.

- bpf_wq callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

- bpf_task_work callbacks: Verify they are always sleepable, even when armed from non-sleepable programs. Both tests should succeed when using sleepable helpers in the callback.

Acked-by Eduard Zingerman <[email protected]>

Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>

---
 .../selftests/bpf/prog_tests/verifier.c       |   2 +
 .../bpf/progs/verifier_async_cb_context.c     | 181 ++++++++++++++++++
 2 files changed, 183 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/verifier_async_cb_context.c

tools/testing/selftests/bpf/prog_tests/verifier.c +2 -0

index 28e81161e6fc..c0e8ffdaa484 100644
@@ -7,6 +7,7 @@
 #include "verifier_arena.skel.h"
 #include "verifier_arena_large.skel.h"
 #include "verifier_array_access.skel.h"
+#include "verifier_async_cb_context.skel.h"
 #include "verifier_basic_stack.skel.h"
 #include "verifier_bitfield_write.skel.h"
 #include "verifier_bounds.skel.h"

@@ -280,6 +281,7 @@ void test_verifier_array_access(void)
                      verifier_array_access__elf_bytes,
                      init_array_access_maps);
 }
+void test_verifier_async_cb_context(void)    { RUN(verifier_async_cb_context); }
 
 static int init_value_ptr_arith_maps(struct bpf_object *obj)
 {

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c new file +53 -0

new file mode 100644
index 000000000000..96ff6749168b
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+
+char _license[] SEC("license") = "GPL";
+
+/* Timer tests */
+
+struct timer_elem {
+       struct bpf_timer t;
+};
+
+struct {
+       __uint(type, BPF_MAP_TYPE_ARRAY);
+       __uint(max_entries, 1);
+       __type(key, int);
+       __type(value, struct timer_elem);
+} timer_map SEC(".maps");
+
+static int timer_cb(void *map, int *key, struct bpf_timer *timer)
+{
+       u32 data;
+       /* Timer callbacks are never sleepable, even from non-sleepable programs */
+       bpf_copy_from_user(&data, sizeof(data), NULL);
+       return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_non_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;
+       int key = 0;
+
+       val = bpf_map_lookup_elem(&timer_map, &key);
+       if (!val)
+               return 0;
+
+       bpf_timer_init(&val->t, &timer_map, 0);
+       bpf_timer_set_callback(&val->t, timer_cb);
+       return 0;
+}
+
+SEC("lsm.s/file_open")
+__failure __msg("helper call might sleep in a non-sleepable prog")
+int timer_sleepable_prog(void *ctx)
+{
+       struct timer_elem *val;

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c line 63

Quoted 10 lines

tools/testing/selftests/bpf/progs/verifier_async_cb_context.c new file +10 -0

+       int key = 0;
+
+       val = bpf_map_lookup_elem(&timer_map, &key);
+       if (!val)
+               return 0;
+
+       bpf_timer_init(&val->t, &timer_map, 0);
+       bpf_timer_set_callback(&val->t, timer_cb);
+       return 0;
+}

can you please also add (as a follow up is fine) this case:

static noinline int mixed_mode_subprog(void)

{
    /* use one of is_storage_get_function() helpers here */
}

static int timer_cb(void *map, int *key, struct bpf_timer *timer)

{
    mixed_mode_subprog();
    return 0;
}

SEC("lsm.s/file_open") /* sleepable entry program */

int sleepable(void *ctx) {
    ...
    bpf_timer_set_callback(&val->t, timer_cb);

    /* !!! important to call it here */
    mixed_mode_subprog();
}

Idea being that we have a subprog that is called in both sleepable and non-sleepable modes. It should be rejected.

Quoted 7 lines

Let's do the same the other way, when the entry program is non-sleepable but an async callback is sleepable, ok?

Note also that noinline part is important, we need to make sure it doesn't get inline anywhere. If necessary, we can do __weak __hidden to make this happen.

I didn't get why it needs to be rejected.
That seems like something that is bound to trip up users. Shouldn't we instead allow this, but choose the most restrictive context when setting any necessary flags etc.?
E.g. if it's both sleepable and non-sleepable, we should simply always do GFP_ATOMIC.

Quoted 2 lines

[...]