Skip to contentSkip to conversation

bpf: expand BPF arena kernel fault handling

PATCH New Emil Tsalapatis 2 participants 4 messages Nov 9 – Nov 12

Subsystems (4)
lore
Share & tools
lore

Emil Tsalapatis

author reviewer
#0 lore raw

v2 cover bpf: expand BPF arena kernel fault handling

Currently, the BPF arena fault handler protects against a misbehaving BPF program passing an arena pointer to a kfunc then unmapping the backing pages, causing a kernel fault. However, it does not handle errors where the now unmapped arena address is instead accessed by a non-BPF path, or by a BPF program that has a different arena. The former case is already possible in sched-ext if a BPF program removes from the arena pages allocated from the genpool allocator.

Keep a list of all arenas in the system and attempt to match the kernel fault to all of them. If the fault does fall in an arena, handle it with the already established scratch page technique and retry the access.

Changes from v1 to v2:
- Replace the global arena list with a latch rbtree.
- Restore the stack lookup fast path, checking the fault address before falling back to the tree lookup.
- Defer arena teardown with RCU work instead of synchronize_rcu().
- Add rate-limited deferred logging for faults outside the owning BPF program.
- Reuse bpf_testmod for the selftests instead of adding a module.
- Skip the selftests when the compiler lacks arena address-space casts.
- Correct the Fixes tag to reference the introduction of kernel arena accesses.

Emil Tsalapatis (2):
bpf: recover page faults across all arenas selftests/bpf: add arena fault recovery tests

 kernel/bpf/arena.c                            | 119 +++++++++++++-----
 .../selftests/bpf/prog_tests/arena_fault.c    | 100 +++++++++++++++
 .../testing/selftests/bpf/progs/arena_fault.c |  65 ++++++++++
 .../selftests/bpf/progs/arena_fault_noarena.c |  17 +++
 .../selftests/bpf/test_kmods/bpf_testmod.c    |  22 ++++
 .../bpf/test_kmods/bpf_testmod_kfunc.h        |   3 +
 6 files changed, 298 insertions(+), 28 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_fault.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_fault.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_fault_noarena.c
Signature · 1 line
2.52.0
author reviewer
#1 lore raw

PATCH v2 1/2 bpf: recover page faults across all arenas

The current arena fault handling code accounts for misbehaving BPF programs that free arena memory while it is being accessed by a kfunc as part of another BPF program's execution. This scenario triggers a kernel page fault that is intercepted by BPF arena-specific code. The code confirms that the fault is within the BPF program's arena, installs a scratch page, and marks the fault as handled. This is necessary to prevent BPF programs from being able to trigger an oops.

However, the protection against misbehaving BPF programs is currently incomplete: Arena memory is accessed by kernel code not just as part of a kfunc, but also from non-BPF paths. This is the case with sched-ext that includes non-BPF code paths that access arena pointers. These paths do not have an arena associated with them, and a fault there causes an oops. Such a fault can be triggered by a misbehaving BPF program by unmapping the pages nominally owned by the genpool allocator used for cid management.

Extend the kernel arena fault handling to consider all arenas in the system. The handler now scans a global arena tree till it finds an arena that includes the address, and fixes up the address. Retain the stream-based reporting code for when the faulting call stack includes the BPF program that owns the arena.

Fixes abdc2516f100 ("sched_ext: Convert ops.set_cmask() to arena-resident cmask")
Cc Tejun Heo <[email protected]>
---
 kernel/bpf/arena.c | 119 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 91 insertions(+), 28 deletions(-)
kernel/bpf/arena.c +91 -28
index 366c4c2b6a41..aff231922e11 100644
@@ -11,6 +11,9 @@
 #include <linux/btf_ids.h>
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/rbtree_latch.h>
 #include <asm/tlbflush.h>
 #include "range_tree.h"
 
@@ -47,10 +50,15 @@
 #define GUARD_SZ round_up(1ull << sizeof_field(struct bpf_insn, off) * 8, PAGE_SIZE << 1)
 #define KERN_VM_SZ (SZ_4G + GUARD_SZ)
 
+static struct latch_tree_root arena_tree;
+static DEFINE_SPINLOCK(arena_tree_lock);
+
 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable);
 
 struct bpf_arena {
 	struct bpf_map map;
+	struct latch_tree_node tnode;
+	struct rcu_work destroy_work;
 	u64 user_vm_start;
 	u64 user_vm_end;
 	struct vm_struct *kern_vm;
@@ -142,6 +150,33 @@ struct bpf_map *bpf_prog_arena(struct bpf_prog *prog)
 	return arena ? &arena->map : NULL;
 }
 
+static __always_inline bool arena_tree_less(struct latch_tree_node *a,
+					   struct latch_tree_node *b)
+{
+	struct bpf_arena *arena_a = container_of(a, struct bpf_arena, tnode);
+	struct bpf_arena *arena_b = container_of(b, struct bpf_arena, tnode);
+
+	return bpf_arena_get_kern_vm_start(arena_a) < bpf_arena_get_kern_vm_start(arena_b);
+}
+
+static __always_inline int arena_tree_comp(void *key, struct latch_tree_node *node)
+{
+	struct bpf_arena *arena = container_of(node, struct bpf_arena, tnode);
+	unsigned long kbase = bpf_arena_get_kern_vm_start(arena);
+	unsigned long addr = (unsigned long)key;
+
+	if (addr < kbase)
+		return -1;
+	if (addr >= kbase + SZ_4G + GUARD_SZ / 2)
+		return 1;
+	return 0;
+}
+
+static const struct latch_tree_ops arena_tree_ops = {
+	.less = arena_tree_less,
+	.comp = arena_tree_comp,
+};
+
 static long arena_map_peek_elem(struct bpf_map *map, void *value)
 {
 	return -EOPNOTSUPP;
@@ -360,6 +395,10 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
 	if (err)
 		goto err_destroy_rt;
 
+	spin_lock(&arena_tree_lock);
+	latch_tree_insert(&arena->tnode, &arena_tree, &arena_tree_ops);
+	spin_unlock(&arena_tree_lock);
+
 	return &arena->map;
 
 err_destroy_rt:
@@ -400,6 +439,26 @@ static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data)
 	return 0;
 }
 
+static void arena_map_destroy(struct work_struct *work)
+{
+	struct bpf_arena *arena = container_of(to_rcu_work(work), struct bpf_arena,
+					     destroy_work);
+
+	/*
+	 * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area().
+	 * It unmaps everything from vmalloc area and clears pgtables.
+	 * Call apply_to_existing_page_range() first to find populated ptes and
+	 * free those pages.
+	 */
+	apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),
+				     SZ_4G + GUARD_SZ / 2, existing_page_cb, arena);
+	free_vm_area(arena->kern_vm);
+	range_tree_destroy(&arena->rt);
+	__free_page(arena->scratch_page);
+	free_percpu(arena->stacks);
+	bpf_map_area_free(arena);
+}
+
 static void arena_map_free(struct bpf_map *map)
 {
 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
@@ -413,23 +472,16 @@ static void arena_map_free(struct bpf_map *map)
 	if (WARN_ON_ONCE(!list_empty(&arena->vma_list)))
 		return;
 
+	spin_lock(&arena_tree_lock);
+	latch_tree_erase(&arena->tnode, &arena_tree, &arena_tree_ops);
+	spin_unlock(&arena_tree_lock);
+
 	/* Ensure no pending deferred frees */
 	irq_work_sync(&arena->free_irq);
 	flush_work(&arena->free_work);
 
-	/*
-	 * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area().
-	 * It unmaps everything from vmalloc area and clears pgtables.
-	 * Call apply_to_existing_page_range() first to find populated ptes and
-	 * free those pages.
-	 */
-	apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),
-				     SZ_4G + GUARD_SZ / 2, existing_page_cb, arena);
-	free_vm_area(arena->kern_vm);
-	range_tree_destroy(&arena->rt);
-	__free_page(arena->scratch_page);
-	free_percpu(arena->stacks);
-	bpf_map_area_free(arena);
+	INIT_RCU_WORK(&arena->destroy_work, arena_map_destroy);
+	queue_rcu_work(system_dfl_wq, &arena->destroy_work);
 }
 
 static void *arena_map_lookup_elem(struct bpf_map *map, void *key)
@@ -1357,35 +1409,46 @@ static void __bpf_prog_report_arena_violation(struct bpf_prog *prog, bool write,
 
 bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip)
 {
+	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
+				     DEFAULT_RATELIMIT_BURST);
+	struct latch_tree_node *node;
 	struct bpf_arena *arena;
 	struct bpf_prog *prog;
 	unsigned long kbase;
 	unsigned long page_addr = addr & PAGE_MASK;
-
-	prog = bpf_prog_find_from_stack();
-	if (!prog)
-		return false;
-
-	arena = prog->aux->arena;
-	/* a prog not using arena may be on stack, so arena can be NULL */
-	if (!arena)
-		return false;
-
-	kbase = bpf_arena_get_kern_vm_start(arena);
+	bool handled = false;
 
 	/*
 	 * Recovery covers the 4 GiB mappable band plus the upper half-guard.
 	 * Lower guard is unreachable from kfuncs; an address there indicates
 	 * a different bug class - leave it to the regular kernel oops path.
 	 */
-	if (page_addr < kbase || page_addr >= kbase + SZ_4G + GUARD_SZ / 2)
-		return false;
+	rcu_read_lock();
+	prog = bpf_prog_find_from_stack();
+	arena = prog ? prog->aux->arena : NULL;
+	if (!arena || arena_tree_comp((void *)page_addr, &arena->tnode)) {
+		node = latch_tree_find((void *)page_addr, &arena_tree, &arena_tree_ops);
+		if (!node)
+			goto out;
+		arena = container_of(node, struct bpf_arena, tnode);
+	}
+	kbase = bpf_arena_get_kern_vm_start(arena);
 
 	apply_to_page_range(&init_mm, page_addr, PAGE_SIZE,
 			    apply_range_set_scratch_cb, arena->scratch_page);
 	flush_vmap_cache(page_addr, PAGE_SIZE);
-	__bpf_prog_report_arena_violation(prog, is_write, page_addr - kbase, fault_ip);
-	return true;
+
+	if (prog && prog->aux->arena == arena)
+		__bpf_prog_report_arena_violation(prog, is_write,
+						  page_addr - kbase, fault_ip);
+	else if (__ratelimit(&rs))
+		printk_deferred(KERN_ERR "BPF arena %u: %s fault at offset 0x%lx, IP %pS\n",
+				arena->map.id, is_write ? "WRITE" : "READ",
+				page_addr - kbase, (void *)fault_ip);
+	handled = true;
+out:
+	rcu_read_unlock();
+	return handled;
 }
 
 void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip)
Signature · 1 line
2.52.0
author reviewer
#2 lore raw

PATCH v2 2/2 selftests/bpf: add arena fault recovery tests

In reply to Emil Tsalapatis

Test the arena fault handling code with two new selftests. The first one confirms that a program without an arena (and by extension a code path that doesn't involve BPF at all) properly recovers after accessing an unbacked arena page. The second test deploys the same configuration for a program that has its own arena map.

---
 .../selftests/bpf/prog_tests/arena_fault.c    | 100 ++++++++++++++++++
 .../testing/selftests/bpf/progs/arena_fault.c |  65 ++++++++++++
 .../selftests/bpf/progs/arena_fault_noarena.c |  17 +++
 .../selftests/bpf/test_kmods/bpf_testmod.c    |  22 ++++
 .../bpf/test_kmods/bpf_testmod_kfunc.h        |   3 +
 5 files changed, 207 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_fault.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_fault.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_fault_noarena.c
tools/testing/selftests/bpf/prog_tests/arena_fault.c new file +100 -0
new file mode 100644
index 000000000000..8bde1f67e4b4
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include "arena_fault.skel.h"
+#include "arena_fault_noarena.skel.h"
+
+static int run_syscall_prog(struct bpf_program *prog, const char *name)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	int err;
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
+	if (!ASSERT_OK(err, name))
+		return -1;
+	if (!ASSERT_OK(opts.retval, name))
+		return -1;
+	return 0;
+}
+
+static void test_wrong_arena(void)
+{
+	struct arena_fault *fault_skel = NULL;
+	struct arena_fault *stash_skel = NULL;
+
+	stash_skel = arena_fault__open();
+	if (!ASSERT_OK_PTR(stash_skel, "open_stash"))
+		return;
+
+	if (stash_skel->data->skip) {
+		test__skip();
+		goto out;
+	}
+	if (!ASSERT_OK(arena_fault__load(stash_skel), "load_stash"))
+		goto out;
+
+	fault_skel = arena_fault__open_and_load();
+	if (!ASSERT_OK_PTR(fault_skel, "open_load_fault"))
+		goto out;
+
+	if (run_syscall_prog(stash_skel->progs.stash_and_free, "stash_and_free"))
+		goto out;
+	run_syscall_prog(fault_skel->progs.fault_stashed_wrong_arena,
+			 "fault_stashed_wrong_arena");
+
+out:
+	if (stash_skel && bpf_program__fd(stash_skel->progs.clear_stashed) >= 0)
+		run_syscall_prog(stash_skel->progs.clear_stashed, "clear_stashed");
+	arena_fault__destroy(fault_skel);
+	arena_fault__destroy(stash_skel);
+}
+
+static void test_no_arena(void)
+{
+	struct arena_fault_noarena *fault_skel = NULL;
+	struct arena_fault *stash_skel = NULL;
+
+	stash_skel = arena_fault__open();
+	if (!ASSERT_OK_PTR(stash_skel, "open_stash"))
+		return;
+
+	if (stash_skel->data->skip) {
+		test__skip();
+		goto out;
+	}
+	if (!ASSERT_OK(arena_fault__load(stash_skel), "load_stash"))
+		goto out;
+
+	fault_skel = arena_fault_noarena__open_and_load();
+	if (!ASSERT_OK_PTR(fault_skel, "open_load_noarena"))
+		goto out;
+
+	if (run_syscall_prog(stash_skel->progs.stash_and_free, "stash_and_free"))
+		goto out;
+	run_syscall_prog(fault_skel->progs.fault_stashed_noarena,
+			 "fault_stashed_noarena");
+
+out:
+	if (stash_skel && bpf_program__fd(stash_skel->progs.clear_stashed) >= 0)
+		run_syscall_prog(stash_skel->progs.clear_stashed, "clear_stashed");
+	arena_fault_noarena__destroy(fault_skel);
+	arena_fault__destroy(stash_skel);
+}
+
+void serial_test_arena_fault(void)
+{
+#if !defined(__x86_64__) && !defined(__aarch64__)
+	test__skip();
+	return;
+#endif
+
+	if (!env.has_testmod) {
+		test__skip();
+		return;
+	}
+
+	if (test__start_subtest("wrong_arena"))
+		test_wrong_arena();
+	if (test__start_subtest("no_arena"))
+		test_no_arena();
+}
tools/testing/selftests/bpf/progs/arena_fault.c new file +65 -0

65 lines not on this page · raw message

tools/testing/selftests/bpf/progs/arena_fault_noarena.c new file +17 -0
new file mode 100644
index 000000000000..7bb0e6dbd260
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define BPF_NO_KFUNC_PROTOTYPES
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "../test_kmods/bpf_testmod_kfunc.h"
+
+volatile u64 observed;
+
+SEC("syscall")
+int fault_stashed_noarena(void *ctx)
+{
+	observed = bpf_test_arena_fault_load_stashed();
+	return observed != 0;
+}
+
+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +22 -0
index 76fbe64755ef..f33a7345f9ce 100644
@@ -344,6 +344,25 @@ __bpf_kfunc u64 bpf_kfunc_arena_mixed_test(u64 *a__arena, u64 *b__arena__nullabl
 	return *a__arena + (b__arena__nullable ? *b__arena__nullable : 0);
 }
 
+static u64 *arena_fault_stashed_ptr;
+
+__bpf_kfunc void bpf_test_arena_fault_stash(u64 *ptr__arena)
+{
+	WRITE_ONCE(arena_fault_stashed_ptr, ptr__arena);
+}
+
+__bpf_kfunc u64 bpf_test_arena_fault_load_stashed(void)
+{
+	u64 *ptr = READ_ONCE(arena_fault_stashed_ptr);
+
+	return ptr ? READ_ONCE(*ptr) : U64_MAX;
+}
+
+__bpf_kfunc void bpf_test_arena_fault_clear_stashed(void)
+{
+	WRITE_ONCE(arena_fault_stashed_ptr, NULL);
+}
+
 __bpf_kfunc void bpf_kfunc_dynptr_test(struct bpf_dynptr *ptr,
 				       struct bpf_dynptr *ptr__nullable)
 {
@@ -1048,6 +1067,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_nullable_test)
 BTF_ID_FLAGS(func, bpf_kfunc_arena_args5_test)
 BTF_ID_FLAGS(func, bpf_kfunc_arena_stack_arg_test)
 BTF_ID_FLAGS(func, bpf_kfunc_arena_mixed_test)
+BTF_ID_FLAGS(func, bpf_test_arena_fault_stash)
+BTF_ID_FLAGS(func, bpf_test_arena_fault_load_stashed)
+BTF_ID_FLAGS(func, bpf_test_arena_fault_clear_stashed)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1)
 BTF_ID_FLAGS(func, bpf_kfunc_dynptr_test)
 BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_nonzero_offset_test, KF_ACQUIRE)
tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +3 -0
index e9902e651d72..9ec69fd57d98 100644
@@ -212,6 +212,9 @@ __u64 bpf_kfunc_arena_args5_test(__u64 *a__arena, __u64 *b__arena,
 __u64 bpf_kfunc_arena_stack_arg_test(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e,
 				     __u64 *f__arena) __ksym;
 __u64 bpf_kfunc_arena_mixed_test(__u64 *a__arena, __u64 *b__arena__nullable) __ksym;
+void bpf_test_arena_fault_stash(__u64 *ptr__arena) __ksym;
+__u64 bpf_test_arena_fault_load_stashed(void) __ksym;
+void bpf_test_arena_fault_clear_stashed(void) __ksym;
 int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym;
 int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym;
 int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym;
Signature · 1 line
2.52.0
#3 lore raw
In reply to Emil Tsalapatis
This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Quoted 11 lines · On Mon, 9 Nov 2026 18:19:04 +0000 you wrote:

Currently, the BPF arena fault handler protects against a misbehaving BPF program passing an arena pointer to a kfunc then unmapping the backing pages, causing a kernel fault. However, it does not handle errors where the now unmapped arena address is instead accessed by a non-BPF path, or by a BPF program that has a different arena. The former case is already possible in sched-ext if a BPF program removes from the arena pages allocated from the genpool allocator.

[...]

Here is the summary with links:
- [bpf-next,v2,1/2] bpf: recover page faults across all arenas

    https://git.kernel.org/bpf/bpf-next/c/92a4f1dddd99

- [bpf-next,v2,2/2] selftests/bpf: add arena fault recovery tests (no matching commit)

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html