Skip to contentSkip to conversation

bpf: arena: Flush stale TLB entries before reusing ranges

PATCH In review Andrea Righi 2 participants 3 messages Sep 19 – Sep 20

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 bpf: arena: Flush stale TLB entries before reusing ranges

arena_free_pages() returns a virtual range to the range tree before it flushes stale kernel TLB entries for that range. Once the arena lock is released, another CPU can allocate the range and install new PTEs before the flush completes.

An allocator running with a stale translation can then initialize the old page instead of the newly installed page. The freeing CPU subsequently invalidates the translation and frees the old page, leaving the new page zeroed despite the allocator having initialized it.

This corruption was observed with scx_eevdf while repeatedly running stress-ng --sleep 0 --timeout 10.

Keep the range unavailable while clearing its PTEs, invalidating kernel translations, zapping user mappings, and freeing the old pages. Reacquire the arena lock and publish the range only after teardown is complete.

After page teardown, reacquire the arena lock to return the virtual range to the free-range tree. Resilient spin lock acquisition can fail, so defer this final step when that happens. Record that teardown is already complete so the worker only publishes the range instead of trying to tear down the pages again.

Also add a selftest that primes the old translation on multiple CPUs, races a free against concurrent allocations, and verifies the contents of the reallocated page. Exercise both sleepable frees and frees deferred from a non-sleepable context.

Fixes b8467290edab ("bpf: arena: make arena kfuncs any context safe")
---
 kernel/bpf/arena.c                            |  98 ++++++++---
 .../selftests/bpf/prog_tests/arena_reuse.c    | 165 ++++++++++++++++++
 .../testing/selftests/bpf/progs/arena_reuse.c | 113 ++++++++++++
 3 files changed, 356 insertions(+), 20 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_reuse.c
 create mode 100644 tools/testing/selftests/bpf/progs/arena_reuse.c
kernel/bpf/arena.c +78 -20
index 7b6847200b431..73a3103a401b5 100644
@@ -76,6 +76,8 @@ struct arena_free_span {
 	struct llist_node node;
 	unsigned long uaddr;
 	u32 page_cnt;
+	/* PTE and TLB teardown complete. */
+	bool publish;
 };
 
 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena)
@@ -873,15 +875,13 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 	bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
 
 	if (!sleepable)
-		goto defer;
+		goto defer_free;
 
 	ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags);
 
 	/* Can't proceed without holding the spinlock so defer the free */
 	if (ret)
-		goto defer;
-
-	range_tree_set(&arena->rt, pgoff, page_cnt);
+		goto defer_free;
 
 	init_llist_head(&free_pages);
 	cdata.arena = arena;
@@ -890,10 +890,9 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 	apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
 				     apply_range_clear_cb, &cdata);
 
-	/* drop the lock to do the tlb flush and zap pages */
 	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
 
-	/* ensure no stale TLB entries */
+	/* Ensure no stale kernel translations before publishing the range. */
 	flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE));
 
 	if (page_cnt > 1)
@@ -911,11 +910,17 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 			zap_pages(arena, full_uaddr, 1);
 		__free_page(page);
 	}
+
+	ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags);
+	if (ret)
+		goto defer_publish;
+	range_tree_set(&arena->rt, pgoff, page_cnt);
+	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
 	bpf_map_memcg_exit(old_memcg, new_memcg);
 
 	return;
 
-defer:
+defer_free:
 	s = kmalloc_nolock(sizeof(struct arena_free_span), __GFP_ACCOUNT, -1);
 	bpf_map_memcg_exit(old_memcg, new_memcg);
 	if (!s)
@@ -928,6 +933,21 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 
 	s->page_cnt = page_cnt;
 	s->uaddr = uaddr;
+	s->publish = false;
+	llist_add(&s->node, &arena->free_spans);
+	irq_work_queue(&arena->free_irq);
+	return;
+
+defer_publish:
+	s = kmalloc_nolock(sizeof(struct arena_free_span), __GFP_ACCOUNT, -1);
+	bpf_map_memcg_exit(old_memcg, new_memcg);
+	if (!s)
+		/* The pages are gone; leaking the unavailable range is safe. */
+		return;
+
+	s->page_cnt = page_cnt;
+	s->uaddr = uaddr;
+	s->publish = true;
 	llist_add(&s->node, &arena->free_spans);
 	irq_work_queue(&arena->free_irq);
 }
@@ -977,7 +997,8 @@ static void arena_free_worker(struct work_struct *work)
 	struct llist_node *list, *pos, *t;
 	struct arena_free_span *s;
 	u64 arena_vm_start, user_vm_start;
-	struct llist_head free_pages;
+	struct llist_head completed_spans, free_pages;
+	struct llist_head publish_spans, teardown_spans;
 	struct clear_range_data cdata;
 	struct page *page;
 	unsigned long full_uaddr;
@@ -991,41 +1012,51 @@ static void arena_free_worker(struct work_struct *work)
 
 	bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
 
+	init_llist_head(&completed_spans);
 	init_llist_head(&free_pages);
+	init_llist_head(&publish_spans);
+	init_llist_head(&teardown_spans);
 	cdata.arena = arena;
 	cdata.free_pages = &free_pages;
 	arena_vm_start = bpf_arena_get_kern_vm_start(arena);
 	user_vm_start = bpf_arena_get_user_vm_start(arena);
 
 	list = llist_del_all(&arena->free_spans);
-	llist_for_each(pos, list) {
+	llist_for_each_safe(pos, t, list) {
 		s = llist_entry(pos, struct arena_free_span, node);
 		page_cnt = s->page_cnt;
-		kaddr = arena_vm_start + s->uaddr;
 		pgoff = compute_pgoff(arena, s->uaddr);
 
+		if (s->publish) {
+			range_tree_set(&arena->rt, pgoff, page_cnt);
+			llist_add(&s->node, &completed_spans);
+			continue;
+		}
+
+		kaddr = arena_vm_start + s->uaddr;
 		/* clear ptes and collect pages in free_pages llist */
 		apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
 					     apply_range_clear_cb, &cdata);
-
-		range_tree_set(&arena->rt, pgoff, page_cnt);
+		llist_add(&s->node, &teardown_spans);
 	}
 	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
 
-	/* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */
-	llist_for_each_safe(pos, t, list) {
+	llist_for_each_safe(pos, t, __llist_del_all(&completed_spans)) {
+		s = llist_entry(pos, struct arena_free_span, node);
+		kfree_nolock(s);
+	}
+
+	/* Tear ranges down while they are still unavailable to allocation. */
+	llist_for_each_safe(pos, t, __llist_del_all(&teardown_spans)) {
 		s = llist_entry(pos, struct arena_free_span, node);
 		page_cnt = s->page_cnt;
-		full_uaddr = clear_lo32(user_vm_start) + s->uaddr;
 		kaddr = arena_vm_start + s->uaddr;
+		full_uaddr = clear_lo32(user_vm_start) + s->uaddr;
 
-		/* ensure no stale TLB entries */
 		flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE));
-
-		/* remove pages from user vmas */
 		zap_pages(arena, full_uaddr, page_cnt);
-
-		kfree_nolock(s);
+		s->publish = true;
+		llist_add(&s->node, &publish_spans);
 	}
 
 	/* free all pages collected by apply_to_existing_page_range() in the first loop */
@@ -1034,6 +1065,33 @@ static void arena_free_worker(struct work_struct *work)
 		__free_page(page);
 	}
 
+	list = __llist_del_all(&publish_spans);
+	if (!list)
+		goto out;
+
+	if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) {
+		llist_for_each_safe(pos, t, list) {
+			s = llist_entry(pos, struct arena_free_span, node);
+			llist_add(&s->node, &arena->free_spans);
+		}
+		schedule_work(work);
+		goto out;
+	}
+
+	llist_for_each_safe(pos, t, list) {
+		s = llist_entry(pos, struct arena_free_span, node);
+		pgoff = compute_pgoff(arena, s->uaddr);
+		range_tree_set(&arena->rt, pgoff, s->page_cnt);
+		llist_add(&s->node, &completed_spans);
+	}
+	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+
+	llist_for_each_safe(pos, t, __llist_del_all(&completed_spans)) {
+		s = llist_entry(pos, struct arena_free_span, node);
+		kfree_nolock(s);
+	}
+
+out:
 	bpf_map_memcg_exit(old_memcg, new_memcg);
 }
 
tools/testing/selftests/bpf/prog_tests/arena_reuse.c new file +165 -0

165 lines not on this page · raw message

tools/testing/selftests/bpf/progs/arena_reuse.c new file +113 -0

113 lines not on this page · raw message

Signature · 1 line
2.55.0
#1 lore raw

kernel/bpf/arena.c static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, · line 918

Quoted 10 lines · Andrea Righi
kernel/bpf/arena.c +6 -0
@@ -911,11 +910,17 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 			zap_pages(arena, full_uaddr, 1);
 		__free_page(page);
 	}
+
+	ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags);
+	if (ret)
+		goto defer_publish;
+	range_tree_set(&arena->rt, pgoff, page_cnt);
+	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);

Now the ptes are cleared in one locked section and the range is set in another, and arena_vm_fault() can get in between. It doesn't look at the range tree to decide. It does vmalloc_to_page(), sees no page, range_tree_clear() of an already clear range returns 0, so it allocates a page and installs it into the kernel pte. Then the code above marks that range free. The next bpf_arena_alloc_pages() that picks it hits
WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)
in apply_range_set_cb(), returns NULL, puts t
he range back into the
tree, and the next call does the same. All it takes is user space touching the page while the prog frees it.

iirc Emil has patches that address this issue differently.

pw-bot: cr

#2 lore raw

Hi Alexei,

Quoted 15 lines · Alexei Starovoitov

kernel/bpf/arena.c static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, · line 918

Quoted 10 lines · Andrea Righi
kernel/bpf/arena.c +6 -0
@@ -911,11 +910,17 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 			zap_pages(arena, full_uaddr, 1);
 		__free_page(page);
 	}
+
+	ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags);
+	if (ret)
+		goto defer_publish;
+	range_tree_set(&arena->rt, pgoff, page_cnt);
+	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);

Now the ptes are cleared in one locked section and the range is set in another, and arena_vm_fault() can get in between. It doesn't look at the range tree to decide. It does vmalloc_to_page(), sees no page,

Quoted 8 lines

range_tree_clear() of an already clear range returns 0, so it allocates a page and installs it into the kernel pte. Then the code above marks that range free. The next bpf_arena_alloc_pages() that picks it hits
WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)
in apply_range_set_cb(), returns NULL, puts t
he range back into the
tree, and the next call does the same. All it takes is user space touching the page while the prog frees it.

You're right, I missed that range_tree_clear() succeeds even when the range is already clear, so arena_vm_fault() populates the range during the teardown-to-publish window, leaving the range marked free with a valid PTE installed.

Quoted 2 lines

iirc Emil has patches that address this issue differently.

Thanks for catching this! I'll sync with Emil.

-Andrea