Exceptions - Resource Cleanup - Redux

index b8d765d6ab07..83a858c6d472 100644
@@ -854,6 +854,42 @@ static void find_used_callee_regs(struct jit_ctx *ctx)
 	ctx->nr_used_callee_reg = i;
 }
 
+/*
+ * Offset from the frame pointer of the @idx-th register pushed by push_callee_regs(). The
+ * frame record sits at the frame pointer, the tail call count pair right below it, and
+ * then the pushed pairs, each 16 bytes with its first register at the lower address.
+ */
+static int callee_reg_slot_off(int idx)
+{
+	return -32 - 16 * (idx / 2) + 8 * (idx % 2);
+}
+
+/*
+ * Record where push_callee_regs() saved the caller's R6..R9, relative to the frame
+ * pointer that arch_bpf_stack_walk() reports for this frame, for the exception unwinder.
+ * An exception boundary pushes x19..x28 in order, so R6..R9 take the first four slots.
+ */
+static void record_callee_reg_off(struct jit_ctx *ctx)
+{
+	struct bpf_prog_aux *aux = ctx->prog->aux;
+	int i, j;
+
+	if (aux->exception_boundary) {
+		for (i = 0; i < 4; i++)
+			aux->callee_reg_off[i] = callee_reg_slot_off(i);
+		return;
+	}
+	for (i = 0; i < 4; i++) {
+		aux->callee_reg_off[i] = 0;
+		for (j = 0; j < ctx->nr_used_callee_reg; j++) {
+			if (ctx->used_callee_reg[j] != bpf2a64[BPF_REG_6 + i])
+				continue;
+			aux->callee_reg_off[i] = callee_reg_slot_off(j);
+			break;
+		}
+	}
+}
+
 /* Save callee-saved registers */
 static void push_callee_regs(struct jit_ctx *ctx)
 {

@@ -884,6 +920,7 @@ static void push_callee_regs(struct jit_ctx *ctx)
 			emit(A64_PUSH(reg1, A64_ZR, A64_SP), ctx);
 		}
 	}
+	record_callee_reg_off(ctx);
 }
 
 /* Restore callee-saved registers */

index 5bc744c42f7f..19fef39f1445 100644
@@ -382,6 +382,24 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)
 	*pprog = prog;
 }
 
+/*
+ * Record where push_callee_regs() saved the caller's R6..R9 relative to rbp, for the
+ * exception unwinder. @off is the offset of rsp from rbp right before the first push.
+ */
+static void record_callee_reg_off(struct bpf_prog *prog, s32 off, bool *callee_regs_used)
+{
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		if (!callee_regs_used[i]) {
+			prog->aux->callee_reg_off[i] = 0;
+			continue;
+		}
+		off -= 8;
+		prog->aux->callee_reg_off[i] = off;
+	}
+}
+
 /* add rsp, depth */
 static void emit_add_rsp(u8 **pprog, u32 depth)
 {

@@ -2402,6 +2420,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	/* it has to be saved around calls of helpers and kfuncs */
 	bool arena_sp_save = arena_sp && calls_clobber_arena_sp(bpf_prog);
 	s32 arena_sp_off = 0;
+	s32 callee_regs_off;
 	int err;
 
 	stack_depth = bpf_prog->aux->stack_depth;

@@ -2472,6 +2491,20 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 		push_callee_regs(&prog, callee_regs_used);
 	}
 
+	/*
+	 * Record where the pushes above saved the caller's R6..R9, relative to the
+	 * rbp that arch_bpf_stack_walk() reports for this frame. Below rbp lie, from
+	 * high to low addresses: round_up(stack_depth, 8) bytes of program stack
+	 * (none with a private stack), the tail call count and its pointer (16 bytes)
+	 * when tail_call_reachable, r12 when exception_boundary or arena_vm_start,
+	 * and then rbx, r13, r14 and r15 in push order, each only when pushed.
+	 */
+	callee_regs_off = -(s32)round_up(stack_depth, 8) - (tail_call_reachable ? 16 : 0);
+	if (bpf_prog->aux->exception_boundary || arena_vm_start)
+		callee_regs_off -= 8;
+	record_callee_reg_off(bpf_prog, callee_regs_off,
+			      bpf_prog->aux->exception_boundary ? all_callee_regs_used : callee_regs_used);
+
 	/* Compute callee-saved register area size. */
 	callee_saved_size = 0;
 	if (bpf_prog->aux->exception_boundary || arena_vm_start)

index 520fd72e21fd..da42dbd32fd4 100644
@@ -1997,6 +1997,12 @@ struct bpf_prog_aux {
 	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
 	u16 stack_arg_sp_adjust;
 	u16 freplace_link_cnt; /* counts freplace links extending this prog */
+	/*
+	 * Offsets from the native frame pointer of this JITed program at which
+	 * its prologue saved the caller's R6..R9, or 0 when that register is not
+	 * saved. Filled by the JIT; consumed by the exception unwinder.
+	 */
+	s16 callee_reg_off[4];
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif

index da42dbd32fd4..b01820e67795 100644
@@ -7,6 +7,7 @@
 #include <uapi/linux/bpf.h>
 #include <uapi/linux/filter.h>
 #include <linux/bpf_defs.h>
+#include <linux/bpf_throw.h>
 
 #include <crypto/sha2.h>
 #include <linux/workqueue.h>

52 lines not on this page · raw message

index f88a1c041a7a..7a25d1cb2c30 100644
@@ -3930,6 +3930,15 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp,
 {
 }
 
+/*
+ * The native entry of bpf_throw(), which captures the BPF register file of
+ * the throw site, or NULL when the architecture calls the kfunc directly.
+ */
+void * __weak bpf_arch_throw_entry(void)
+{
+	return NULL;
+}
+
 bool __weak bpf_jit_supports_timed_may_goto(void)
 {
 	return false;

index 937a0d7d6839..de31fdd18411 100644
@@ -3460,7 +3460,13 @@ static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp)
 	return false;
 }
 
-__bpf_kfunc void bpf_throw(u64 cookie)
+/*
+ * Unwind the stack to the exception callback of the main program. @regs holds
+ * the callee-saved registers of the throw site, or is NULL when the
+ * architecture has no native entry and the kfunc was called directly.
+ */
+__bpf_unwind_entry
+notrace void bpf_throw_unwind(struct bpf_throw_regs *regs, u64 cookie)
 {
 	struct bpf_throw_ctx ctx = {};
 

@@ -3472,8 +3478,8 @@ __bpf_kfunc void bpf_throw(u64 cookie)
 	WARN_ON_ONCE(!ctx.cnt);
 	/*
 	 * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning
-	 * deeper stack depths than ctx.sp as we do not return from bpf_throw,
-	 * which skips compiler generated instrumentation to do the same. Some
+	 * deeper stack depths than ctx.sp as we do not return from here, which
+	 * skips compiler generated instrumentation to do the same. Some
 	 * architectures cannot recover sp while unwinding, so fall back to bp.
 	 */
 	kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp));

@@ -3481,6 +3487,11 @@ __bpf_kfunc void bpf_throw(u64 cookie)
 	WARN(1, "A call to BPF exception callback should never return\n");
 }
 
+__bpf_kfunc void bpf_throw(u64 cookie)
+{
+	bpf_throw_unwind(NULL, cookie);
+}
+
 __bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags)
 {
 	struct bpf_async_kern *async = (struct bpf_async_kern *)wq;

index 20b1b9ff574c..c5b4025ee222 100644
@@ -24434,6 +24434,8 @@ BTF_ID(func, do_exit)
 BTF_ID(func, do_group_exit)
 BTF_ID(func, kthread_complete_and_exit)
 BTF_ID(func, make_task_dead)
+BTF_ID(func, bpf_throw)
+BTF_ID(func, bpf_throw_unwind)
 BTF_SET_END(noreturn_deny)
 
 static bool can_be_sleepable(struct bpf_prog *prog)

@@ -24793,6 +24795,11 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc
 	} else if (func_id == special_kfunc_list[KF_bpf_arena_free_pages]) {
 		if (env->insn_aux_data[insn_idx].non_sleepable)
 			addr = (unsigned long)bpf_arena_free_pages_non_sleepable;
+	} else if (func_id == special_kfunc_list[KF_bpf_throw]) {
+		void *entry = bpf_arch_throw_entry();
+
+		if (entry)
+			addr = (unsigned long)entry;
 	}
 	desc->addr = addr;
 	return 0;

index 19fef39f1445..bdfc5329a00c 100644
@@ -5135,6 +5135,14 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp
 #endif
 }
 
+#ifdef CONFIG_BPF_SYSCALL
+/* arch/x86/net/bpf_throw.S, which calls bpf_throw_unwind(); both need the syscall. */
+void *bpf_arch_throw_entry(void)
+{
+	return (void *)bpf_throw_entry;
+}
+#endif
+
 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
 			       struct bpf_prog *new, struct bpf_prog *old)
 {

index 57d974458750..2bb2c18796ac 100644
@@ -6,21 +6,28 @@
 #ifndef _LINUX_BPF_THROW_H
 #define _LINUX_BPF_THROW_H
 
+/*
+ * The callee-saved registers of the architecture that no BPF register maps
+ * to, which an entry captures after R9. None yet.
+ */
+#define BPF_THROW_NR_EXTRA		0
 /* The callee-saved registers the unwinder tracks: R6..R9. */
 #define BPF_THROW_NR_CALLEE_SAVED	4
-/* The size of struct bpf_throw_regs, for the entries. */
-#define BPF_THROW_REGS_SIZE		(8 * BPF_THROW_NR_CALLEE_SAVED)
+/* The size of struct bpf_throw_regs, for the entries: R0..R9 and the extra ones. */
+#define BPF_THROW_REGS_SIZE		(8 * (10 + BPF_THROW_NR_EXTRA))
 
 #ifndef __ASSEMBLER__
 #include <linux/build_bug.h>
 #include <linux/types.h>
+#include <uapi/linux/bpf.h>
 
 /*
- * The callee-saved registers captured at a throw site, indexed like
- * bpf_prog_aux::callee_reg_off.
+ * The register file captured at a throw site: the BPF registers R0..R9, then
+ * the extra callee-saved registers of the architecture; from R6 on, indexed
+ * like bpf_prog_aux::callee_reg_off.
  */
 struct bpf_throw_regs {
-	u64 saved[BPF_THROW_NR_CALLEE_SAVED];
+	u64 r[BPF_REG_10 + BPF_THROW_NR_EXTRA];
 };
 static_assert(sizeof(struct bpf_throw_regs) == BPF_THROW_REGS_SIZE);
 

index 83a858c6d472..33cc50268f3a 100644
@@ -10,6 +10,7 @@
 #include <linux/arm-smccc.h>
 #include <linux/bitfield.h>
 #include <linux/bpf.h>
+#include <linux/bpf_throw.h>
 #include <linux/cfi.h>
 #include <linux/extable.h>
 #include <linux/filter.h>

@@ -3965,6 +3966,13 @@ bool bpf_jit_supports_timed_may_goto(void)
 	return true;
 }
 
+#ifdef CONFIG_BPF_SYSCALL
+void *bpf_arch_throw_entry(void)
+{
+	return (void *)bpf_throw_entry;
+}
+#endif
+
 bool bpf_jit_inlines_helper_call(s32 imm)
 {
 	switch (imm) {

index 33cc50268f3a..c9f851bd5003 100644
@@ -866,24 +866,29 @@ static int callee_reg_slot_off(int idx)
 }
 
 /*
- * Record where push_callee_regs() saved the caller's R6..R9, relative to the frame
- * pointer that arch_bpf_stack_walk() reports for this frame, for the exception unwinder.
- * An exception boundary pushes x19..x28 in order, so R6..R9 take the first four slots.
+ * Record where the prologue saved the caller's callee-saved registers, relative to
+ * the frame pointer that arch_bpf_stack_walk() reports for this frame, for the
+ * exception unwinder. The registers are x19..x28 in the order of struct
+ * bpf_throw_regs, R6..R9 first: the tail call count pointer is saved with the frame
+ * record, the others where push_callee_regs() put them, and x24 nowhere, since
+ * JITed code leaves it alone.
  */
 static void record_callee_reg_off(struct jit_ctx *ctx)
 {
 	struct bpf_prog_aux *aux = ctx->prog->aux;
 	int i, j;
 
-	if (aux->exception_boundary) {
-		for (i = 0; i < 4; i++)
-			aux->callee_reg_off[i] = callee_reg_slot_off(i);
-		return;
-	}
-	for (i = 0; i < 4; i++) {
+	BUILD_BUG_ON(BPF_THROW_NR_CALLEE_SAVED != 10);
+	for (i = 0; i < BPF_THROW_NR_CALLEE_SAVED; i++) {
+		u8 reg = A64_R(19 + i);
+
 		aux->callee_reg_off[i] = 0;
+		if (reg == bpf2a64[TCCNT_PTR]) {
+			aux->callee_reg_off[i] = -8;
+			continue;
+		}
 		for (j = 0; j < ctx->nr_used_callee_reg; j++) {
-			if (ctx->used_callee_reg[j] != bpf2a64[BPF_REG_6 + i])
+			if (ctx->used_callee_reg[j] != reg)
 				continue;
 			aux->callee_reg_off[i] = callee_reg_slot_off(j);
 			break;

@@ -891,35 +896,25 @@ static void record_callee_reg_off(struct jit_ctx *ctx)
 	}
 }
 
-/* Save callee-saved registers */
+/*
+ * Save the callee-saved registers this function uses. An exception boundary
+ * saves nothing more: the unwinder recovers the registers of the frames it
+ * discards and hands them to the exception callback.
+ */
 static void push_callee_regs(struct jit_ctx *ctx)
 {
 	int reg1, reg2, i;
 
-	/*
-	 * Program acting as exception boundary should save all ARM64
-	 * Callee-saved registers as the exception callback needs to recover
-	 * all ARM64 Callee-saved registers in its epilogue.
-	 */
-	if (ctx->prog->aux->exception_boundary) {
-		emit(A64_PUSH(A64_R(19), A64_R(20), A64_SP), ctx);
-		emit(A64_PUSH(A64_R(21), A64_R(22), A64_SP), ctx);
-		emit(A64_PUSH(A64_R(23), A64_R(24), A64_SP), ctx);
-		emit(A64_PUSH(A64_R(25), A64_R(26), A64_SP), ctx);
-		emit(A64_PUSH(A64_R(27), A64_R(28), A64_SP), ctx);
-		ctx->fp_used = true;
-	} else {
-		find_used_callee_regs(ctx);
-		for (i = 0; i + 1 < ctx->nr_used_callee_reg; i += 2) {
-			reg1 = ctx->used_callee_reg[i];
-			reg2 = ctx->used_callee_reg[i + 1];
-			emit(A64_PUSH(reg1, reg2, A64_SP), ctx);
-		}
-		if (i < ctx->nr_used_callee_reg) {
-			reg1 = ctx->used_callee_reg[i];
-			/* keep SP 16-byte aligned */
-			emit(A64_PUSH(reg1, A64_ZR, A64_SP), ctx);
-		}
+	find_used_callee_regs(ctx);
+	for (i = 0; i + 1 < ctx->nr_used_callee_reg; i += 2) {
+		reg1 = ctx->used_callee_reg[i];
+		reg2 = ctx->used_callee_reg[i + 1];
+		emit(A64_PUSH(reg1, reg2, A64_SP), ctx);
+	}
+	if (i < ctx->nr_used_callee_reg) {
+		reg1 = ctx->used_callee_reg[i];
+		/* keep SP 16-byte aligned */
+		emit(A64_PUSH(reg1, A64_ZR, A64_SP), ctx);
 	}
 	record_callee_reg_off(ctx);
 }

@@ -927,34 +922,19 @@ static void push_callee_regs(struct jit_ctx *ctx)
 /* Restore callee-saved registers */
 static void pop_callee_regs(struct jit_ctx *ctx)
 {
-	struct bpf_prog_aux *aux = ctx->prog->aux;
 	int reg1, reg2, i;
 
-	/*
-	 * Program acting as exception boundary pushes R23 and R24 in addition
-	 * to BPF callee-saved registers. Exception callback uses the boundary
-	 * program's stack frame, so recover these extra registers in the above
-	 * two cases.
-	 */
-	if (aux->exception_boundary || aux->exception_cb) {
-		emit(A64_POP(A64_R(27), A64_R(28), A64_SP), ctx);
-		emit(A64_POP(A64_R(25), A64_R(26), A64_SP), ctx);
-		emit(A64_POP(A64_R(23), A64_R(24), A64_SP), ctx);
-		emit(A64_POP(A64_R(21), A64_R(22), A64_SP), ctx);
-		emit(A64_POP(A64_R(19), A64_R(20), A64_SP), ctx);
-	} else {
-		i = ctx->nr_used_callee_reg - 1;
-		if (ctx->nr_used_callee_reg % 2 != 0) {
-			reg1 = ctx->used_callee_reg[i];
-			emit(A64_POP(reg1, A64_ZR, A64_SP), ctx);
-			i--;
-		}
-		while (i > 0) {
-			reg1 = ctx->used_callee_reg[i - 1];
-			reg2 = ctx->used_callee_reg[i];
-			emit(A64_POP(reg1, reg2, A64_SP), ctx);
-			i -= 2;
-		}
+	i = ctx->nr_used_callee_reg - 1;
+	if (ctx->nr_used_callee_reg % 2 != 0) {
+		reg1 = ctx->used_callee_reg[i];
+		emit(A64_POP(reg1, A64_ZR, A64_SP), ctx);
+		i--;
+	}
+	while (i > 0) {
+		reg1 = ctx->used_callee_reg[i - 1];
+		reg2 = ctx->used_callee_reg[i];
+		emit(A64_POP(reg1, reg2, A64_SP), ctx);
+		i -= 2;
 	}
 }
 

@@ -1055,24 +1035,22 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 			/* BTI landing pad for the tail call, done with a BR */
 			emit_bti(A64_BTI_J, ctx);
 		}
-		push_callee_regs(ctx);
 	} else {
 		/*
-		 * Exception callback receives FP of Main Program as third
-		 * parameter
+		 * The exception callback receives the main program's frame
+		 * pointer as its third parameter and, as its fourth, the
+		 * callee-saved registers x19..x28 the unwinder recovered from
+		 * the frames it discarded: the values the main program's caller
+		 * expects back. Load them, then build the rest of the frame
+		 * below the main program's frame record like any function.
 		 */
+		for (int i = 0; i < BPF_THROW_NR_CALLEE_SAVED; i++)
+			emit(A64_LDR64I(A64_R(19 + i), A64_R(3), 8 * i), ctx);
 		emit(A64_MOV(1, A64_FP, A64_R(2)), ctx);
-		/*
-		 * Main Program already pushed the frame record and the
-		 * callee-saved registers. The exception callback will not push
-		 * anything and re-use the main program's stack.
-		 *
-		 * 12 registers are on the stack
-		 */
-		emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx);
-		/* The callback may use its own BPF stack, set up fp for it. */
-		ctx->fp_used = true;
+		emit(A64_MOV(1, A64_SP, A64_FP), ctx);
+		prepare_bpf_tail_call_cnt(ctx);
 	}
+	push_callee_regs(ctx);
 
 	/* Stack must be multiples of 16B */
 	ctx->stack_size = round_up(prog->aux->stack_depth, 16);

202 lines not on this page · raw message

57 lines not on this page · raw message

index 2bb2c18796ac..6449757c18a9 100644
@@ -8,11 +8,23 @@
 
 /*
  * The callee-saved registers of the architecture that no BPF register maps
- * to, which an entry captures after R9. None yet.
+ * to, which the entries capture after R9 and the unwinder restores along
+ * with R6..R9 when it discards frames: r12 on x86-64, which holds the arena
+ * base, and x23..x28 on arm64, of which x23 is the stack pointer of the
+ * stack in arena and x25..x28 are the BPF frame pointer, the tail call count
+ * pointer, the private stack pointer and the arena base, while x24 is left
+ * alone by JITed code but may be in use by the C code of the unwinder when
+ * it enters the exception callback, whose epilogue never runs.
  */
-#define BPF_THROW_NR_EXTRA		0
-/* The callee-saved registers the unwinder tracks: R6..R9. */
-#define BPF_THROW_NR_CALLEE_SAVED	4
+#if defined(CONFIG_X86_64)
+#define BPF_THROW_NR_EXTRA	1
+#elif defined(CONFIG_ARM64)
+#define BPF_THROW_NR_EXTRA	6
+#else
+#define BPF_THROW_NR_EXTRA	0
+#endif
+/* The callee-saved registers the unwinder tracks: R6..R9, then the extra ones. */
+#define BPF_THROW_NR_CALLEE_SAVED	(4 + BPF_THROW_NR_EXTRA)
 /* The size of struct bpf_throw_regs, for the entries: R0..R9 and the extra ones. */
 #define BPF_THROW_REGS_SIZE		(8 * (10 + BPF_THROW_NR_EXTRA))
 

index 1d33a78d9373..eb5a5bca5c8d 100644
@@ -2103,7 +2103,6 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 	prog->aux->real_func_cnt = env->subprog_cnt;
 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;
-	prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust;
 	bpf_prog_jit_attempt_done(prog);
 	return 0;
 out_free:

index de31fdd18411..0164f183aade 100644
@@ -3434,6 +3434,13 @@ struct bpf_throw_ctx {
 	u64 sp;
 	u64 bp;
 	int cnt;
+	/*
+	 * The callee-saved registers of the frame being walked, R6-R9 then the
+	 * architecture's own: what the throw site had, then what each frame
+	 * recovered of its caller's. After the main frame they are the ones
+	 * its caller expects back.
+	 */
+	u64 saved[BPF_THROW_NR_CALLEE_SAVED];
 };
 
 static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp)

@@ -3451,6 +3458,12 @@ static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp)
 	rcu_read_unlock();
 	if (!prog)
 		return !ctx->cnt;
+	/*
+	 * Recover the caller's callee-saved registers: those this frame's
+	 * prologue saved sit in its frame, the rest are still the values the
+	 * frame itself has.
+	 */
+	bpf_throw_caller_regs(bpf_prog_frame_aux(prog), bp, ctx->saved);
 	ctx->cnt++;
 	if (bpf_is_subprog(prog))
 		return true;

@@ -3461,15 +3474,24 @@ static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp)
 }
 
 /*
- * Unwind the stack to the exception callback of the main program. @regs holds
- * the callee-saved registers of the throw site, or is NULL when the
- * architecture has no native entry and the kfunc was called directly.
+ * Unwind the stack to the exception callback of the main program. @regs is the
+ * register file of the throw site, which the entry of the architecture
+ * captured: the callee-saved registers of the frames unwound are restored from
+ * it for the callback. It is NULL when the architecture has no native entry
+ * and the kfunc was called directly.
  */
 __bpf_unwind_entry
 notrace void bpf_throw_unwind(struct bpf_throw_regs *regs, u64 cookie)
 {
 	struct bpf_throw_ctx ctx = {};
 
+	/*
+	 * Without the file, on an architecture that has no native entry, the
+	 * prologue of the main program has saved every callee-saved register
+	 * and the callback restores them from its frame, not from @saved.
+	 */
+	if (regs)
+		memcpy(ctx.saved, &regs->r[BPF_REG_6], sizeof(ctx.saved));
 	arch_bpf_stack_walk(bpf_stack_walker, &ctx);
 	WARN_ON_ONCE(!ctx.aux);
 	if (ctx.aux)

@@ -3483,7 +3505,7 @@ notrace void bpf_throw_unwind(struct bpf_throw_regs *regs, u64 cookie)
 	 * architectures cannot recover sp while unwinding, so fall back to bp.
 	 */
 	kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp));
-	ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0);
+	ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, (u64)(long)ctx.saved, 0);
 	WARN(1, "A call to BPF exception callback should never return\n");
 }
 

index e7f410a74a21..14e5c6bdd4d8 100644
@@ -287,11 +287,7 @@ __jited("	mov	x29, sp")
 __jited("	stp	xzr, x26, [sp, #-0x10]!")
 __jited("	mov	x26, sp")
 __jited("...")
-__jited("	stp	x19, x20, [sp, #-0x10]!")
-__jited("	stp	x21, x22, [sp, #-0x10]!")
-__jited("	stp	x23, x24, [sp, #-0x10]!")
-__jited("	stp	x25, x26, [sp, #-0x10]!")
-__jited("	stp	x27, x28, [sp, #-0x10]!")
+__jited("	stp	x25, x27, [sp, #-0x10]!")
 __jited("	mov	x27, {{.*}}")
 __jited("	movk	x27, {{.*}}, lsl #16")
 __jited("	movk	x27, {{.*}}")

@@ -303,7 +299,7 @@ __jited("	str	x0, [x27]")
 __jited("	mov	x0, #0x0")
 __jited("	bl	0x{{.*}}")
 __jited("	mov	x8, x0")
-__jited("	ldp	x27, x28, [sp], #0x10")
+__jited("	ldp	x25, x27, [sp], #0x10")
 int private_stack_exception_main_prog(void)
 {
 	asm volatile ("			\

@@ -331,7 +327,7 @@ __jited("...")
 __jited("	callq")
 __jited("	popq	%r9")
 __arch_arm64
-__jited("	stp	x27, x28, [sp, #-0x10]!")
+__jited("	stp	x25, x27, [sp, #-0x10]!")
 __jited("	mov	x27, {{.*}}")
 __jited("	movk	x27, {{.*}}, lsl #16")
 __jited("	movk	x27, {{.*}}")

@@ -342,7 +338,7 @@ __jited("	mov	x0, #0x2a")
 __jited("	str	x0, [x27]")
 __jited("	bl	0x{{.*}}")
 __jited("	mov	x8, x0")
-__jited("	ldp	x27, x28, [sp], #0x10")
+__jited("	ldp	x25, x27, [sp], #0x10")
 int private_stack_exception_sub_prog(void)
 {
 	asm volatile ("			\

index eb5a5bca5c8d..8597b443d561 100644
@@ -2280,8 +2280,10 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat
 {
 	struct bpf_subprog_info *info = env->subprog_info;
 	int err, cnt = env->subprog_cnt;
+	struct bpf_insn_aux_data *aux;
 	struct bpf_insn *last;
 	struct bpf_prog *prog;
+	bool indirect;
 
 	/* We only reserve one slot for hidden subprogs in subprog_info. */
 	if (env->hidden_subprog_cnt) {

@@ -2299,6 +2301,17 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat
 	if (err)
 		return err;
 	prog = env->prog;
+	/*
+	 * The patch starts with the original last insn, but the commit moved that
+	 * insn's aux data to the end of the patch: give it back to the insn. All
+	 * but indirect_target, which the commit left with the insn, since an
+	 * indirect jump to it lands there still, see patch_list_adj_aux().
+	 */
+	aux = &env->insn_aux_data[prog->len - len];
+	indirect = aux->indirect_target;
+	swap(aux[0], aux[len - 1]);
+	aux[0].indirect_target = indirect;
+	aux[len - 1].indirect_target = false;
 	info[cnt + 1].start = info[cnt].start;
 	info[cnt].start = prog->len - len + 1;
 	/* The patch ends with the exit. */

index 4771c55d8f41..7a2b6507f562 100644
@@ -1023,6 +1023,7 @@ struct bpf_subprog_info {
 	u32 exit_idx;
 	struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
 	u16 stack_depth; /* max. stack depth used by this function */
+	/* bytes below the verified stack depth reserved for hidden slots, see bpf_reserve_stack_slots() */
 	u16 stack_extra;
 	u16 arena_stack_depth; /* size of the frame in the stack in arena */
 	u32 insns_total;

@@ -1208,6 +1209,13 @@ void bpf_patch_image_install(struct bpf_prog *prog, const struct bpf_patch_image
 /* single container for all structs
  * one verifier_env per bpf_check() call
  */
+/* Initialization of a hidden stack slot, emitted in the subprogram's prologue. */
+struct bpf_stack_init {
+	u32 subprog;
+	s16 off;
+	s32 imm;
+};
+
 struct bpf_verifier_env {
 	u32 insn_idx;
 	u32 prev_insn_idx;

@@ -1233,6 +1241,8 @@ struct bpf_verifier_env {
 	u32 insn_array_map_cnt;		/* number of used maps of type BPF_MAP_TYPE_INSN_ARRAY */
 	u32 id_gen;			/* used to generate unique reg IDs */
 	u32 hidden_subprog_cnt;		/* number of hidden subprogs */
+	struct bpf_stack_init *stack_inits; /* hidden slot initializations, see bpf_stack_slot_init() */
+	u32 stack_init_cnt;
 	int exception_callback_subprog;
 	bool explore_alu_limits;
 	bool allow_ptr_leaks;

@@ -2174,6 +2184,8 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
 
 /* Functions in fixups.c, called from bpf_check() */
 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);
+int bpf_reserve_stack_slots(struct bpf_verifier_env *env, int subprog, int nr, s16 *off);
+int bpf_stack_slot_init(struct bpf_verifier_env *env, int subprog, s16 off, s32 imm);
 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);

368 lines not on this page · raw message

index c5b4025ee222..0684ac1f4f22 100644
@@ -25732,6 +25732,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	for (i = 0; i < env->func_set_cnt; i++)
 		kfree(env->func_sets[i]);
 	kfree(env->func_sets);
+	kvfree(env->stack_inits);
 	bpf_diag_free(env);
 	kvfree(env->idoms);
 	kvfree(env);

index 87fad4076f0e..045c48bcf98c 100644
@@ -141,9 +141,19 @@ struct btf_kfunc_id_set {
 	btf_kfunc_filter_t filter;
 };
 
+/*
+ * Flags for struct btf_id_dtor_kfunc::flags: BPF_DTOR_KPTR destroys the kptr fields of a map
+ * element when it is freed, BPF_DTOR_CLEANUP may release a reference held by a program that
+ * is being unwound.
+ */
+#define BPF_DTOR_KPTR		(1U << 0)
+#define BPF_DTOR_CLEANUP	(1U << 1)
+
 struct btf_id_dtor_kfunc {
-	u32 btf_id;
-	u32 kfunc_btf_id;
+	u32 btf_id;		/* type the destructor applies to */
+	u32 kfunc_btf_id;	/* BTF id of the void (*)(void *) destructor */
+	u32 release_btf_id;	/* KF_RELEASE kfunc of the type, required for BPF_DTOR_CLEANUP */
+	u32 flags;
 };
 
 struct btf_struct_meta {

@@ -627,7 +637,8 @@ u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id,
 int register_btf_kfunc_id_set(enum bpf_prog_type prog_type,
 			      const struct btf_kfunc_id_set *s);
 int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset);
-s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id);
+const struct btf_id_dtor_kfunc *btf_find_dtor(struct btf **btfp, u32 btf_id);
+s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id, u32 flags);
 int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
 				struct module *owner);
 struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id);

@@ -718,7 +729,11 @@ static inline int register_btf_kfunc_id_set(enum bpf_prog_type prog_type,
 {
 	return 0;
 }
-static inline s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id)
+static inline const struct btf_id_dtor_kfunc *btf_find_dtor(struct btf **btfp, u32 btf_id)
+{
+	return NULL;
+}
+static inline s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id, u32 flags)
 {
 	return -ENOENT;
 }

136 lines not on this page · raw message

index 1336a4efa755..f552472d69b1 100644
@@ -511,14 +511,17 @@ static const struct btf_kfunc_id_set cpumask_kfunc_set = {
 BTF_ID_LIST(cpumask_dtor_ids)
 BTF_ID(struct, bpf_cpumask)
 BTF_ID(func, bpf_cpumask_release_dtor)
+BTF_ID(func, bpf_cpumask_release)
 
 static int __init cpumask_kfunc_init(void)
 {
 	int ret;
 	const struct btf_id_dtor_kfunc cpumask_dtors[] = {
 		{
-			.btf_id	      = cpumask_dtor_ids[0],
-			.kfunc_btf_id = cpumask_dtor_ids[1]
+			.btf_id	        = cpumask_dtor_ids[0],
+			.kfunc_btf_id   = cpumask_dtor_ids[1],
+			.release_btf_id = cpumask_dtor_ids[2],
+			.flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 

index 69bc29d0a4d4..cd65da3dffcf 100644
@@ -336,14 +336,17 @@ static const struct btf_kfunc_id_set crypt_kfunc_set = {
 BTF_ID_LIST(bpf_crypto_dtor_ids)
 BTF_ID(struct, bpf_crypto_ctx)
 BTF_ID(func, bpf_crypto_ctx_release_dtor)
+BTF_ID(func, bpf_crypto_ctx_release)
 
 static int __init crypto_kfunc_init(void)
 {
 	int ret;
 	const struct btf_id_dtor_kfunc bpf_crypto_dtors[] = {
 		{
-			.btf_id	      = bpf_crypto_dtor_ids[0],
-			.kfunc_btf_id = bpf_crypto_dtor_ids[1]
+			.btf_id	        = bpf_crypto_dtor_ids[0],
+			.kfunc_btf_id   = bpf_crypto_dtor_ids[1],
+			.release_btf_id = bpf_crypto_dtor_ids[2],
+			.flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 

index 0164f183aade..cbed0c8952a7 100644
@@ -5317,9 +5317,11 @@ static const struct btf_kfunc_id_set user_mem_kfunc_set = {
 BTF_ID_LIST(generic_dtor_ids)
 BTF_ID(struct, task_struct)
 BTF_ID(func, bpf_task_release_dtor)
+BTF_ID(func, bpf_task_release)
 #ifdef CONFIG_CGROUPS
 BTF_ID(struct, cgroup)
 BTF_ID(func, bpf_cgroup_release_dtor)
+BTF_ID(func, bpf_cgroup_release)
 #endif
 
 BTF_KFUNCS_START(common_btf_ids)

@@ -5429,13 +5431,17 @@ static int __init kfunc_init(void)
 	int ret;
 	const struct btf_id_dtor_kfunc generic_dtors[] = {
 		{
-			.btf_id       = generic_dtor_ids[0],
-			.kfunc_btf_id = generic_dtor_ids[1]
+			.btf_id         = generic_dtor_ids[0],
+			.kfunc_btf_id   = generic_dtor_ids[1],
+			.release_btf_id = generic_dtor_ids[2],
+			.flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 #ifdef CONFIG_CGROUPS
 		{
-			.btf_id       = generic_dtor_ids[2],
-			.kfunc_btf_id = generic_dtor_ids[3]
+			.btf_id         = generic_dtor_ids[3],
+			.kfunc_btf_id   = generic_dtor_ids[4],
+			.release_btf_id = generic_dtor_ids[5],
+			.flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 #endif
 	};

index d6c9dc62182e..0eb11688aaa4 100644
@@ -1888,19 +1888,25 @@ static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
 BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
 BTF_ID(struct, prog_test_ref_kfunc)
 BTF_ID(func, bpf_kfunc_call_test_release_dtor)
+BTF_ID(func, bpf_kfunc_call_test_release)
 BTF_ID(struct, prog_test_member)
 BTF_ID(func, bpf_kfunc_call_memb_release_dtor)
+BTF_ID(func, bpf_kfunc_call_memb_release)
 
 static int __init bpf_prog_test_run_init(void)
 {
 	const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = {
 		{
-		  .btf_id       = bpf_prog_test_dtor_kfunc_ids[0],
-		  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1]
+		  .btf_id         = bpf_prog_test_dtor_kfunc_ids[0],
+		  .kfunc_btf_id   = bpf_prog_test_dtor_kfunc_ids[1],
+		  .release_btf_id = bpf_prog_test_dtor_kfunc_ids[2],
+		  .flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 		{
-		  .btf_id	= bpf_prog_test_dtor_kfunc_ids[2],
-		  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3],
+		  .btf_id         = bpf_prog_test_dtor_kfunc_ids[3],
+		  .kfunc_btf_id   = bpf_prog_test_dtor_kfunc_ids[4],
+		  .release_btf_id = bpf_prog_test_dtor_kfunc_ids[5],
+		  .flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 	int ret;

index 25111712b944..e7fba11cdb12 100644
@@ -366,6 +366,7 @@ static const struct btf_kfunc_id_set ksock_kfunc_set = {
 BTF_ID_LIST(bpf_ksock_dtor_ids)
 BTF_ID(struct, bpf_ksock)
 BTF_ID(func, bpf_ksock_release_dtor)
+BTF_ID(func, bpf_ksock_release)
 
 static int __init bpf_ksock_kfunc_init(void)
 {

@@ -374,6 +375,8 @@ static int __init bpf_ksock_kfunc_init(void)
 		{
 			.btf_id = bpf_ksock_dtor_ids[0],
 			.kfunc_btf_id = bpf_ksock_dtor_ids[1],
+			.release_btf_id = bpf_ksock_dtor_ids[2],
+			.flags = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 

index 5691c13781a8..70e7a6dafc58 100644
@@ -453,15 +453,19 @@ static struct bpf_struct_ops bpf_Qdisc_ops = {
 	.owner = THIS_MODULE,
 };
 
-BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb_dtor)
+BTF_ID_LIST(bpf_sk_buff_dtor_ids)
+BTF_ID(func, bpf_kfree_skb_dtor)
+BTF_ID(func, bpf_kfree_skb)
 
 static int __init bpf_qdisc_kfunc_init(void)
 {
 	int ret;
 	const struct btf_id_dtor_kfunc skb_kfunc_dtors[] = {
 		{
-			.btf_id       = bpf_sk_buff_ids[0],
-			.kfunc_btf_id = bpf_sk_buff_dtor_ids[0]
+			.btf_id         = bpf_sk_buff_ids[0],
+			.kfunc_btf_id   = bpf_sk_buff_dtor_ids[0],
+			.release_btf_id = bpf_sk_buff_dtor_ids[1],
+			.flags          = BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 

index 41ff36f0ca77..891f777a265a 100644
@@ -1084,6 +1084,7 @@ BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids)
 BTF_ID_LIST(bpf_testmod_dtor_ids)
 BTF_ID(struct, bpf_testmod_ctx)
 BTF_ID(func, bpf_testmod_ctx_release_dtor)
+BTF_ID(func, bpf_testmod_ctx_release)
 
 static const struct btf_kfunc_id_set bpf_testmod_common_kfunc_set = {
 	.owner = THIS_MODULE,

@@ -2622,7 +2623,9 @@ static int bpf_testmod_init(void)
 	const struct btf_id_dtor_kfunc bpf_testmod_dtors[] = {
 		{
 			.btf_id		= bpf_testmod_dtor_ids[0],
-			.kfunc_btf_id	= bpf_testmod_dtor_ids[1]
+			.kfunc_btf_id	= bpf_testmod_dtor_ids[1],
+			.release_btf_id	= bpf_testmod_dtor_ids[2],
+			.flags		= BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
 	};
 	void **tramp;

index 80b497de2deb..16ccafbfcd6b 100644
@@ -1170,6 +1170,7 @@ static const struct file_operations wakeup_sources_stats_fops = {
 
 #ifdef CONFIG_BPF_SYSCALL
 #include <linux/btf.h>
+#include <linux/btf_ids.h>
 
 __bpf_kfunc_start_defs();
 

@@ -1202,6 +1203,12 @@ __bpf_kfunc void bpf_wakeup_sources_read_unlock(struct bpf_ws_lock *lock)
 	wakeup_sources_read_unlock((int)(long)lock - 1);
 }
 
+__bpf_kfunc void bpf_wakeup_sources_read_unlock_dtor(void *lock)
+{
+	bpf_wakeup_sources_read_unlock(lock);
+}
+CFI_NOSEAL(bpf_wakeup_sources_read_unlock_dtor);
+
 /**
  * bpf_wakeup_sources_get_head - Get the head of the wakeup sources list
  *

@@ -1224,9 +1231,25 @@ static const struct btf_kfunc_id_set wakeup_source_kfunc_set = {
 	.set   = &wakeup_source_kfunc_ids,
 };
 
+BTF_ID_LIST(wakeup_source_dtor_ids)
+BTF_ID(struct, bpf_ws_lock)
+BTF_ID(func, bpf_wakeup_sources_read_unlock_dtor)
+BTF_ID(func, bpf_wakeup_sources_read_unlock)
+
 static void __init wakeup_sources_bpf_init(void)
 {
-	if (register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &wakeup_source_kfunc_set))
+	const struct btf_id_dtor_kfunc wakeup_source_dtors[] = {
+		{
+			.btf_id         = wakeup_source_dtor_ids[0],
+			.kfunc_btf_id   = wakeup_source_dtor_ids[1],
+			.release_btf_id = wakeup_source_dtor_ids[2],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
+
+	if (register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &wakeup_source_kfunc_set) ||
+	    register_btf_id_dtor_kfuncs(wakeup_source_dtors, ARRAY_SIZE(wakeup_source_dtors),
+					THIS_MODULE))
 		pm_pr_dbg("Wakeup: failed to register BTF kfuncs\n");
 }
 #else

index 6b467c74601c..897bae3f8c1b 100644
@@ -357,6 +357,12 @@ hid_bpf_release_context(struct hid_bpf_ctx *ctx)
 	hid_put_device(hid);
 }
 
+__bpf_kfunc void hid_bpf_release_context_dtor(void *ctx)
+{
+	hid_bpf_release_context(ctx);
+}
+CFI_NOSEAL(hid_bpf_release_context_dtor);
+
 static int
 __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz,
 			  enum hid_report_type rtype, bool hw_request)

@@ -659,8 +665,21 @@ int hid_bpf_device_init(struct hid_device *hdev)
 }
 EXPORT_SYMBOL_GPL(hid_bpf_device_init);
 
+BTF_ID_LIST(hid_bpf_dtor_ids)
+BTF_ID(struct, hid_bpf_ctx)
+BTF_ID(func, hid_bpf_release_context_dtor)
+BTF_ID(func, hid_bpf_release_context)
+
 static int __init hid_bpf_init(void)
 {
+	const struct btf_id_dtor_kfunc hid_bpf_dtors[] = {
+		{
+			.btf_id         = hid_bpf_dtor_ids[0],
+			.kfunc_btf_id   = hid_bpf_dtor_ids[1],
+			.release_btf_id = hid_bpf_dtor_ids[2],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
 	int err;
 
 	/* Note: if we exit with an error any time here, we would entirely break HID, which

@@ -681,6 +700,10 @@ static int __init hid_bpf_init(void)
 		return 0;
 	}
 
+	err = register_btf_id_dtor_kfuncs(hid_bpf_dtors, ARRAY_SIZE(hid_bpf_dtors), THIS_MODULE);
+	if (err)
+		pr_warn("error while registering HID BPF destructors: %d\n", err);
+
 	return 0;
 }
 

index 44c6ea22233f..a198700a4394 100644
@@ -16,6 +16,7 @@ void hid_put_device(struct hid_device *hid);
 int hid_bpf_allocate_event_data(struct hid_device *hdev);
 void __hid_bpf_ops_destroy_device(struct hid_device *hdev);
 int hid_bpf_reconnect(struct hid_device *hdev);
+void hid_bpf_release_context_dtor(void *ctx);
 
 struct bpf_prog;
 

index 35ccd98a4de6..0c8e1d16a1ef 100644
@@ -63,6 +63,12 @@ __bpf_kfunc void bpf_put_file(struct file *file)
 	fput(file);
 }
 
+__bpf_kfunc void bpf_put_file_dtor(void *file)
+{
+	bpf_put_file(file);
+}
+CFI_NOSEAL(bpf_put_file_dtor);
+
 /**
  * bpf_path_d_path - resolve the pathname for the supplied path
  * @path: path to resolve the pathname for

@@ -584,15 +590,28 @@ static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {
 	.filter = bpf_fs_kfuncs_filter,
 };
 
+BTF_ID_LIST(bpf_fs_dtor_ids)
+BTF_ID(struct, file)
+BTF_ID(func, bpf_put_file_dtor)
+BTF_ID(func, bpf_put_file)
+
 static int __init bpf_fs_kfuncs_init(void)
 {
+	const struct btf_id_dtor_kfunc bpf_fs_dtors[] = {
+		{
+			.btf_id         = bpf_fs_dtor_ids[0],
+			.kfunc_btf_id   = bpf_fs_dtor_ids[1],
+			.release_btf_id = bpf_fs_dtor_ids[2],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
 	int ret;
 
 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fs_kfunc_set);
-	if (ret || !IS_ENABLED(CONFIG_BINFMT_MISC_BPF))
-		return ret;
-	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					 &bpf_fs_kfunc_set);
+	if (!ret && IS_ENABLED(CONFIG_BINFMT_MISC_BPF))
+		ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_fs_kfunc_set);
+	return ret ?: register_btf_id_dtor_kfuncs(bpf_fs_dtors, ARRAY_SIZE(bpf_fs_dtors),
+						  THIS_MODULE);
 }
 
 late_initcall(bpf_fs_kfuncs_init);

index 2b9adc4866bc..7f2db4e9afe0 100644
@@ -3301,6 +3301,19 @@ int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
 int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
 				   const struct bpf_map *map);
 
+/*
+ * void (*)(void *) wrappers of the open-coded iterator destroy kfuncs, which the BTF
+ * destructor registry refers to.
+ */
+void bpf_iter_num_destroy_dtor(void *it);
+void bpf_iter_task_destroy_dtor(void *it);
+void bpf_iter_task_vma_destroy_dtor(void *it);
+void bpf_iter_css_destroy_dtor(void *it);
+void bpf_iter_css_task_destroy_dtor(void *it);
+void bpf_iter_bits_destroy_dtor(void *it);
+void bpf_iter_kmem_cache_destroy_dtor(void *it);
+void bpf_iter_dmabuf_destroy_dtor(void *it);
+
 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
 				   struct bpf_func_state *caller,
 				   struct bpf_func_state *callee);

@@ -4105,6 +4118,7 @@ struct bpf_key *bpf_lookup_system_key(u64 id);
 struct bpf_key *bpf_lookup_keyring(void);
 bool bpf_keyring_enforced(void);
 void bpf_key_put(struct bpf_key *bkey);
+void bpf_key_put_dtor(void *bkey);
 int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
 			       const struct bpf_dynptr *sig_p,
 			       struct bpf_key *trusted_keyring);

index 2d0da478c8e0..60ab309452c5 100644
@@ -15,6 +15,7 @@ struct nf_conn___init {
 
 extern int register_nf_conntrack_bpf(void);
 extern void cleanup_nf_conntrack_bpf(void);
+void bpf_ct_release_dtor(void *nfct);
 
 #else
 

index 298ca9bb0083..561418052d33 100644
@@ -2378,6 +2378,7 @@ static inline int register_xfrm_interface_bpf(void)
 
 #if IS_ENABLED(CONFIG_DEBUG_INFO_BTF)
 int register_xfrm_state_bpf(void);
+void bpf_xdp_xfrm_state_release_dtor(void *x);
 #else
 static inline int register_xfrm_state_bpf(void)
 {

index f0b29a170935..c5cc7ed1b918 100644
@@ -846,4 +846,10 @@ __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
 	/* no-op */
 }
 
+__bpf_kfunc void bpf_iter_num_destroy_dtor(void *it)
+{
+	bpf_iter_num_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_num_destroy_dtor);
+
 __bpf_kfunc_end_defs();

index fd51fe3d92cc..d1298d0d2404 100644
@@ -372,4 +372,10 @@ __bpf_kfunc void bpf_iter_css_destroy(struct bpf_iter_css *it)
 {
 }
 
+__bpf_kfunc void bpf_iter_css_destroy_dtor(void *it)
+{
+	bpf_iter_css_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_css_destroy_dtor);
+
 __bpf_kfunc_end_defs();

index cd500248abd9..ac140a6cb570 100644
@@ -189,4 +189,10 @@ __bpf_kfunc void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it)
 		dma_buf_put(kit->dmabuf);
 }
 
+__bpf_kfunc void bpf_iter_dmabuf_destroy_dtor(void *it)
+{
+	bpf_iter_dmabuf_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_dmabuf_destroy_dtor);
+
 __bpf_kfunc_end_defs();

118 lines not on this page · raw message

index 3ae2158d767f..c629473a35ee 100644
@@ -100,6 +100,12 @@ __bpf_kfunc void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it)
 		kmem_cache_destroy(s);
 }
 
+__bpf_kfunc void bpf_iter_kmem_cache_destroy_dtor(void *it)
+{
+	bpf_iter_kmem_cache_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_kmem_cache_destroy_dtor);
+
 __bpf_kfunc_end_defs();
 
 struct bpf_iter__kmem_cache {

index 2ee7b3b3c10b..df7c89fb61d0 100644
@@ -1044,6 +1044,12 @@ __bpf_kfunc void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it)
 	}
 }
 
+__bpf_kfunc void bpf_iter_task_vma_destroy_dtor(void *it)
+{
+	bpf_iter_task_vma_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_task_vma_destroy_dtor);
+
 __bpf_kfunc_end_defs();
 
 #ifdef CONFIG_CGROUPS

@@ -1102,6 +1108,12 @@ __bpf_kfunc void bpf_iter_css_task_destroy(struct bpf_iter_css_task *it)
 	bpf_mem_free(&bpf_global_ma, kit->css_it);
 }
 
+__bpf_kfunc void bpf_iter_css_task_destroy_dtor(void *it)
+{
+	bpf_iter_css_task_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_css_task_destroy_dtor);
+
 __bpf_kfunc_end_defs();
 
 #endif /* CONFIG_CGROUPS */

@@ -1192,6 +1204,12 @@ __bpf_kfunc void bpf_iter_task_destroy(struct bpf_iter_task *it)
 {
 }
 
+__bpf_kfunc void bpf_iter_task_destroy_dtor(void *it)
+{
+	bpf_iter_task_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_task_destroy_dtor);
+
 __bpf_kfunc_end_defs();
 
 DEFINE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);

67 lines not on this page · raw message

index d8f579c28560..4cd37301df23 100644
@@ -82,6 +82,12 @@ __bpf_kfunc void bpf_put_mem_cgroup(struct mem_cgroup *memcg)
 	css_put(&memcg->css);
 }
 
+__bpf_kfunc void bpf_put_mem_cgroup_dtor(void *memcg)
+{
+	bpf_put_mem_cgroup(memcg);
+}
+CFI_NOSEAL(bpf_put_mem_cgroup_dtor);
+
 /**
  * bpf_mem_cgroup_vm_events - Read memory cgroup's vm event counter
  * @memcg: memory cgroup

@@ -235,8 +241,21 @@ static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = {
 	.set            = &bpf_memcontrol_reclaim_kfuncs,
 };
 
+BTF_ID_LIST(bpf_memcontrol_dtor_ids)
+BTF_ID(struct, mem_cgroup)
+BTF_ID(func, bpf_put_mem_cgroup_dtor)
+BTF_ID(func, bpf_put_mem_cgroup)
+
 static int __init bpf_memcontrol_init(void)
 {
+	const struct btf_id_dtor_kfunc bpf_memcontrol_dtors[] = {
+		{
+			.btf_id         = bpf_memcontrol_dtor_ids[0],
+			.kfunc_btf_id   = bpf_memcontrol_dtor_ids[1],
+			.release_btf_id = bpf_memcontrol_dtor_ids[2],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
 	int err;
 
 	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,

@@ -248,8 +267,15 @@ static int __init bpf_memcontrol_init(void)
 
 	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
 					&bpf_memcontrol_reclaim_kfunc_set);
-	if (err)
+	if (err) {
 		pr_warn("error registering bpf reclaim kfuncs: %d\n", err);
+		return err;
+	}
+
+	err = register_btf_id_dtor_kfuncs(bpf_memcontrol_dtors, ARRAY_SIZE(bpf_memcontrol_dtors),
+					  THIS_MODULE);
+	if (err)
+		pr_warn("error registering bpf memcontrol destructors: %d\n", err);
 
 	return err;
 }

index c2df7c948281..9d9d4e97ca4e 100644
@@ -457,6 +457,22 @@ __bpf_kfunc void bpf_ct_release(struct nf_conn *nfct)
 	nf_ct_put(nfct);
 }
 
+__bpf_kfunc void bpf_ct_release_dtor(void *nfct)
+{
+	bpf_ct_release(nfct);
+}
+CFI_NOSEAL(bpf_ct_release_dtor);
+
+/*
+ * An entry that was allocated and not inserted has no reference to put: it is
+ * freed, as bpf_ct_insert_entry() does with one it cannot insert.
+ */
+__bpf_kfunc void bpf_ct_free_dtor(void *nfct)
+{
+	nf_conntrack_free(nfct);
+}
+CFI_NOSEAL(bpf_ct_free_dtor);
+
 /* bpf_ct_set_timeout - Set timeout of allocated nf_conn
  *
  * Sets the default timeout of newly allocated nf_conn before insertion.

@@ -537,12 +553,34 @@ static const struct btf_kfunc_id_set nf_conntrack_kfunc_set = {
 	.set   = &nf_ct_kfunc_set,
 };
 
+BTF_ID_LIST(nf_ct_dtor_ids)
+BTF_ID(func, bpf_ct_release_dtor)
+BTF_ID(func, bpf_ct_release)
+BTF_ID(func, bpf_ct_free_dtor)
+BTF_ID(func, bpf_ct_insert_entry)
+
 int register_nf_conntrack_bpf(void)
 {
+	/* The only kfunc that takes an allocated entry off a program is bpf_ct_insert_entry() */
+	const struct btf_id_dtor_kfunc nf_ct_dtors[] = {
+		{
+			.btf_id         = btf_nf_conn_ids[0],
+			.kfunc_btf_id   = nf_ct_dtor_ids[0],
+			.release_btf_id = nf_ct_dtor_ids[1],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+		{
+			.btf_id         = btf_nf_conn_ids[1],
+			.kfunc_btf_id   = nf_ct_dtor_ids[2],
+			.release_btf_id = nf_ct_dtor_ids[3],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
 	int ret;
 
 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set);
+	ret = ret ?: register_btf_id_dtor_kfuncs(nf_ct_dtors, ARRAY_SIZE(nf_ct_dtors), THIS_MODULE);
 	if (!ret) {
 		mutex_lock(&nf_conn_btf_access_lock);
 		nfct_btf_struct_access = _nf_conntrack_btf_struct_access;

index 4180c317f9bc..560b444aee80 100644
@@ -115,6 +115,12 @@ __bpf_kfunc void bpf_xdp_xfrm_state_release(struct xfrm_state *x)
 	xfrm_state_put(x);
 }
 
+__bpf_kfunc void bpf_xdp_xfrm_state_release_dtor(void *x)
+{
+	bpf_xdp_xfrm_state_release(x);
+}
+CFI_NOSEAL(bpf_xdp_xfrm_state_release_dtor);
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(xfrm_state_kfunc_set)

@@ -127,8 +133,24 @@ static const struct btf_kfunc_id_set xfrm_state_xdp_kfunc_set = {
 	.set   = &xfrm_state_kfunc_set,
 };
 
+BTF_ID_LIST(xfrm_state_dtor_ids)
+BTF_ID(struct, xfrm_state)
+BTF_ID(func, bpf_xdp_xfrm_state_release_dtor)
+BTF_ID(func, bpf_xdp_xfrm_state_release)
+
 int __init register_xfrm_state_bpf(void)
 {
-	return register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP,
-					 &xfrm_state_xdp_kfunc_set);
+	const struct btf_id_dtor_kfunc xfrm_state_dtors[] = {
+		{
+			.btf_id         = xfrm_state_dtor_ids[0],
+			.kfunc_btf_id   = xfrm_state_dtor_ids[1],
+			.release_btf_id = xfrm_state_dtor_ids[2],
+			.flags          = BPF_DTOR_CLEANUP,
+		},
+	};
+	int ret;
+
+	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &xfrm_state_xdp_kfunc_set);
+	return ret ?: register_btf_id_dtor_kfuncs(xfrm_state_dtors, ARRAY_SIZE(xfrm_state_dtors),
+						  THIS_MODULE);
 }

index 891f777a265a..db1a7073aee3 100644
@@ -281,6 +281,12 @@ __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it)
 	it->cnt = 0;
 }
 
+__bpf_kfunc void bpf_iter_testmod_seq_destroy_dtor(void *it)
+{
+	bpf_iter_testmod_seq_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_testmod_seq_destroy_dtor);
+
 __bpf_kfunc int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it)
 {
 	it->remaining = 2;

@@ -302,6 +308,12 @@ __bpf_kfunc void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_curren
 	it->remaining = 0;
 }
 
+__bpf_kfunc void bpf_iter_testmod_current_destroy_dtor(void *it)
+{
+	bpf_iter_testmod_current_destroy(it);
+}
+CFI_NOSEAL(bpf_iter_testmod_current_destroy_dtor);
+
 __bpf_kfunc void bpf_kfunc_common_test(void)
 {
 }

@@ -1085,6 +1097,12 @@ BTF_ID_LIST(bpf_testmod_dtor_ids)
 BTF_ID(struct, bpf_testmod_ctx)
 BTF_ID(func, bpf_testmod_ctx_release_dtor)
 BTF_ID(func, bpf_testmod_ctx_release)
+BTF_ID(struct, bpf_iter_testmod_seq)
+BTF_ID(func, bpf_iter_testmod_seq_destroy_dtor)
+BTF_ID(func, bpf_iter_testmod_seq_destroy)
+BTF_ID(struct, bpf_iter_testmod_current)
+BTF_ID(func, bpf_iter_testmod_current_destroy_dtor)
+BTF_ID(func, bpf_iter_testmod_current_destroy)
 
 static const struct btf_kfunc_id_set bpf_testmod_common_kfunc_set = {
 	.owner = THIS_MODULE,

@@ -2627,6 +2645,18 @@ static int bpf_testmod_init(void)
 			.release_btf_id	= bpf_testmod_dtor_ids[2],
 			.flags		= BPF_DTOR_KPTR | BPF_DTOR_CLEANUP,
 		},
+		{
+			.btf_id		= bpf_testmod_dtor_ids[3],
+			.kfunc_btf_id	= bpf_testmod_dtor_ids[4],
+			.release_btf_id	= bpf_testmod_dtor_ids[5],
+			.flags		= BPF_DTOR_CLEANUP,
+		},
+		{
+			.btf_id		= bpf_testmod_dtor_ids[6],
+			.kfunc_btf_id	= bpf_testmod_dtor_ids[7],
+			.release_btf_id	= bpf_testmod_dtor_ids[8],
+			.flags		= BPF_DTOR_CLEANUP,
+		},
 	};
 	void **tramp;
 	int ret;

index c9f851bd5003..418bda24020c 100644
@@ -1055,6 +1055,16 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 	/* Stack must be multiples of 16B */
 	ctx->stack_size = round_up(prog->aux->stack_depth, 16);
 
+	/*
+	 * Where the exception unwinder finds this frame's BPF stack: the
+	 * private stack, or below the frame record, the tail call count pair
+	 * and the pairs push_callee_regs() pushed.
+	 */
+	if (ctx->priv_sp_used)
+		prog->aux->priv_frame_off = PRIV_STACK_GUARD_SZ + ctx->stack_size;
+	else
+		prog->aux->frame_fp_off = -16 - 16 * DIV_ROUND_UP(ctx->nr_used_callee_reg, 2);
+
 	if (ctx->fp_used) {
 		if (ctx->priv_sp_used) {
 			/* Set up private stack pointer */

@@ -3904,6 +3914,12 @@ bool bpf_jit_supports_exceptions(void)
 	return true;
 }
 
+/* build_prologue() records where each frame keeps its BPF stack. */
+bool bpf_jit_supports_exception_cleanup(void)
+{
+	return true;
+}
+
 bool bpf_jit_supports_arena(void)
 {
 	return true;

index 9b7466ca1a82..9170cbe3fa16 100644
@@ -2425,6 +2425,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
 	if (priv_stack_ptr) {
 		priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
+		bpf_prog->aux->priv_frame_off = PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
 		stack_depth = 0;
 	}
 

@@ -5066,6 +5067,12 @@ bool bpf_jit_supports_exceptions(void)
 	return IS_ENABLED(CONFIG_UNWINDER_ORC);
 }
 
+/* On x86-64 R10 is the frame pointer; do_jit() records where a private stack has it. */
+bool bpf_jit_supports_exception_cleanup(void)
+{
+	return bpf_jit_supports_exceptions();
+}
+
 bool bpf_jit_supports_private_stack(void)
 {
 	return true;

index 7f2db4e9afe0..17b13b16899d 100644
@@ -2010,6 +2010,10 @@ struct bpf_prog_aux {
 	 * by the JIT; consumed by the exception unwinder.
 	 */
 	s16 callee_reg_off[BPF_THROW_NR_CALLEE_SAVED];
+	/* BPF frame pointer relative to the native frame pointer the stack walker reports */
+	s16 frame_fp_off;
+	/* With a private stack, the frame's R10 is this far into this CPU's priv_stack_ptr. */
+	u32 priv_frame_off;
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif

index 61448be53f8d..08ad906f649f 100644
@@ -1391,6 +1391,7 @@ bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_arena_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
 bool bpf_jit_supports_exceptions(void);
+bool bpf_jit_supports_exception_cleanup(void);
 bool bpf_jit_supports_ptr_xchg(void);
 bool bpf_jit_supports_arena(void);
 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);

index 7a25d1cb2c30..59478a08b247 100644
@@ -3896,6 +3896,16 @@ bool __weak bpf_jit_supports_exceptions(void)
 	return false;
 }
 
+/*
+ * Does the JIT record where each frame keeps its BPF stack, see
+ * bpf_prog_aux::frame_fp_off, so that the unwinder can release what a frame
+ * holds when bpf_throw() discards it?
+ */
+bool __weak bpf_jit_supports_exception_cleanup(void)
+{
+	return false;
+}
+
 bool __weak bpf_jit_supports_private_stack(void)
 {
 	return false;

index 7a2b6507f562..c18efe4bd54a 100644
@@ -327,6 +327,12 @@ struct bpf_reference_state {
 	 * is used purely to inform the user of a reference leak.
 	 */
 	int insn_idx;
+	/*
+	 * The frame that owns the reference: the one that acquired it, or the
+	 * caller a returning frame handed it to. -1 for what the callers of a
+	 * function verified on its own hold, which are not in the state.
+	 */
+	s8 frameno;
 	union {
 		/* For REF_TYPE_PTR */
 		int parent_id;

@@ -503,6 +509,13 @@ struct bpf_func_state {
 	u32 entry_refs;
 	u32 entry_rcu_locks;
 	u32 entry_preempt_locks;
+	/*
+	 * How many of the RCU read and preemption disabled sections held when the
+	 * frame was entered it left, or a callee of it did: a frame that returns
+	 * after that is in fewer sections than it was called in.
+	 */
+	u32 rcu_released_below;
+	u32 preempt_released_below;
 
 	/* The following fields should be last. See copy_func_state() */
 	/* The state of the stack. Each element of the array describes BPF_REG_SIZE

index 0684ac1f4f22..6c71ceb1dd4c 100644
@@ -1545,6 +1545,7 @@ static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_e
 	if (err)
 		return NULL;
 	state->refs[new_ofs].insn_idx = insn_idx;
+	state->refs[new_ofs].frameno = state->curframe;
 
 	return &state->refs[new_ofs];
 }

@@ -1625,7 +1626,11 @@ static void release_reference_state(struct bpf_verifier_state *state, int idx)
 	return;
 }
 
-/* bpf_rcu_read_unlock() or bpf_preempt_enable() is done, see release_reference_state() */
+/*
+ * bpf_rcu_read_unlock() or bpf_preempt_enable() is done, see release_reference_state().
+ * A frame that was entered with more sections than are held now has left one of its
+ * callers', and counts that for its exit to tell.
+ */
 static void release_context_state(struct bpf_verifier_state *state)
 {
 	struct bpf_func_state *frame;

@@ -1633,6 +1638,11 @@ static void release_context_state(struct bpf_verifier_state *state)
 
 	for (fr = 0; fr <= state->curframe; fr++) {
 		frame = state->frame[fr];
+		if (frame->entry_rcu_locks > state->active_rcu_locks)
+			frame->rcu_released_below += frame->entry_rcu_locks - state->active_rcu_locks;
+		if (frame->entry_preempt_locks > state->active_preempt_locks)
+			frame->preempt_released_below += frame->entry_preempt_locks -
+							 state->active_preempt_locks;
 		frame->entry_rcu_locks = min(frame->entry_rcu_locks, state->active_rcu_locks);
 		frame->entry_preempt_locks = min(frame->entry_preempt_locks,
 						 state->active_preempt_locks);

@@ -11709,6 +11719,8 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
 		entry.refs[i].id = inherited_id(state, entry.refs[i].id);
 		if (entry.refs[i].type == REF_TYPE_PTR)
 			entry.refs[i].parent_id = inherited_id(state, entry.refs[i].parent_id);
+		/* the frames that own what is inherited are not in the state */
+		entry.refs[i].frameno = -1;
 	}
 	entry.active_irq_id = inherited_id(state, entry.active_irq_id);
 	entry.active_lock_id = inherited_id(state, entry.active_lock_id);

@@ -11878,11 +11890,12 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
 		reg->parent_id = fresh_id(env, idmap, reg->parent_id);
 		reg->map_uid = fresh_id(env, idmap, reg->map_uid);
 	}
-	/* those that were held are the first */
+	/* those that were held are the first; what the function took is the caller's now */
 	for (i = nheld; i < caller->acquired_refs; i++) {
 		caller->refs[i].id = fresh_id(env, idmap, caller->refs[i].id);
 		if (caller->refs[i].type == REF_TYPE_PTR)
 			caller->refs[i].parent_id = fresh_id(env, idmap, caller->refs[i].parent_id);
+		caller->refs[i].frameno = caller->curframe;
 	}
 	caller->active_irq_id = fresh_id(env, idmap, caller->active_irq_id);
 	caller->active_lock_id = fresh_id(env, idmap, caller->active_lock_id);

@@ -12688,6 +12701,11 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 		return err;
 	invalidate_stack_slices(env, callee->frameno, INT_MIN, 0);
 
+	/* What the frame holds when it returns is the caller's to release. */
+	for (i = 0; i < state->acquired_refs; i++)
+		if (state->refs[i].frameno == callee->frameno)
+			state->refs[i].frameno = caller->frameno;
+
 	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
 	 * there function call logic would reschedule callback visit.
 	 */

index 17b13b16899d..e7cce9080531 100644
@@ -1829,6 +1829,72 @@ struct bpf_jit_poke_descriptor {
 	u32 insn_idx;
 };
 
+/*
+ * Frame descriptors: the hidden stack slots in which the frames of a function
+ * keep the handles of what they hold, and how the unwinder releases them, see
+ * kernel/bpf/frame_desc.c.
+ */
+/* Kinds of resources the unwinder can release. */
+enum bpf_resource_kind {
+	BPF_RESOURCE_OBJ,		/* bpf_obj_new() object: bpf_obj_drop_impl(p, meta) */
+	BPF_RESOURCE_PERCPU_OBJ,	/* bpf_percpu_obj_new() object: bpf_percpu_obj_drop_impl() */
+	BPF_RESOURCE_KPTR,		/* referenced kernel object: the BTF registry's dtor(p) */
+	BPF_RESOURCE_SOCK,		/* socket acquired by a helper: bpf_sk_release() */
+	BPF_RESOURCE_RINGBUF,		/* ringbuf record, also of a dynptr: bpf_ringbuf_discard() */
+	BPF_RESOURCE_DYNPTR_FILE,	/* reader of a file dynptr: bpf_dynptr_file_free() */
+	BPF_RESOURCE_ITER,		/* open-coded iterator on the stack: its destroy kfunc */
+	BPF_RESOURCE_SPIN_LOCK,		/* bpf_spin_lock() held: bpf_spin_unlock() */
+	BPF_RESOURCE_RES_LOCK,		/* bpf_res_spin_lock() held: bpf_res_spin_unlock() */
+	BPF_RESOURCE_IRQ_FLAG,		/* saved IRQ flags on the stack: bpf_local_irq_restore() */
+	BPF_RESOURCE_RCU,		/* bpf_rcu_read_lock() sections: unlock each */
+	BPF_RESOURCE_PREEMPT,		/* bpf_preempt_disable() sections: enable each */
+};
+
+/* A resource class of a program; descriptor entries refer to it by index. */
+struct bpf_resource {
+	u8 kind;	/* enum bpf_resource_kind */
+	bool sleeps;	/* the destructor may sleep */
+	bool rcu;	/* the destructor needs RCU protection */
+	u32 btf_id;
+	struct btf *btf;
+	union {
+		void (*dtor)(void *);
+		struct btf_struct_meta *meta;
+	};
+};
+
+/*
+ * The slot holds a count of sections instead of a handle, signed: the
+ * sections the frame entered less those of its callers it left.
+ */
+#define BPF_FRAME_ENT_COUNTER	BIT(0)
+
+/*
+ * One hidden slot of a frame: it holds the handle of a resource of the class
+ * the frame holds, or 0. The handle is what the release takes: the pointer
+ * to an object, to a ringbuf record or to a lock, or the address of an
+ * iterator or of saved IRQ flags on the stack of some frame.
+ */
+struct bpf_frame_desc_entry {
+	u16 res;	/* index into the program's resource classes */
+	s16 off;	/* the slot, relative to the frame pointer */
+	u16 flags;
+};
+
+struct bpf_frame_desc {
+	u32 size;	/* total size in bytes, including this header */
+	u32 nent;	/* number of entries that follow */
+	struct bpf_frame_desc_entry ent[];
+};
+
+/* Owned by the main program and shared by its subprograms. */
+struct bpf_frame_desc_pool {
+	u32 cnt;
+	u32 resource_cnt;
+	struct bpf_resource *resources;
+	struct bpf_frame_desc *desc[];
+};
+
 /* reg_type info for ctx arguments */
 struct bpf_ctx_arg_aux {
 	u32 offset;

@@ -2053,6 +2119,14 @@ struct bpf_prog_aux {
 	struct module *mod;
 	u32 num_exentries;
 	struct exception_table_entry *extable;
+	/*
+	 * The frame descriptor of this JITed function in the pool of the main
+	 * program, plus one, or 0: the hidden slots its frames keep the handles
+	 * of their resources in, see frame_desc.c. The main and sub progs share
+	 * the pool, which is freed with the main prog only.
+	 */
+	u32 frame_desc;
+	struct bpf_frame_desc_pool *frame_desc_pool;
 	union {
 		struct work_struct work;
 		struct rcu_head	rcu;

@@ -2999,6 +3073,12 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
 void bpf_obj_cancel_fields(struct bpf_map *map, void *obj);
 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
+void bpf_obj_drop_impl(void *p__alloc, void *meta__ign);
+void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign);
+void bpf_rcu_read_unlock(void);
+void bpf_preempt_enable(void);
+void bpf_local_irq_restore(unsigned long *flags__irq_flag);
+void bpf_res_spin_unlock(struct bpf_res_spin_lock *lock);
 
 struct bpf_map *bpf_map_get(u32 ufd);
 struct bpf_map *bpf_map_get_with_uref(u32 ufd);

@@ -4218,6 +4298,7 @@ extern const struct bpf_func_proto bpf_ringbuf_output_proto;
 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
 extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
 extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
+extern const struct bpf_func_proto bpf_sk_release_proto;
 extern const struct bpf_func_proto bpf_ringbuf_query_proto;
 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;

@@ -4508,11 +4589,21 @@ void bpf_insn_array_remap(struct bpf_map *map, const u32 *new_off, const u32 *ne
 
 #ifdef CONFIG_BPF_SYSCALL
 void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image);
+void bpf_unwind_release(const struct bpf_resource *res, void *handle);
+void bpf_dynptr_file_free(void *reader);
+void bpf_unwind_release_frames(void);
+void bpf_frame_desc_pool_free(struct bpf_frame_desc_pool *pool);
 #else
 static inline void
 bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
 {
 }
+static inline void bpf_frame_desc_pool_free(struct bpf_frame_desc_pool *pool)
+{
+}
+static inline void bpf_unwind_release_frames(void)
+{
+}
 #endif
 
 static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type)

index c18efe4bd54a..98d26a4c4a91 100644
@@ -1035,6 +1035,8 @@ struct bpf_subprog_info {
 	 */
 	u32 exit_idx;
 	struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
+	/* the frame descriptor of this subprogram in the pool of the program, plus one; see frame_desc.c */
+	u32 frame_desc;
 	u16 stack_depth; /* max. stack depth used by this function */
 	/* bytes below the verified stack depth reserved for hidden slots, see bpf_reserve_stack_slots() */
 	u16 stack_extra;

@@ -1293,6 +1295,8 @@ struct bpf_verifier_env {
 	/* What a pointer that was read from a table of functions may point to. */
 	struct bpf_func_set **func_sets;
 	u32 func_set_cnt;
+	/* frame descriptors of the subprograms, handed to prog->aux once verification succeeds */
+	struct bpf_frame_desc_pool *frame_desc_pool;
 	bool has_callx;
 	/*
 	 * Call graph edges created by callx instructions. A bitmap of

index 59478a08b247..80321ab16b32 100644
@@ -3459,6 +3459,7 @@ static void bpf_prog_free_images(struct work_struct *work)
 	int i;
 
 	aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork);
+	bpf_frame_desc_pool_free(aux->frame_desc_pool);
 	for (i = 0; i < aux->real_func_cnt; i++) {
 		/* We can just unlink the subprog poke descriptor table as
 		 * it was originally linked to the main program and is also

index 04d89113e596..4e46f8931775 100644
@@ -2047,6 +2047,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
+		/* The pool is freed with the main prog. */
+		func[i]->aux->frame_desc = env->subprog_info[i].frame_desc;
+		func[i]->aux->frame_desc_pool = env->frame_desc_pool;
 		func[i]->aux->arena = prog->aux->arena;
 		func[i]->aux->arena_stack_size = prog->aux->arena_stack_size;
 		func[i]->aux->arena_stack_entry = env->subprog_info[i].arena_stack_entry;

index cc6d3a6a4de0..793eb457fb62 100644
@@ -3492,6 +3492,8 @@ notrace void bpf_throw_unwind(struct bpf_throw_regs *regs, u64 cookie)
 	 */
 	if (regs)
 		memcpy(ctx.saved, &regs->r[BPF_REG_6], sizeof(ctx.saved));
+	/* What the frames about to be discarded hold is released first. */
+	bpf_unwind_release_frames();
 	arch_bpf_stack_walk(bpf_stack_walker, &ctx);
 	WARN_ON_ONCE(!ctx.aux);
 	if (ctx.aux)

@@ -5136,6 +5138,18 @@ int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags, struct bpf_dynp
 	return make_file_dynptr(file, flags, true, (struct bpf_dynptr_kern *)ptr__uninit);
 }
 
+/*
+ * Free the reader of a file dynptr, which the dynptr and its clones point
+ * at: what the unwinder does for a dynptr the program did not discard.
+ */
+void bpf_dynptr_file_free(void *reader)
+{
+	struct bpf_dynptr_file_impl *df = reader;
+
+	freader_cleanup(&df->freader);
+	kfree_nolock(df);
+}
+
 __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)
 {
 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)dynptr;

@@ -5144,8 +5158,7 @@ __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)
 	if (!df)
 		return 0;
 
-	freader_cleanup(&df->freader);
-	kfree_nolock(df);
+	bpf_dynptr_file_free(df);
 	bpf_dynptr_set_null(ptr);
 	return 0;
 }

243 lines not on this page · raw message

index 6c71ceb1dd4c..cf3cfc734ed4 100644
@@ -25695,6 +25695,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (env->prog->type == BPF_PROG_TYPE_EXT)
 		env->prog->expected_attach_type = 0;
 
+	/* The program owns the frame descriptors from here on. */
+	env->prog->aux->frame_desc_pool = env->frame_desc_pool;
+	env->frame_desc_pool = NULL;
+
 	env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
 
 err_release_maps:

@@ -25731,6 +25735,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	release_btfs(env);
 err_free_env:
 	bpf_free_subprog_jts(env);
+	bpf_frame_desc_pool_free(env->frame_desc_pool);
 	if (env->insn_aux_data)
 		bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len);
 	vfree(env->insn_aux_data);

index 9245009d9f1e..9f59b1c002d6 100644
@@ -7577,7 +7577,7 @@ BPF_CALL_1(bpf_sk_release, struct sock *, sk)
 	return 0;
 }
 
-static const struct bpf_func_proto bpf_sk_release_proto = {
+const struct bpf_func_proto bpf_sk_release_proto = {
 	.func		= bpf_sk_release,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,

index 98d26a4c4a91..5cbac24102f4 100644
@@ -333,6 +333,14 @@ struct bpf_reference_state {
 	 * function verified on its own hold, which are not in the state.
 	 */
 	s8 frameno;
+	/*
+	 * The resource class the unwinder releases it as, plus one, see
+	 * frame_desc.c; 0 for a reference the frames keep no handle of, which
+	 * a throw may not be reached with.
+	 */
+	u16 res;
+	/* The acquisition whose slot holds its handle, plus one, see struct bpf_res_acq. */
+	u16 acq;
 	union {
 		/* For REF_TYPE_PTR */
 		int parent_id;

@@ -908,6 +916,13 @@ struct bpf_insn_aux_data {
 	u64 jump_target:1;
 
 	unsigned int orig_idx; /* original instruction index, initialized once */
+	/*
+	 * Some visit of the call acquired nothing a hidden slot can hold where
+	 * another may have acquired a resource, see frame_desc.c.
+	 */
+	bool res_none;
+	/* Some visit of the call left an R0 that is not a resource. */
+	bool res_none_r0;
 	/* registers alive before this instruction. */
 	u16 live_regs_before;
 	/*

@@ -1037,6 +1052,8 @@ struct bpf_subprog_info {
 	struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
 	/* the frame descriptor of this subprogram in the pool of the program, plus one; see frame_desc.c */
 	u32 frame_desc;
+	/* hidden slots: the counts of the sections the frame entered, by kind; 0 when it has none */
+	s16 section_off[2];
 	u16 stack_depth; /* max. stack depth used by this function */
 	/* bytes below the verified stack depth reserved for hidden slots, see bpf_reserve_stack_slots() */
 	u16 stack_extra;

@@ -1224,6 +1241,42 @@ void bpf_patch_image_install(struct bpf_prog *prog, const struct bpf_patch_image
 /* single container for all structs
  * one verifier_env per bpf_check() call
  */
+#define BPF_RES_F_ARG		BIT(0)	/* the handle is the pointer the call takes in the register */
+#define BPF_RES_F_MAY_FAIL	BIT(1)	/* acquired only when the call returns 0 */
+#define BPF_RES_F_DEREF		BIT(2)	/* the handle is the first word of a dynptr on the stack */
+#define BPF_RES_F_CTX		BIT(3)	/* a __ref argument the kernel passes the main program */
+
+/*
+ * A call that acquires a resource, see frame_desc.c: the handle of what it
+ * acquired is kept in a hidden slot of the frame that runs the call, one slot
+ * per acquiring call.
+ */
+struct bpf_res_acq {
+	u32 insn;	/* the insn as the main pass named it; the argument with BPF_RES_F_CTX */
+	u16 res;	/* the class of what it acquires */
+	u8 reg;		/* R0 for what the call returns, else the argument that is the handle */
+	u8 flags;	/* BPF_RES_F_* */
+	s16 adj;	/* R0 points that many bytes into the object: the node of a list */
+	s16 obj_off;	/* BPF_RES_F_DEREF: the dynptr, relative to R10; BPF_RES_F_CTX: in the context */
+	s16 off;	/* the slot, relative to R10 */
+};
+
+/* A call that releases what an acquisition acquired: the slot of the acquisition is cleared. */
+struct bpf_res_rel {
+	u32 insn;
+	u16 acq;	/* index of the acquisition */
+	u8 reg;		/* the argument the call releases */
+	u8 flags;	/* BPF_RES_F_DEREF */
+	s16 adj;	/* the argument points that many bytes into the object */
+};
+
+/* A call that enters or leaves a section, for the count of its frame, see frame_desc.c. */
+struct bpf_section_insn {
+	u32 insn;
+	u8 kind;	/* 0 RCU, 1 preemption */
+	s8 delta;
+};
+
 /* Initialization of a hidden stack slot, emitted in the subprogram's prologue. */
 struct bpf_stack_init {
 	u32 subprog;

@@ -1258,6 +1311,19 @@ struct bpf_verifier_env {
 	u32 hidden_subprog_cnt;		/* number of hidden subprogs */
 	struct bpf_stack_init *stack_inits; /* hidden slot initializations, see bpf_stack_slot_init() */
 	u32 stack_init_cnt;
+	struct bpf_resource *resources;	/* resource classes of frame descriptors */
+	u32 resource_cnt;
+	struct bpf_res_acq *res_acqs;		/* see frame_desc.c */
+	u32 res_acq_cnt;
+	struct bpf_res_rel *res_rels;
+	u32 res_rel_cnt;
+	struct bpf_section_insn *section_insns;
+	u32 section_insn_cnt;
+	int release_regno;			/* the argument the current call releases, or -1 */
+	u32 frame_desc_slots;			/* hidden slots that hold handles and counts */
+	u32 frame_desc_insns;			/* instructions keeping them up to date */
+	u32 frame_desc_bytes;			/* descriptors and resource classes */
+	bool describe_frames;			/* frame descriptors are generated: the program may throw */
 	int exception_callback_subprog;
 	bool explore_alu_limits;
 	bool allow_ptr_leaks;

@@ -1929,6 +1995,7 @@ u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t);
 
 int bpf_find_subprog(struct bpf_verifier_env *env, int off);
 bool bpf_is_throw_kfunc(struct bpf_insn *insn);
+bool bpf_nmi_unsafe_prog(struct bpf_verifier_env *env);
 int bpf_compute_const_regs(struct bpf_verifier_env *env);
 int bpf_prune_dead_branches(struct bpf_verifier_env *env);
 int bpf_check_cfg(struct bpf_verifier_env *env);

@@ -2202,7 +2269,24 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
 /* Functions in fixups.c, called from bpf_check() */
 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);
 int bpf_reserve_stack_slots(struct bpf_verifier_env *env, int subprog, int nr, s16 *off);
+int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta);
 int bpf_stack_slot_init(struct bpf_verifier_env *env, int subprog, s16 off, s32 imm);
+
+/* Functions in frame_desc.c */
+int bpf_frame_desc_record_acquire(struct bpf_verifier_env *env, int idx, int regno, bool after,
+				  bool may_fail);
+void bpf_frame_desc_record_none(struct bpf_verifier_env *env, const struct bpf_reg_state *r0);
+int bpf_frame_desc_record_release(struct bpf_verifier_env *env, int idx);
+int bpf_frame_desc_record_ctx_arg(struct bpf_verifier_env *env, int idx, int argno);
+int bpf_frame_desc_record_section(struct bpf_verifier_env *env, u8 kind, s8 delta);
+int bpf_frame_desc_record_exit(struct bpf_verifier_env *env);
+int bpf_frame_desc_record_return(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+				 int idx, int callsite, const struct bpf_reg_state *r0);
+int bpf_frame_desc_resolve(struct bpf_verifier_env *env);
+int bpf_frame_desc_emit(struct bpf_verifier_env *env);
+int bpf_frame_desc_build(struct bpf_verifier_env *env);
+void bpf_frame_desc_reset(struct bpf_verifier_env *env);
+void bpf_frame_desc_env_free(struct bpf_verifier_env *env);
 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
 void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);

index 4e46f8931775..984d6287241a 100644
@@ -553,7 +553,7 @@ int bpf_patch_list_commit(struct bpf_verifier_env *env)
  * insn was moved down by delta insns inside its own patch. Operands relative
  * to the pc that point in front of the old position did not move with it.
  */
-static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta)
+int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta)
 {
 	u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
 	s64 off = insn->imm, off_min = S32_MIN;

1,186 lines not on this page · raw message

index cc4494aef187..ba1176dc868e 100644
@@ -462,6 +462,10 @@ static bool join_func(struct bpf_join_state *js, struct bpf_func_state *fd,
 		    fa->entry_rcu_locks != fb->entry_rcu_locks ||
 		    fa->entry_preempt_locks != fb->entry_preempt_locks)
 			return false;
+		/* a frame that left a section of its callers hands its count over, see frame_desc.c */
+		if (env->describe_frames && (fa->rcu_released_below != fb->rcu_released_below ||
+					     fa->preempt_released_below != fb->preempt_released_below))
+			return false;
 		if (!join_loop_stack(fa, fb))
 			return false;
 	}

@@ -523,7 +527,7 @@ static bool join_func(struct bpf_join_state *js, struct bpf_func_state *fd,
 }
 
 /* Things that can't be different in two states that are joined */
-static bool join_check_state(const struct bpf_verifier_state *a,
+static bool join_check_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *a,
 			     const struct bpf_verifier_state *b)
 {
 	int i;

@@ -546,6 +550,16 @@ static bool join_check_state(const struct bpf_verifier_state *a,
 	for (i = 0; i < a->acquired_refs; i++) {
 		if (a->refs[i].type != b->refs[i].type)
 			return false;
+		/*
+		 * The frames of a program that may throw keep the handle of a
+		 * reference in the slot of the call that acquired it, in the frame
+		 * that holds it, see frame_desc.c, and the result keeps those of
+		 * 'a'.
+		 */
+		if (env->describe_frames &&
+		    (a->refs[i].frameno != b->refs[i].frameno || a->refs[i].res != b->refs[i].res ||
+		     (a->refs[i].frameno >= 0 && a->refs[i].acq != b->refs[i].acq)))
+			return false;
 		switch (a->refs[i].type) {
 		case REF_TYPE_PTR:
 		case REF_TYPE_IRQ:

@@ -632,7 +646,7 @@ int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst
 	js->npairs = 0;
 	js->pairs_overflow = false;
 
-	if (!join_check_state(a, b))
+	if (!join_check_state(env, a, b))
 		return 1;
 	err = bpf_copy_verifier_state(dst, a);
 	if (err)

index efc0870b85c2..8f357fee6104 100644
@@ -562,8 +562,8 @@ static bool stack_arg_safe(struct bpf_verifier_env *env, struct bpf_func_state *
 	return true;
 }
 
-static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur,
-		    struct bpf_idmap *idmap)
+static bool refsafe(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
+		    struct bpf_verifier_state *cur, struct bpf_idmap *idmap)
 {
 	int i;
 

@@ -590,6 +590,19 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c
 		if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
 		    old->refs[i].type != cur->refs[i].type)
 			return false;
+		/*
+		 * The frames of a program that may throw keep the handle of a
+		 * reference in the slot of the call that acquired it, in the frame
+		 * that holds it, see frame_desc.c: a state doesn't stand for one
+		 * that holds the reference as another class, in another frame, or
+		 * from another call. What the callers of a function verified on its
+		 * own hold has no slot in its frames.
+		 */
+		if (env->describe_frames &&
+		    (old->refs[i].frameno != cur->refs[i].frameno ||
+		     old->refs[i].res != cur->refs[i].res ||
+		     (old->refs[i].frameno >= 0 && old->refs[i].acq != cur->refs[i].acq)))
+			return false;
 		switch (old->refs[i].type) {
 		case REF_TYPE_PTR:
 			if (!check_ids(old->refs[i].parent_id, cur->refs[i].parent_id, idmap))

@@ -695,6 +708,10 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
 	    old->entry_rcu_locks != cur->entry_rcu_locks ||
 	    old->entry_preempt_locks != cur->entry_preempt_locks)
 		return false;
+	/* a frame that left a section of its callers hands its count over, see frame_desc.c */
+	if (env->describe_frames && (old->rcu_released_below != cur->rcu_released_below ||
+				     old->preempt_released_below != cur->preempt_released_below))
+		return false;
 
 	for (i = 0; i < MAX_BPF_REG; i++)
 		if (((1 << i) & live_regs) &&

@@ -748,7 +765,7 @@ bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state
 	if (old->unwound != cur->unwound || old->changed_pkt_data != cur->changed_pkt_data)
 		return false;
 
-	if (!refsafe(old, cur, &env->idmap_scratch))
+	if (!refsafe(env, old, cur, &env->idmap_scratch))
 		return false;
 
 	/* for states to be equal callsites have to be the same

586 lines not on this page · raw message

index 22503cf62e9f..f02882224a03 100644
@@ -133,6 +133,7 @@ __noinline static int subprog_lock(struct __sk_buff *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("function calls are not allowed while holding a lock")
 int reject_with_lock(void *ctx)
 {

@@ -142,6 +143,7 @@ int reject_with_lock(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("function calls are not allowed while holding a lock")
 int reject_subprog_with_lock(void *ctx)
 {

@@ -149,6 +151,7 @@ int reject_subprog_with_lock(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region")
 int reject_with_rcu_read_lock(void *ctx)
 {

@@ -165,6 +168,7 @@ __noinline static int throwing_subprog(struct __sk_buff *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region")
 int reject_subprog_with_rcu_read_lock(void *ctx)
 {

@@ -181,7 +185,8 @@ static bool rbless(struct bpf_rb_node *n1, const struct bpf_rb_node *n2)
 }
 
 SEC("?tc")
-__failure __msg("function calls are not allowed while holding a lock")
+__arch_x86_64 __arch_arm64
+__failure __msg("cannot be called from callback subprog")
 int reject_with_rbtree_add_throw(void *ctx)
 {
 	struct foo *f;

@@ -196,6 +201,7 @@ int reject_with_rbtree_add_throw(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("Unreleased reference")
 int reject_with_reference(void *ctx)
 {

@@ -216,6 +222,7 @@ __noinline int global_subprog_may_throw(struct __sk_buff *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("Unreleased reference")
 int reject_global_subprog_throw_with_reference(struct __sk_buff *ctx)
 {

@@ -248,7 +255,8 @@ __noinline static int subprog_cb_ref(u32 i, void *ctx)
 }
 
 SEC("?tc")
-__failure __msg("Unreleased reference")
+__arch_x86_64 __arch_arm64
+__failure __msg("cannot be called from callback subprog")
 int reject_with_cb_reference(void *ctx)
 {
 	struct foo *f;

@@ -270,6 +278,7 @@ int reject_with_cb(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("Unreleased reference")
 int reject_with_subprog_reference(void *ctx)
 {

@@ -417,6 +426,7 @@ __noinline static int rcu_lock_then_throw(void)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("bpf_throw cannot be used inside bpf_rcu_read_lock-ed region")
 int reject_subprog_rcu_lock_throw(void *ctx)
 {

@@ -425,6 +435,7 @@ int reject_subprog_rcu_lock_throw(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("bpf_throw cannot be used inside bpf_preempt_disable-ed region")
 int reject_subprog_throw_preempt_lock(void *ctx)
 {

@@ -435,6 +446,7 @@ int reject_subprog_throw_preempt_lock(void *ctx)
 }
 
 SEC("?tc")
+__arch_s390x
 __failure __msg("bpf_throw cannot be used inside bpf_local_irq_save-ed region")
 int reject_subprog_throw_irq_lock(void *ctx)
 {

index 418bda24020c..801b507a4644 100644
@@ -2902,6 +2902,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 		 * reasons, expects to point to the next instruction)
 		 */
 		bpf_prog_update_insn_ptrs(prog, ctx.offset, ctx.ro_image);
+		bpf_prog_fill_frame_desc_sites(prog, ctx.offset, ctx.ro_image);
 out_off:
 		if (!ro_header && priv_stack_ptr) {
 			free_percpu(priv_stack_ptr);

index 9170cbe3fa16..2c2d571b21c4 100644
@@ -4932,8 +4932,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	}
 
 	if (!image || !prog->is_func || extra_pass) {
-		if (image)
+		if (image) {
 			bpf_prog_fill_jited_linfo(prog, addrs + 1);
+			bpf_prog_fill_frame_desc_sites(prog, addrs, image);
+		}
 out_addrs:
 		kvfree(addrs);
 		if (!image && priv_stack_ptr) {

101 lines not on this page · raw message

104 lines not on this page · raw message

index bbcf42eae9d9..fad8affe7fbf 100644
@@ -535,22 +535,41 @@ static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env,
 static void close_callee_effects(struct bpf_verifier_env *env)
 {
 	struct bpf_insn *insn = env->prog->insnsi;
+	bool changed, taken_throws = false, throws;
+	struct bpf_subprog_info *sub;
 	struct bpf_func_ptr *ptrs;
-	bool changed;
 	u32 cnt, j;
 	int t;
 
 	do {
 		changed = false;
+		throws = taken_throws;
 		for (t = 0; t < env->prog->len; t++) {
 			if (bpf_pseudo_call(&insn[t]) || bpf_pseudo_func(&insn[t]))
 				changed |= merge_callee_effects(env, t, t + insn[t].imm + 1);
+			if (bpf_pseudo_func(&insn[t]))
+				taken_throws |= bpf_find_containing_subprog(env, t + insn[t].imm + 1)->might_throw;
 			ptrs = insn_func_ptrs(env, t, &cnt);
-			for (j = 0; j < cnt; j++)
+			for (j = 0; j < cnt; j++) {
 				changed |= merge_callee_effects(env, t, ptrs[j].xlated_off);
+				taken_throws |= bpf_find_containing_subprog(env, ptrs[j].xlated_off)->might_throw;
+			}
+			/*
+			 * It is not known which function a callx calls: the subprog may
+			 * throw when a function whose address is taken may. The marks
+			 * above are for the subprog that takes the address, which need
+			 * not be the one that calls through it, and the frames of the
+			 * callers of the one that does are unwound, see frame_desc.c.
+			 */
+			if (taken_throws && bpf_is_callx(&insn[t])) {
+				sub = bpf_find_containing_subprog(env, t);
+				changed |= !sub->might_throw;
+				sub->might_throw = true;
+			}
 			if (bpf_is_ldimm64(&insn[t]))
 				t++;
 		}
+		changed |= throws != taken_throws;
 	} while (changed);
 }
 

index 80321ab16b32..66815fd8f856 100644
@@ -295,6 +295,7 @@ void __bpf_prog_free(struct bpf_prog *fp)
 		mutex_destroy(&fp->aux->dst_mutex);
 		kfree(fp->aux->poke_tab);
 		free_percpu(fp->aux->timed_may_goto);
+		kvfree(fp->aux->frame_desc_sites);
 		kfree(fp->aux);
 	}
 	free_percpu(fp->stats);

index 984d6287241a..0dfb33bb8bb8 100644
@@ -2047,9 +2047,15 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
-		/* The pool is freed with the main prog. */
+		/* The pool is freed with the main prog, the sites with func[i]. */
 		func[i]->aux->frame_desc = env->subprog_info[i].frame_desc;
 		func[i]->aux->frame_desc_pool = env->frame_desc_pool;
+		func[i]->aux->frame_desc_sites = bpf_frame_desc_sites(env, i);
+		if (IS_ERR(func[i]->aux->frame_desc_sites)) {
+			err = PTR_ERR(func[i]->aux->frame_desc_sites);
+			func[i]->aux->frame_desc_sites = NULL;
+			goto out_free;
+		}
 		func[i]->aux->arena = prog->aux->arena;
 		func[i]->aux->arena_stack_size = prog->aux->arena_stack_size;
 		func[i]->aux->arena_stack_entry = env->subprog_info[i].arena_stack_entry;

733 lines not on this page · raw message

index ba1176dc868e..4ec834e31dff 100644
@@ -16,7 +16,8 @@
  *
  * Inputs have to be at the same insn and cleaned with
  * bpf_clean_verifier_state(): dead registers and stack slots are the same on
- * both sides then, but for dynptrs, iterators and irq flags: they stay.
+ * both sides then, but for dynptrs, iterators and irq flags: they stay. So
+ * does what a program that may throw keeps for it, see bpf_frame_desc_keeps().
  */
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>

index 8f357fee6104..33cf3376f276 100644
@@ -129,6 +129,7 @@ static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
 }
 
 static void __clean_func_state(struct bpf_verifier_env *env,
+			       const struct bpf_verifier_state *vst,
 			       struct bpf_func_state *st,
 			       u16 live_regs, int frame)
 {

@@ -168,6 +169,11 @@ static void __clean_func_state(struct bpf_verifier_env *env,
 			    stype == STACK_IRQ_FLAG)
 				continue;
 
+			/* a throw finds what the frame holds through it, see frame_desc.c */
+			if (stype == STACK_SPILL &&
+			    bpf_frame_desc_keeps(env, vst, frame, &bpf_stack_slot(st, i)->spilled_ptr))
+				continue;
+
 			/*
 			 * Only scalar spills can be degraded to raw stack bytes
 			 * when their high half is dead. Pointer spills need the

@@ -209,7 +215,7 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
 		u32 ip = bpf_frame_insn_idx(st, i);
 		u16 live_regs = env->insn_aux_data[ip].live_regs_before;
 
-		__clean_func_state(env, st->frame[i], live_regs, i);
+		__clean_func_state(env, st, st->frame[i], live_regs, i);
 	}
 	return 0;
 }

189 lines not on this page · raw message

index 4d85a42a18d5..cbe99c346731 100644
@@ -3727,6 +3727,7 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
 	 * fastcall_stack_off to a value smaller than any possible offset.
 	 */
 	subprog->fastcall_stack_off = S16_MIN;
+	subprog->fastcall_stack_end = S16_MIN;
 	/* reset fastcall aux flags within subprogram,
 	 * happens at most once per subprogram
 	 */

@@ -11847,6 +11848,18 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
 			inherited_id(state, callee->stack_arg_regs[i].map_uid);
 	}
 
+	/*
+	 * A throw of the function finds the frames of the caller as they are
+	 * now: it has nothing that points to their stack.
+	 */
+	if (env->subprog_info[subprog].might_throw) {
+		err = bpf_frame_desc_record_site(env, state, insn_idx);
+		if (err) {
+			bpf_free_verifier_state(&entry, false);
+			return err;
+		}
+	}
+
 	account_current_path(env);
 	env->subprog_info[subprog].on_its_own = true;
 	err = bpf_flow_call(env, state, &entry, subprog, insn_idx);

@@ -20836,6 +20849,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
 	else
 		subprog->keep_fastcall_stack = 1;
 	subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
+	subprog->fastcall_stack_end = max(subprog->fastcall_stack_end, off);
 }
 
 static int mark_fastcall_patterns(struct bpf_verifier_env *env)

@@ -20846,6 +20860,7 @@ static int mark_fastcall_patterns(struct bpf_verifier_env *env)
 	int s, i;
 
 	for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {
+		subprog->fastcall_stack_end = S16_MIN;
 		/* find lowest stack spill offset used in this subprog */
 		lowest_off = 0;
 		for (i = subprog->start; i < (subprog + 1)->start; ++i) {

@@ -21072,7 +21087,8 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
 	 * for exceptional exits.
 	 */
 	if (exception_exit)
-		return PROCESS_BPF_EXIT;
+		return bpf_frame_desc_record_site(env, env->cur_state, env->insn_idx) ?:
+		       PROCESS_BPF_EXIT;
 
 	if (env->cur_state->curframe) {
 		/* exit from nested function */

@@ -23742,8 +23758,8 @@ static void print_verification_stats(struct bpf_verifier_env *env)
 		env->max_states_per_insn, env->total_states,
 		env->peak_states);
 	if (env->log.level & BPF_LOG_STATS)
-		verbose(env, "frame descriptors: slots %u insns %u classes %u bytes %u\n",
-			env->frame_desc_slots, env->frame_desc_insns,
+		verbose(env, "frame descriptors: slots %u insns %u sites %u classes %u bytes %u\n",
+			env->frame_desc_slots, env->frame_desc_insns, env->frame_desc_sites,
 			env->frame_desc_pool ? env->frame_desc_pool->resource_cnt : env->resource_cnt,
 			env->frame_desc_bytes);
 }

index e8eda32bc05d..b157863cb952 100644
@@ -1873,6 +1873,11 @@ struct bpf_resource {
  *    slot of the frame, or a slot of the program's own stack that holds the
  *    pointer whenever the frame is at the site of the descriptor; the pointer
  *    may then point adj bytes into the object;
+ *  - with BPF_FRAME_ENT_REG a register instead of a location, which holds
+ *    such a pointer whenever the frame is at the site: off is its number. The
+ *    unwinder has R6 to R9 of every frame, and R0 to R5 of the one that
+ *    called bpf_throw(), where the architecture captures them, see
+ *    bpf_arch_throw_entry();
  *  - with BPF_FRAME_ENT_OBJ the resource itself, an iterator or saved IRQ
  *    flags the frame has on its stack whenever it is at the site;
  *  - with BPF_FRAME_ENT_COUNTER a hidden slot that holds a count of sections,

@@ -1885,12 +1890,14 @@ struct bpf_resource {
 #define BPF_FRAME_ENT_COUNT	BIT(2)
 /* Released before the other entries of the frame, which it may be using. */
 #define BPF_FRAME_ENT_FIRST	BIT(3)
+#define BPF_FRAME_ENT_REG	BIT(4)
 
 struct bpf_frame_desc_entry {
 	u16 res;	/* index into the program's resource classes */
 	u16 flags;	/* BPF_FRAME_ENT_* */
 	union {
 		s16 off;	/* the location, relative to the frame pointer */
+		s16 reg;	/* BPF_FRAME_ENT_REG */
 		s16 count;	/* BPF_FRAME_ENT_COUNT */
 	};
 	s16 adj;

@@ -4631,7 +4638,7 @@ void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
 void bpf_prog_fill_frame_desc_sites(struct bpf_prog *prog, const u32 *offsets, void *image);
 void bpf_unwind_release(const struct bpf_resource *res, void *handle);
 void bpf_dynptr_file_free(void *reader);
-void bpf_unwind_release_frames(void);
+void bpf_unwind_release_frames(const struct bpf_throw_regs *regs);
 void bpf_frame_desc_pool_free(struct bpf_frame_desc_pool *pool);
 #else
 static inline void

@@ -4645,7 +4652,7 @@ bpf_prog_fill_frame_desc_sites(struct bpf_prog *prog, const u32 *offsets, void *
 static inline void bpf_frame_desc_pool_free(struct bpf_frame_desc_pool *pool)
 {
 }
-static inline void bpf_unwind_release_frames(void)
+static inline void bpf_unwind_release_frames(const struct bpf_throw_regs *regs)
 {
 }
 #endif

index ca50c85006d7..158e7eddc19f 100644
@@ -1273,8 +1273,8 @@ struct bpf_res_acq {
 	bool slot;	/* and some site cannot say where: the handle is kept in a slot */
 };
 
-/* How many places of its stack a site remembers a frame keeping one handle in. */
-#define BPF_RES_LOC_MAX		3
+/* How many places a site remembers a frame keeping one handle in. */
+#define BPF_RES_LOC_MAX		4
 
 /*
  * A site: a call a frame can be unwound at, bpf_throw() or the call of a

@@ -1292,7 +1292,8 @@ struct bpf_res_site {
 
 /*
  * Where the frame at a site keeps the handle of what an acquisition acquired:
- * the slots of its stack that every visit holding it found the handle in.
+ * the slots of its stack and the registers that every visit holding it found
+ * the handle in.
  */
 struct bpf_res_loc {
 	u32 site;	/* index of the site */

@@ -1301,8 +1302,9 @@ struct bpf_res_loc {
 	u8 cnt;		/* the places left */
 	bool obj;	/* they are the resource itself, an object on the stack */
 	struct {
-		s16 off;	/* relative to R10 */
+		s16 off;	/* relative to R10, or the number of the register */
 		s16 adj;	/* the pointer there points that many bytes into the object */
+		bool reg;
 	} at[BPF_RES_LOC_MAX];
 };
 

@@ -1374,6 +1376,8 @@ struct bpf_verifier_env {
 	u32 frame_desc_insns;			/* instructions keeping them up to date */
 	u32 frame_desc_bytes;			/* descriptors and resource classes */
 	bool describe_frames;			/* frame descriptors are generated: the program may throw */
+	bool describe_regs;			/* and may name registers, see bpf_arch_throw_entry() */
+	u16 frame_desc_regs;			/* the registers that some site names */
 	int exception_callback_subprog;
 	bool explore_alu_limits;
 	bool allow_ptr_leaks;

@@ -2332,6 +2336,8 @@ int bpf_frame_desc_record_section(struct bpf_verifier_env *env, u8 kind, s8 delt
 int bpf_frame_desc_record_exit(struct bpf_verifier_env *env);
 bool bpf_frame_desc_keeps(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
 			  int f, const struct bpf_reg_state *reg);
+u16 bpf_frame_desc_kept_regs(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+			     int f);
 int bpf_frame_desc_record_site(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
 			       int insn_idx);
 int bpf_frame_desc_record_return(struct bpf_verifier_env *env, struct bpf_verifier_state *st,

index 0dfb33bb8bb8..b02ba82f5d10 100644
@@ -2985,7 +2985,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 		}
 
 		if (bpf_is_may_goto_insn(insn) && timed_may_goto) {
-			u16 live = env->insn_aux_data[i].live_regs_before;
+			/* a site of a program that throws reads a register the program is done with */
+			u16 live = env->insn_aux_data[i].live_regs_before | env->frame_desc_regs;
 			int reg, save_off = 0;
 
 			/* a register that is dead here holds the address of the count */

176 lines not on this page · raw message

index 793eb457fb62..648443e8f009 100644
@@ -3493,7 +3493,7 @@ notrace void bpf_throw_unwind(struct bpf_throw_regs *regs, u64 cookie)
 	if (regs)
 		memcpy(ctx.saved, &regs->r[BPF_REG_6], sizeof(ctx.saved));
 	/* What the frames about to be discarded hold is released first. */
-	bpf_unwind_release_frames();
+	bpf_unwind_release_frames(regs);
 	arch_bpf_stack_walk(bpf_stack_walker, &ctx);
 	WARN_ON_ONCE(!ctx.aux);
 	if (ctx.aux)

index 4ec834e31dff..180b0a96029c 100644
@@ -663,6 +663,9 @@ int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst
 			u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
 			u32 cnt;
 
+			/* the registers both keep for a throw are joined, see bpf_frame_desc_keeps() */
+			live_regs |= bpf_frame_desc_kept_regs(env, a, fr) &
+				     bpf_frame_desc_kept_regs(env, b, fr);
 			/* not the states that get to the head through a call of the function */
 			frame = a->frame[fr];
 			cnt = frame->loop_stack_cnt;

index 33cf3376f276..5a773bfb9e84 100644
@@ -215,6 +215,8 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
 		u32 ip = bpf_frame_insn_idx(st, i);
 		u16 live_regs = env->insn_aux_data[ip].live_regs_before;
 
+		/* a throw finds what the frame holds through them, see frame_desc.c */
+		live_regs |= bpf_frame_desc_kept_regs(env, st, i);
 		__clean_func_state(env, st, st->frame[i], live_regs, i);
 	}
 	return 0;

@@ -691,9 +693,10 @@ static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state
  * the type in the current state is meaningful.
  */
 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
-			      struct bpf_func_state *cur, u32 insn_idx)
+			      struct bpf_func_state *cur, u32 insn_idx, u16 kept)
 {
-	u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
+	/* and the registers that @old keeps for a throw, see bpf_frame_desc_keeps() */
+	u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before | kept;
 	u16 i;
 
 	if (old->callback_depth > cur->callback_depth)

@@ -781,7 +784,8 @@ bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state
 		insn_idx = bpf_frame_insn_idx(old, i);
 		if (old->frame[i]->callsite != cur->frame[i]->callsite)
 			return false;
-		if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx))
+		if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx,
+				       bpf_frame_desc_kept_regs(env, old, i)))
 			return false;
 	}
 

index b8845d7a0220..473897c3aff7 100644
@@ -179,6 +179,13 @@ struct bpf_unwind_ctx {
 	/* The sections the frames hold, per kind, summed by the first walk. */
 	const struct bpf_resource *section[BPF_UNWIND_NR_SECTIONS];
 	s64 nr_sections[BPF_UNWIND_NR_SECTIONS];
+	/*
+	 * The registers of the frame the walk is at, when the entry of
+	 * bpf_throw() captured those of the throw site: the callee-saved ones,
+	 * and R0 to R5 while the walk is at the frame that threw.
+	 */
+	const struct bpf_throw_regs *entry;
+	struct bpf_throw_regs regs;
 };
 
 /*

@@ -212,6 +219,7 @@ static void bpf_unwind_release_desc(struct bpf_unwind_ctx *ctx, const struct bpf
 		const struct bpf_frame_desc_entry *ent = &d->ent[i];
 		const struct bpf_resource *res = &pool->resources[ent->res];
 		u64 *loc = (u64 *)(long)(fp + ent->off);
+		u64 val;
 
 		if (ent->flags & (BPF_FRAME_ENT_COUNTER | BPF_FRAME_ENT_COUNT)) {
 			if (!locks || first || late)

@@ -226,9 +234,13 @@ static void bpf_unwind_release_desc(struct bpf_unwind_ctx *ctx, const struct bpf
 		if (ent->flags & BPF_FRAME_ENT_OBJ) {
 			handle = loc;
 		} else {
-			if (!*loc)
+			/* no descriptor names a register that was not captured */
+			if ((ent->flags & BPF_FRAME_ENT_REG) && WARN_ON_ONCE(!ctx->entry))
+				continue;
+			val = ent->flags & BPF_FRAME_ENT_REG ? ctx->regs.r[ent->reg] : *loc;
+			if (!val)
 				continue;
-			handle = (void *)(long)(*loc - ent->adj);
+			handle = (void *)(long)(val - ent->adj);
 		}
 		if (locks && late != (res->kind == BPF_RESOURCE_IRQ_FLAG &&
 				      !arch_irqs_disabled_flags(*(unsigned long *)handle)))

@@ -290,6 +302,8 @@ static bool bpf_unwind_walker(void *cookie, u64 ip, u64 sp, u64 bp)
 		return !ctx->cnt;
 	ctx->cnt++;
 	bpf_unwind_release_frame(ctx, prog, ip, bp);
+	/* the walk leaves the frame for its caller, which has R6 to R9 as it left them */
+	bpf_throw_caller_regs(bpf_prog_frame_aux(prog), bp, &ctx->regs.r[BPF_REG_6]);
 	/* the main program is where the exception callback takes over */
 	return bpf_is_subprog(prog);
 }

@@ -302,16 +316,21 @@ static bool bpf_unwind_walker(void *cookie, u64 ip, u64 sp, u64 bp)
  * each frame's entries of the phase in the order of its descriptor; the
  * frames are released as the walk passes them, so their number is not
  * bounded. The sections need no walk of their own: the first one summed
- * their counts, and sections of one kind are all the same.
+ * their counts, and sections of one kind are all the same. @regs are the
+ * registers of the frame that called bpf_throw(), or NULL when the
+ * architecture does not capture them.
  */
-void bpf_unwind_release_frames(void)
+void bpf_unwind_release_frames(const struct bpf_throw_regs *regs)
 {
-	struct bpf_unwind_ctx ctx = {};
+	struct bpf_unwind_ctx ctx = { .entry = regs };
 	int s;
 
 	for (ctx.phase = 0; ctx.phase < BPF_UNWIND_NR_PHASES; ctx.phase++) {
 		if (ctx.phase != BPF_UNWIND_PHASE_SECTIONS) {
 			ctx.cnt = 0;
+			/* every walk starts at the throw site */
+			if (regs)
+				ctx.regs = *regs;
 			arch_bpf_stack_walk(bpf_unwind_walker, &ctx);
 			continue;
 		}

index cbe99c346731..b32c5a4f844f 100644
@@ -12217,6 +12217,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 				verbose(env, "failed to push state for global subprog exception path\n");
 				return PTR_ERR(branch);
 			}
+			/* a throw of the subprog finds the frames as they are at the call */
+			err = bpf_frame_desc_record_site(env, env->cur_state, *insn_idx);
+			if (err)
+				return err;
 			return process_bpf_exit_full(env, NULL, true);
 		}
 

@@ -16477,6 +16481,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			if (err < 0)
 				return err;
 		}
+		/* with the registers the frame calls it with, which the call clobbers */
+		err = bpf_frame_desc_record_site(env, env->cur_state, insn_idx);
+		if (err)
+			return err;
 	}
 
 	bpf_diag_record_caller_saved(env, regs);

@@ -21087,8 +21095,7 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
 	 * for exceptional exits.
 	 */
 	if (exception_exit)
-		return bpf_frame_desc_record_site(env, env->cur_state, env->insn_idx) ?:
-		       PROCESS_BPF_EXIT;
+		return PROCESS_BPF_EXIT;
 
 	if (env->cur_state->curframe) {
 		/* exit from nested function */

@@ -25710,6 +25717,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	 */
 	for (i = 0; bpf_jit_supports_exception_cleanup() && i < env->subprog_cnt; i++)
 		env->describe_frames |= env->subprog_info[i].might_throw;
+	/* the unwinder has the registers of the frames where bpf_throw() has a native entry */
+	env->describe_regs = env->describe_frames && bpf_arch_throw_entry();
 
 	ret = check_unwind_insns(env);
 	if (ret < 0)

index b157863cb952..cbb0d7a61f98 100644
@@ -1860,6 +1860,7 @@ struct bpf_resource {
 	union {
 		void (*dtor)(void *);
 		struct btf_struct_meta *meta;
+		void *addr;	/* a lock at a fixed address, in the only element of an array map */
 	};
 };
 

@@ -1880,6 +1881,8 @@ struct bpf_resource {
  *    bpf_arch_throw_entry();
  *  - with BPF_FRAME_ENT_OBJ the resource itself, an iterator or saved IRQ
  *    flags the frame has on its stack whenever it is at the site;
+ *  - with BPF_FRAME_ENT_FIXED no location: a lock at the address its class
+ *    gives, which the frame holds whenever it is at the site;
  *  - with BPF_FRAME_ENT_COUNTER a hidden slot that holds a count of sections,
  *    signed: the sections the frame entered less those of its callers it left;
  *  - with BPF_FRAME_ENT_COUNT no location: the count of the sections the frame

@@ -1891,6 +1894,7 @@ struct bpf_resource {
 /* Released before the other entries of the frame, which it may be using. */
 #define BPF_FRAME_ENT_FIRST	BIT(3)
 #define BPF_FRAME_ENT_REG	BIT(4)
+#define BPF_FRAME_ENT_FIXED	BIT(5)
 
 struct bpf_frame_desc_entry {
 	u16 res;	/* index into the program's resource classes */

index 158e7eddc19f..a7c9b113e071 100644
@@ -1253,6 +1253,7 @@ void bpf_patch_image_install(struct bpf_prog *prog, const struct bpf_patch_image
 #define BPF_RES_F_MAY_FAIL	BIT(1)	/* acquired only when the call returns 0 */
 #define BPF_RES_F_DEREF		BIT(2)	/* the handle is the first word of a dynptr on the stack */
 #define BPF_RES_F_CTX		BIT(3)	/* a __ref argument the kernel passes the main program */
+#define BPF_RES_F_NO_PTR	BIT(5)	/* a lock that no pointer to what it is in leads to */
 
 /*
  * A call that acquires a resource, see frame_desc.c. When a frame can be

@@ -1301,6 +1302,7 @@ struct bpf_res_loc {
 	u16 acq;	/* index of the acquisition */
 	u8 cnt;		/* the places left */
 	bool obj;	/* they are the resource itself, an object on the stack */
+	bool fixed;	/* it is a lock at the address of its class, and needs no place */
 	struct {
 		s16 off;	/* relative to R10, or the number of the register */
 		s16 adj;	/* the pointer there points that many bytes into the object */

147 lines not on this page · raw message

index 473897c3aff7..d3cd95f08769 100644
@@ -233,6 +233,8 @@ static void bpf_unwind_release_desc(struct bpf_unwind_ctx *ctx, const struct bpf
 			continue;
 		if (ent->flags & BPF_FRAME_ENT_OBJ) {
 			handle = loc;
+		} else if (ent->flags & BPF_FRAME_ENT_FIXED) {
+			handle = res->addr;
 		} else {
 			/* no descriptor names a register that was not captured */
 			if ((ent->flags & BPF_FRAME_ENT_REG) && WARN_ON_ONCE(!ctx->entry))

index a7c9b113e071..edd6005c32f6 100644
@@ -1272,6 +1272,7 @@ struct bpf_res_acq {
 	s16 off;	/* the slot, relative to R10 */
 	bool held;	/* some frame holds what it acquired at a site */
 	bool slot;	/* and some site cannot say where: the handle is kept in a slot */
+	bool multi;	/* a frame held two of what it acquired at once: one slot cannot */
 };
 
 /* How many places a site remembers a frame keeping one handle in. */

@@ -1317,8 +1318,16 @@ struct bpf_res_rel {
 	u8 reg;		/* the argument the call releases */
 	u8 flags;	/* BPF_RES_F_DEREF */
 	s16 adj;	/* the argument points that many bytes into the object */
+	u8 bad;		/* BPF_RES_REL_*: why no insns can clear a slot here */
 };
 
+/* The handle is in no argument register. */
+#define BPF_RES_REL_NO_REG	1
+/* The register does not point to what is released. */
+#define BPF_RES_REL_NOT_PTR	2
+/* The call takes the handle in different ways on different paths. */
+#define BPF_RES_REL_DIFFER	3
+
 /* A call that enters or leaves a section, for the count of its frame, see frame_desc.c. */
 struct bpf_section_insn {
 	u32 insn;

267 lines not on this page · raw message

index edd6005c32f6..9a0ea25ab5c2 100644
@@ -1112,6 +1112,12 @@ struct bpf_subprog_info {
 	bool stack_args_from_code: 1;
 	/* it releases what its caller holds, so it's verified in the context of such caller */
 	bool releases_inherited: 1;
+	/*
+	 * The kinds of sections, a bit each, that a callee returned to it in
+	 * more or fewer of than it called the callee in: a count it keeps in a
+	 * slot does not follow that, see frame_desc.c.
+	 */
+	u8 section_delta;
 	/* it loads the end of the packet from the context */
 	bool loads_pkt_end: 1;
 	/* a call of it was verified on its own: its insns are not in insns_total of the caller */

@@ -1272,6 +1278,13 @@ struct bpf_res_acq {
 	s16 off;	/* the slot, relative to R10 */
 	bool held;	/* some frame holds what it acquired at a site */
 	bool slot;	/* and some site cannot say where: the handle is kept in a slot */
+	/*
+	 * An insn, plus one, that leaves what it acquired with another frame
+	 * than the one that ran the acquisition: the call a subprog returns
+	 * holding it from, or the acquisition itself when it creates a stack
+	 * object in the stack of a caller. A slot cannot follow it there.
+	 */
+	u32 elsewhere;
 	bool multi;	/* a frame held two of what it acquired at once: one slot cannot */
 };
 

@@ -2345,6 +2358,8 @@ int bpf_frame_desc_record_release(struct bpf_verifier_env *env, int idx);
 int bpf_frame_desc_record_ctx_arg(struct bpf_verifier_env *env, int idx, int argno);
 int bpf_frame_desc_record_section(struct bpf_verifier_env *env, u8 kind, s8 delta);
 int bpf_frame_desc_record_exit(struct bpf_verifier_env *env);
+void bpf_frame_desc_record_sections(struct bpf_verifier_env *env, int subprog, bool rcu,
+				    bool preempt);
 bool bpf_frame_desc_keeps(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
 			  int f, const struct bpf_reg_state *reg);
 u16 bpf_frame_desc_kept_regs(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st,

263 lines not on this page · raw message

index 180b0a96029c..87d87d359312 100644
@@ -463,7 +463,7 @@ static bool join_func(struct bpf_join_state *js, struct bpf_func_state *fd,
 		    fa->entry_rcu_locks != fb->entry_rcu_locks ||
 		    fa->entry_preempt_locks != fb->entry_preempt_locks)
 			return false;
-		/* a frame that left a section of its callers hands its count over, see frame_desc.c */
+		/* the sections a frame left of those it was entered in matter to a throw, see frame_desc.c */
 		if (env->describe_frames && (fa->rcu_released_below != fb->rcu_released_below ||
 					     fa->preempt_released_below != fb->preempt_released_below))
 			return false;

index 5a773bfb9e84..01eb8d9077ef 100644
@@ -717,7 +717,7 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
 	    old->entry_rcu_locks != cur->entry_rcu_locks ||
 	    old->entry_preempt_locks != cur->entry_preempt_locks)
 		return false;
-	/* a frame that left a section of its callers hands its count over, see frame_desc.c */
+	/* the sections a frame left of those it was entered in matter to a throw, see frame_desc.c */
 	if (env->describe_frames && (old->rcu_released_below != cur->rcu_released_below ||
 				     old->preempt_released_below != cur->preempt_released_below))
 		return false;

index b32c5a4f844f..a710da1a15c6 100644
@@ -12028,6 +12028,10 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
 	}
 	caller->active_irq_id = fresh_id(env, idmap, caller->active_irq_id);
 	caller->active_lock_id = fresh_id(env, idmap, caller->active_lock_id);
+	/* the sections the function entered and did not leave are the caller's, see frame_desc.c */
+	bpf_frame_desc_record_sections(env, frame->subprogno,
+				       caller->active_rcu_locks != caller_rcu,
+				       caller->active_preempt_locks != caller_preempt);
 	/* fresh_id() had no room for an id, or there are no ids left */
 	if (idmap->cnt == BPF_ID_MAP_SIZE || env->id_gen >= BPF_MAX_ID) {
 		verbose(env, "too many ids at the call at insn %d\n", call_idx);

index 9a0ea25ab5c2..a98023c239c4 100644
@@ -1278,6 +1278,7 @@ struct bpf_res_acq {
 	s16 off;	/* the slot, relative to R10 */
 	bool held;	/* some frame holds what it acquired at a site */
 	bool slot;	/* and some site cannot say where: the handle is kept in a slot */
+	u32 foreign;	/* an insn of another frame that releases what it acquired, plus one */
 	/*
 	 * An insn, plus one, that leaves what it acquired with another frame
 	 * than the one that ran the acquisition: the call a subprog returns

@@ -1341,6 +1342,20 @@ struct bpf_res_rel {
 /* The call takes the handle in different ways on different paths. */
 #define BPF_RES_REL_DIFFER	3
 
+/*
+ * A call of a subprog during which a frame above the caller releases what an
+ * acquisition of the caller acquired. The caller clears its slot in front of
+ * the call when no throw finds it holding the resource there and no return
+ * from the call does, see frame_desc.c.
+ */
+struct bpf_res_call {
+	u32 insn;	/* the call; U32_MAX when the caller is not in the state */
+	u16 acq;	/* index of the acquisition */
+	bool released;
+	bool held_after; /* some return from the call finds the caller holding it */
+	bool clear;	/* decided once the pass is over */
+};
+
 /* A call that enters or leaves a section, for the count of its frame, see frame_desc.c. */
 struct bpf_section_insn {
 	u32 insn;

@@ -1390,6 +1405,8 @@ struct bpf_verifier_env {
 	u32 res_rel_cnt;
 	struct bpf_section_insn *section_insns;
 	u32 section_insn_cnt;
+	struct bpf_res_call *res_calls;
+	u32 res_call_cnt;
 	struct bpf_res_site *res_sites;
 	u32 res_site_cnt;
 	struct bpf_res_loc *res_locs;

@@ -1402,6 +1419,12 @@ struct bpf_verifier_env {
 	bool describe_frames;			/* frame descriptors are generated: the program may throw */
 	bool describe_regs;			/* and may name registers, see bpf_arch_throw_entry() */
 	u16 frame_desc_regs;			/* the registers that some site names */
+	/*
+	 * The round of the main pass found out that a subprog releases what
+	 * its callers hold: they are recorded differently then, so the round
+	 * is done again, see do_check_all().
+	 */
+	bool frame_desc_redo;
 	int exception_callback_subprog;
 	bool explore_alu_limits;
 	bool allow_ptr_leaks;

258 lines not on this page · raw message

index a710da1a15c6..ec9f77a63197 100644
@@ -11998,6 +11998,9 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
 		if (env->log.level & BPF_LOG_LEVEL2)
 			verbose(env, "func#%d releases what its caller holds at insn %d\n",
 				callee->subprogno, call_idx);
+		/* what the round recorded of the function so far does not hold, see frame_desc.c */
+		if (env->describe_frames && !env->subprog_info[callee->subprogno].releases_inherited)
+			env->frame_desc_redo = true;
 		env->subprog_info[callee->subprogno].releases_inherited = true;
 		return 1;
 	}

@@ -22273,7 +22276,8 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
 /* All is done again, whatever came out, see do_check_all() */
 static bool round_is_dropped(struct bpf_verifier_env *env)
 {
-	return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late;
+	return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late ||
+	       env->frame_desc_redo;
 }
 
 static int do_check(struct bpf_verifier_env *env)