Skip to contentSkip to conversation

x86/bpf: Fixes for the BPF JIT with retbleed=stuff

PATCH Superseded Uros Bizjak 1 participant 3 messages Apr 1, 2024

Subsystems (5)
Networking Network drivers x86 BPF +1 more subsystems
lore
Share & tools
lore
#0 lore raw

v2 cover x86/bpf: Fixes for the BPF JIT with retbleed=stuff

Fixes two issues that cause kernels panic when using the BPF JIT with the call depth tracking / stuffing mitigation for Skylake processors (retbleed=stuff). Both issues can be triggered by running simple BPF programs (e.g. running the test suite should trigger both).

The first (resubmit) fixes a trivial issue related to calculating the destination IP for call instructions with call depth tracking.

The second is related to using the correct IP for relocations, related to the recently introduced %rip-relative addressing for PER_CPU_VAR.

Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>

---
v2:
Simplify calculation of "ip".
Explicitly list offending commit in the commit message.

Joan Bruguera Micó (1):
x86/bpf: Fix IP for relocating call depth accounting

Uros Bizjak (1):
x86/bpf: Fix IP after emitting call depth accounting

 arch/x86/include/asm/alternative.h |  4 ++--
 arch/x86/kernel/callthunks.c       |  4 ++--
 arch/x86/net/bpf_jit_comp.c        | 19 ++++++++-----------
 3 files changed, 12 insertions(+), 15 deletions(-)
Signature · 1 line
2.42.0
#1 lore raw

PATCH v2 1/2 x86/bpf: Fix IP after emitting call depth accounting

Adjust the IP passed to emit_patch so it calculates the correct offset for the CALL instruction if x86_call_depth_emit_accounting emits code. Otherwise we will skip some instructions and most likely crash.

Fixes b2e9dfe54be4 ("x86/bpf: Emit call depth accounting if required")
Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>
---
 arch/x86/net/bpf_jit_comp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/net/bpf_jit_comp.c +1 -1
index a7ba8e178645..e55745f512e1 100644
@@ -480,7 +480,7 @@ static int emit_call(u8 **pprog, void *func, void *ip)
 static int emit_rsb_call(u8 **pprog, void *func, void *ip)
 {
 	OPTIMIZER_HIDE_VAR(func);
-	x86_call_depth_emit_accounting(pprog, func);
+	ip += x86_call_depth_emit_accounting(pprog, func);
 	return emit_patch(pprog, func, ip, 0xE8);
 }
 
Signature · 1 line
2.42.0
#2 lore raw

PATCH v2 2/2 x86/bpf: Fix IP for relocating call depth accounting

In reply to Uros Bizjak

From: Joan Bruguera Micó <[email protected]>

The recently introduced support for %rip-relative relocations in the call thunk template:

17bce3b2ae2d ("x86/callthunks: Handle %rip-relative relocations in call thunk template")

assumes that the code is being patched in-place, so the destination of the relocation matches the address of the code. This is not true for the call depth accounting emitted by the BPF JIT, so the calculated address is wrong and usually causes a page fault.

Pass the destination IP when the BPF JIT emits call depth accounting.

Fixes 17bce3b2ae2d ("x86/callthunks: Handle %rip-relative relocations in call thunk template")
Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>
---
 arch/x86/include/asm/alternative.h |  4 ++--
 arch/x86/kernel/callthunks.c       |  4 ++--
 arch/x86/net/bpf_jit_comp.c        | 19 ++++++++-----------
 3 files changed, 12 insertions(+), 15 deletions(-)
arch/x86/include/asm/alternative.h +2 -2
index fcd20c6dc7f9..67b68d0d17d1 100644
@@ -117,7 +117,7 @@ extern void callthunks_patch_builtin_calls(void);
 extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
 					  struct module *mod);
 extern void *callthunks_translate_call_dest(void *dest);
-extern int x86_call_depth_emit_accounting(u8 **pprog, void *func);
+extern int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip);
 #else
 static __always_inline void callthunks_patch_builtin_calls(void) {}
 static __always_inline void
@@ -128,7 +128,7 @@ static __always_inline void *callthunks_translate_call_dest(void *dest)
 	return dest;
 }
 static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
-							  void *func)
+							  void *func, void *ip)
 {
 	return 0;
 }
arch/x86/kernel/callthunks.c +2 -2
index 30335182b6b0..e92ff0c11db8 100644
@@ -314,7 +314,7 @@ static bool is_callthunk(void *addr)
 	return !bcmp(pad, insn_buff, tmpl_size);
 }
 
-int x86_call_depth_emit_accounting(u8 **pprog, void *func)
+int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
 {
 	unsigned int tmpl_size = SKL_TMPL_SIZE;
 	u8 insn_buff[MAX_PATCH_LEN];
@@ -327,7 +327,7 @@ int x86_call_depth_emit_accounting(u8 **pprog, void *func)
 		return 0;
 
 	memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
-	apply_relocation(insn_buff, tmpl_size, *pprog,
+	apply_relocation(insn_buff, tmpl_size, ip,
 			 skl_call_thunk_template, tmpl_size);
 
 	memcpy(*pprog, insn_buff, tmpl_size);
arch/x86/net/bpf_jit_comp.c +8 -11
index e55745f512e1..df5fac428408 100644
@@ -480,7 +480,7 @@ static int emit_call(u8 **pprog, void *func, void *ip)
 static int emit_rsb_call(u8 **pprog, void *func, void *ip)
 {
 	OPTIMIZER_HIDE_VAR(func);
-	ip += x86_call_depth_emit_accounting(pprog, func);
+	ip += x86_call_depth_emit_accounting(pprog, func, ip);
 	return emit_patch(pprog, func, ip, 0xE8);
 }
 
@@ -1972,20 +1972,17 @@ st:			if (is_imm8(insn->off))
 
 			/* call */
 		case BPF_JMP | BPF_CALL: {
-			int offs;
+			u8 *ip = image + addrs[i - 1];
 
 			func = (u8 *) __bpf_call_base + imm32;
 			if (tail_call_reachable) {
 				RESTORE_TAIL_CALL_CNT(bpf_prog->aux->stack_depth);
-				if (!imm32)
-					return -EINVAL;
-				offs = 7 + x86_call_depth_emit_accounting(&prog, func);
-			} else {
-				if (!imm32)
-					return -EINVAL;
-				offs = x86_call_depth_emit_accounting(&prog, func);
+				ip += 7;
 			}
-			if (emit_call(&prog, func, image + addrs[i - 1] + offs))
+			if (!imm32)
+				return -EINVAL;
+			ip += x86_call_depth_emit_accounting(&prog, func, ip);
+			if (emit_call(&prog, func, ip))
 				return -EINVAL;
 			break;
 		}
@@ -2835,7 +2832,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		 * Direct-call fentry stub, as such it needs accounting for the
 		 * __fentry__ call.
 		 */
-		x86_call_depth_emit_accounting(&prog, NULL);
+		x86_call_depth_emit_accounting(&prog, NULL, image);
 	}
 	EMIT1(0x55);		 /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
Signature · 1 line
2.42.0