Skip to contentSkip to conversation

bpf, sparc: Add support for BPF_SDIV and BPF_SMOD in the sparc64 JIT

PATCH In review Nicholas Dudar 2 participants 3 messages Jul 16 – Jul 29

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 bpf, sparc: Add support for BPF_SDIV and BPF_SMOD in the sparc64 JIT

build_insn()'s div/mod emit sites pick DIV/UDIVX by ALU width only, never by insn->off, so the sparc64 JIT does not implement signed BPF_SDIV and BPF_SMOD (off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands.

Emit SDIV/SDIVX on the signed path across the ALU32 and ALU64, X-form and K-form div/mod sites. The 32-bit signed divide takes a 64-bit dividend from %y:dst, so sign-fill %y from dst before SDIV where the unsigned path zeroes it; the 64-bit SDIVX skips the %y write. Route the signed ALU32 result through do_alu32_trunc, since SDIV does not leave bits 63:32 clear the way unsigned DIV does.

The verifier and bpf_do_misc_fixups() remove the zero divisor and INT_MIN / -1 cases before the JIT runs.

Assisted-by Claude:claude-opus-4-8
---
 arch/sparc/net/bpf_jit_comp_64.c | 92 +++++++++++++++++++++++++++-----
 1 file changed, 80 insertions(+), 12 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +80 -12
index 2fa0e9375127..6de9fb29944c 100644
@@ -149,6 +149,8 @@ static u32 WDISP10(u32 off)
 #define MULX		F3(2, 0x09)
 #define UDIVX		F3(2, 0x0d)
 #define DIV		F3(2, 0x0e)
+#define SDIV		F3(2, 0x0f)
+#define SDIVX		F3(2, 0x2d)
 #define SLL		F3(2, 0x25)
 #define SLLX		(F3(2, 0x25)|(1<<12))
 #define SRA		F3(2, 0x27)
@@ -941,32 +943,69 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_X:
 		emit_alu(MULX, src, dst, ctx);
 		break;
-	case BPF_ALU | BPF_DIV | BPF_X:
+	case BPF_ALU | BPF_DIV | BPF_X: {
+		const bool is_signed = (off == 1);
+
+		if (is_signed) {
+			const u8 tmp = bpf2sparc[TMP_REG_1];
+
+			ctx->tmp_1_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp, ctx);
+			emit_write_y(tmp, ctx);
+			emit_alu(SDIV, src, dst, ctx);
+			/*
+			 * SDIV does not guarantee the zero-extension that
+			 * the unsigned DIV path relies on, so clear bits
+			 * 63:32 through the shared truncation instead of
+			 * consuming the verifier's zext marker, as the
+			 * other ALU32 ops do.
+			 */
+			goto do_alu32_trunc;
+		}
 		emit_write_y(G0, ctx);
 		emit_alu(DIV, src, dst, ctx);
 		if (insn_is_zext(&insn[1]))
 			return 1;
 		break;
+	}
 	case BPF_ALU64 | BPF_DIV | BPF_X:
-		emit_alu(UDIVX, src, dst, ctx);
+		if (off == 1)
+			emit_alu(SDIVX, src, dst, ctx);
+		else
+			emit_alu(UDIVX, src, dst, ctx);
 		break;
 	case BPF_ALU | BPF_MOD | BPF_X: {
 		const u8 tmp = bpf2sparc[TMP_REG_1];
+		const bool is_signed = (off == 1);
 
 		ctx->tmp_1_used = true;
 
-		emit_write_y(G0, ctx);
-		emit_alu3(DIV, dst, src, tmp, ctx);
+		if (is_signed) {
+			const u8 tmp2 = bpf2sparc[TMP_REG_2];
+
+			ctx->tmp_2_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp2, ctx);
+			emit_write_y(tmp2, ctx);
+			emit_alu3(SDIV, dst, src, tmp, ctx);
+		} else {
+			emit_write_y(G0, ctx);
+			emit_alu3(DIV, dst, src, tmp, ctx);
+		}
 		emit_alu3(MULX, tmp, src, tmp, ctx);
 		emit_alu3(SUB, dst, tmp, dst, ctx);
 		goto do_alu32_trunc;
 	}
 	case BPF_ALU64 | BPF_MOD | BPF_X: {
 		const u8 tmp = bpf2sparc[TMP_REG_1];
+		const unsigned int mod = (off == 1) ? SDIVX : UDIVX;
 
 		ctx->tmp_1_used = true;
 
-		emit_alu3(UDIVX, dst, src, tmp, ctx);
+		emit_alu3(mod, dst, src, tmp, ctx);
 		emit_alu3(MULX, tmp, src, tmp, ctx);
 		emit_alu3(SUB, dst, tmp, dst, ctx);
 		break;
@@ -1096,33 +1135,62 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_K:
 		emit_alu_K(MULX, dst, imm, ctx);
 		break;
-	case BPF_ALU | BPF_DIV | BPF_K:
+	case BPF_ALU | BPF_DIV | BPF_K: {
+		const bool is_signed = (off == 1);
+
 		if (imm == 0)
 			return -EINVAL;
 
-		emit_write_y(G0, ctx);
-		emit_alu_K(DIV, dst, imm, ctx);
+		if (is_signed) {
+			const u8 tmp = bpf2sparc[TMP_REG_2];
+
+			ctx->tmp_2_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp, ctx);
+			emit_write_y(tmp, ctx);
+			emit_alu_K(SDIV, dst, imm, ctx);
+		} else {
+			emit_write_y(G0, ctx);
+			emit_alu_K(DIV, dst, imm, ctx);
+		}
 		goto do_alu32_trunc;
+	}
 	case BPF_ALU64 | BPF_DIV | BPF_K:
 		if (imm == 0)
 			return -EINVAL;
 
-		emit_alu_K(UDIVX, dst, imm, ctx);
+		emit_alu_K((off == 1) ? SDIVX : UDIVX, dst, imm, ctx);
 		break;
 	case BPF_ALU64 | BPF_MOD | BPF_K:
 	case BPF_ALU | BPF_MOD | BPF_K: {
 		const u8 tmp = bpf2sparc[TMP_REG_2];
+		const bool is_signed = (off == 1);
 		unsigned int div;
 
 		if (imm == 0)
 			return -EINVAL;
 
-		div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
+		if (BPF_CLASS(code) == BPF_ALU64)
+			div = is_signed ? SDIVX : UDIVX;
+		else
+			div = is_signed ? SDIV : DIV;
 
 		ctx->tmp_2_used = true;
 
-		if (BPF_CLASS(code) != BPF_ALU64)
-			emit_write_y(G0, ctx);
+		if (BPF_CLASS(code) != BPF_ALU64) {
+			if (is_signed) {
+				const u8 tmp3 = bpf2sparc[TMP_REG_3];
+
+				ctx->tmp_3_used = true;
+
+				/* Sign-extend dst into %y for 32-bit sdiv. */
+				emit_alu3_K(SRA, dst, 31, tmp3, ctx);
+				emit_write_y(tmp3, ctx);
+			} else {
+				emit_write_y(G0, ctx);
+			}
+		}
 		if (is_simm13(imm)) {
 			emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
 			emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);

base-commit: d1f4b56417a3dc1a0600f960b14f46bd25eda89d

Signature · 1 line
2.34.1
#1 lore raw
Quoted 13 lines · Nicholas Dudar

build_insn()'s div/mod emit sites pick DIV/UDIVX by ALU width only, never by insn->off, so the sparc64 JIT does not implement signed BPF_SDIV and BPF_SMOD (off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands.

Emit SDIV/SDIVX on the signed path across the ALU32 and ALU64, X-form and K-form div/mod sites. The 32-bit signed divide takes a 64-bit dividend from %y:dst, so sign-fill %y from dst before SDIV where the unsigned path zeroes it; the 64-bit SDIVX skips the %y write. Route the signed ALU32 result through do_alu32_trunc, since SDIV does not leave bits 63:32 clear the way unsigned DIV does.

Quoted 8 lines

The verifier and bpf_do_misc_fixups() remove the zero divisor and INT_MIN / -1 cases before the JIT runs.

Assisted-by Claude:claude-opus-4-8
---
 arch/sparc/net/bpf_jit_comp_64.c | 92 +++++++++++++++++++++++++++-----
 1 file changed, 80 insertions(+), 12 deletions(-)

I saw that your bpf jit patches span multiple architectures. Assuming you don't have access to all these different hardware platforms. Just out of curiosity how were these patches tested?

Regards,
Kuan-Wei

Quoted 164 lines
arch/sparc/net/bpf_jit_comp_64.c +80 -12
index 2fa0e9375127..6de9fb29944c 100644
@@ -149,6 +149,8 @@ static u32 WDISP10(u32 off)
 #define MULX		F3(2, 0x09)
 #define UDIVX		F3(2, 0x0d)
 #define DIV		F3(2, 0x0e)
+#define SDIV		F3(2, 0x0f)
+#define SDIVX		F3(2, 0x2d)
 #define SLL		F3(2, 0x25)
 #define SLLX		(F3(2, 0x25)|(1<<12))
 #define SRA		F3(2, 0x27)
@@ -941,32 +943,69 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_X:
 		emit_alu(MULX, src, dst, ctx);
 		break;
-	case BPF_ALU | BPF_DIV | BPF_X:
+	case BPF_ALU | BPF_DIV | BPF_X: {
+		const bool is_signed = (off == 1);
+
+		if (is_signed) {
+			const u8 tmp = bpf2sparc[TMP_REG_1];
+
+			ctx->tmp_1_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp, ctx);
+			emit_write_y(tmp, ctx);
+			emit_alu(SDIV, src, dst, ctx);
+			/*
+			 * SDIV does not guarantee the zero-extension that
+			 * the unsigned DIV path relies on, so clear bits
+			 * 63:32 through the shared truncation instead of
+			 * consuming the verifier's zext marker, as the
+			 * other ALU32 ops do.
+			 */
+			goto do_alu32_trunc;
+		}
 		emit_write_y(G0, ctx);
 		emit_alu(DIV, src, dst, ctx);
 		if (insn_is_zext(&insn[1]))
 			return 1;
 		break;
+	}
 	case BPF_ALU64 | BPF_DIV | BPF_X:
-		emit_alu(UDIVX, src, dst, ctx);
+		if (off == 1)
+			emit_alu(SDIVX, src, dst, ctx);
+		else
+			emit_alu(UDIVX, src, dst, ctx);
 		break;
 	case BPF_ALU | BPF_MOD | BPF_X: {
 		const u8 tmp = bpf2sparc[TMP_REG_1];
+		const bool is_signed = (off == 1);
 
 		ctx->tmp_1_used = true;
 
-		emit_write_y(G0, ctx);
-		emit_alu3(DIV, dst, src, tmp, ctx);
+		if (is_signed) {
+			const u8 tmp2 = bpf2sparc[TMP_REG_2];
+
+			ctx->tmp_2_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp2, ctx);
+			emit_write_y(tmp2, ctx);
+			emit_alu3(SDIV, dst, src, tmp, ctx);
+		} else {
+			emit_write_y(G0, ctx);
+			emit_alu3(DIV, dst, src, tmp, ctx);
+		}
 		emit_alu3(MULX, tmp, src, tmp, ctx);
 		emit_alu3(SUB, dst, tmp, dst, ctx);
 		goto do_alu32_trunc;
 	}
 	case BPF_ALU64 | BPF_MOD | BPF_X: {
 		const u8 tmp = bpf2sparc[TMP_REG_1];
+		const unsigned int mod = (off == 1) ? SDIVX : UDIVX;
 
 		ctx->tmp_1_used = true;
 
-		emit_alu3(UDIVX, dst, src, tmp, ctx);
+		emit_alu3(mod, dst, src, tmp, ctx);
 		emit_alu3(MULX, tmp, src, tmp, ctx);
 		emit_alu3(SUB, dst, tmp, dst, ctx);
 		break;
@@ -1096,33 +1135,62 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_K:
 		emit_alu_K(MULX, dst, imm, ctx);
 		break;
-	case BPF_ALU | BPF_DIV | BPF_K:
+	case BPF_ALU | BPF_DIV | BPF_K: {
+		const bool is_signed = (off == 1);
+
 		if (imm == 0)
 			return -EINVAL;
 
-		emit_write_y(G0, ctx);
-		emit_alu_K(DIV, dst, imm, ctx);
+		if (is_signed) {
+			const u8 tmp = bpf2sparc[TMP_REG_2];
+
+			ctx->tmp_2_used = true;
+
+			/* Sign-extend dst into %y for 32-bit sdiv. */
+			emit_alu3_K(SRA, dst, 31, tmp, ctx);
+			emit_write_y(tmp, ctx);
+			emit_alu_K(SDIV, dst, imm, ctx);
+		} else {
+			emit_write_y(G0, ctx);
+			emit_alu_K(DIV, dst, imm, ctx);
+		}
 		goto do_alu32_trunc;
+	}
 	case BPF_ALU64 | BPF_DIV | BPF_K:
 		if (imm == 0)
 			return -EINVAL;
 
-		emit_alu_K(UDIVX, dst, imm, ctx);
+		emit_alu_K((off == 1) ? SDIVX : UDIVX, dst, imm, ctx);
 		break;
 	case BPF_ALU64 | BPF_MOD | BPF_K:
 	case BPF_ALU | BPF_MOD | BPF_K: {
 		const u8 tmp = bpf2sparc[TMP_REG_2];
+		const bool is_signed = (off == 1);
 		unsigned int div;
 
 		if (imm == 0)
 			return -EINVAL;
 
-		div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
+		if (BPF_CLASS(code) == BPF_ALU64)
+			div = is_signed ? SDIVX : UDIVX;
+		else
+			div = is_signed ? SDIV : DIV;
 
 		ctx->tmp_2_used = true;
 
-		if (BPF_CLASS(code) != BPF_ALU64)
-			emit_write_y(G0, ctx);
+		if (BPF_CLASS(code) != BPF_ALU64) {
+			if (is_signed) {
+				const u8 tmp3 = bpf2sparc[TMP_REG_3];
+
+				ctx->tmp_3_used = true;
+
+				/* Sign-extend dst into %y for 32-bit sdiv. */
+				emit_alu3_K(SRA, dst, 31, tmp3, ctx);
+				emit_write_y(tmp3, ctx);
+			} else {
+				emit_write_y(G0, ctx);
+			}
+		}
 		if (is_simm13(imm)) {
 			emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
 			emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);

base-commit: d1f4b56417a3dc1a0600f960b14f46bd25eda89d

Signature · 1 line
2.34.1
#2 lore raw
Quoted 3 lines · Kuan-Wei Chiu

I saw that your bpf jit patches span multiple architectures. Assuming you don't have access to all these different hardware platforms. Just out of curiosity how were these patches tested?

Correct, I used QEMU for my local testing. For sparc64 I cross-compiled bpf-next 520d7d794202 with sparc64-linux-gnu-gcc 11.4.0, then compared complete lib/test_bpf runs from stock and patched kernels under qemu-system-sparc64 -M sun4u. Ten tests changed from FAIL to PASS.

I also compared stock and patched runs for x86_32, parisc32/64 and MIPS32/64, including MIPS R6.

For LoongArch I reproduced the issue under QEMU and Tiezhu Yang later ran test_bpf with the patch and supplied Tested-by.

Thanks,
Nicholas