bpf, sparc: Add support for BPF_SDIV and BPF_SMOD in the sparc64 JIT
PATCH 1/1 bpf, sparc: Add support for BPF_SDIV and BPF_SMOD in the sparc64 JIT
build_insn()'s div/mod emit sites pick DIV/UDIVX by ALU width only, never by insn->off, so the sparc64 JIT does not implement signed BPF_SDIV and BPF_SMOD (off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands.
Emit SDIV/SDIVX on the signed path across the ALU32 and ALU64, X-form and K-form div/mod sites. The 32-bit signed divide takes a 64-bit dividend from %y:dst, so sign-fill %y from dst before SDIV where the unsigned path zeroes it; the 64-bit SDIVX skips the %y write. Route the signed ALU32 result through do_alu32_trunc, since SDIV does not leave bits 63:32 clear the way unsigned DIV does.
The verifier and bpf_do_misc_fixups() remove the zero divisor and INT_MIN / -1 cases before the JIT runs.
--- arch/sparc/net/bpf_jit_comp_64.c | 92 +++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 12 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +80 -12
@@ -149,6 +149,8 @@ static u32 WDISP10(u32 off)#define MULX F3(2, 0x09)#define UDIVX F3(2, 0x0d)#define DIV F3(2, 0x0e) +#define SDIV F3(2, 0x0f) +#define SDIVX F3(2, 0x2d)#define SLL F3(2, 0x25)#define SLLX (F3(2, 0x25)|(1<<12))#define SRA F3(2, 0x27)
@@ -941,32 +943,69 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)case BPF_ALU64 |BPF_MUL |BPF_X :emit_alu (MULX , src, dst, ctx);break ; -case BPF_ALU |BPF_DIV |BPF_X : +case BPF_ALU |BPF_DIV |BPF_X : { +const bool is_signed = (off ==1 ); + +if (is_signed) { +const u8 tmp = bpf2sparc[TMP_REG_1 ]; + + ctx->tmp_1_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp, ctx); +emit_write_y (tmp, ctx); +emit_alu (SDIV , src, dst, ctx); +/* +* SDIV does not guarantee the zero-extension that +* the unsigned DIV path relies on, so clear bits +* 63:32 through the shared truncation instead of +* consuming the verifier's zext marker, as the +* other ALU32 ops do. +*/ + goto do_alu32_trunc; + }emit_write_y (G0 , ctx);emit_alu (DIV , src, dst, ctx);if (insn_is_zext (&insn[1 ]))return 1 ;break ; + }case BPF_ALU64 |BPF_DIV |BPF_X : -emit_alu (UDIVX , src, dst, ctx); +if (off ==1 ) +emit_alu (SDIVX , src, dst, ctx); +else +emit_alu (UDIVX , src, dst, ctx);break ;case BPF_ALU |BPF_MOD |BPF_X : {const u8 tmp = bpf2sparc[TMP_REG_1 ]; +const bool is_signed = (off ==1 ); ctx->tmp_1_used = true; -emit_write_y (G0 , ctx); -emit_alu3 (DIV , dst, src, tmp, ctx); +if (is_signed) { +const u8 tmp2 = bpf2sparc[TMP_REG_2 ]; + + ctx->tmp_2_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp2, ctx); +emit_write_y (tmp2, ctx); +emit_alu3 (SDIV , dst, src, tmp, ctx); + }else { +emit_write_y (G0 , ctx); +emit_alu3 (DIV , dst, src, tmp, ctx); + }emit_alu3 (MULX , tmp, src, tmp, ctx);emit_alu3 (SUB , dst, tmp, dst, ctx); goto do_alu32_trunc; }case BPF_ALU64 |BPF_MOD |BPF_X : {const u8 tmp = bpf2sparc[TMP_REG_1 ]; +const unsigned int mod = (off ==1 ) ?SDIVX :UDIVX ; ctx->tmp_1_used = true; -emit_alu3 (UDIVX , dst, src, tmp, ctx); +emit_alu3 (mod, dst, src, tmp, ctx);emit_alu3 (MULX , tmp, src, tmp, ctx);emit_alu3 (SUB , dst, tmp, dst, ctx);break ;
@@ -1096,33 +1135,62 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)case BPF_ALU64 |BPF_MUL |BPF_K :emit_alu_K (MULX , dst, imm, ctx);break ; -case BPF_ALU |BPF_DIV |BPF_K : +case BPF_ALU |BPF_DIV |BPF_K : { +const bool is_signed = (off ==1 ); +if (imm ==0 )return -EINVAL ; -emit_write_y (G0 , ctx); -emit_alu_K (DIV , dst, imm, ctx); +if (is_signed) { +const u8 tmp = bpf2sparc[TMP_REG_2 ]; + + ctx->tmp_2_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp, ctx); +emit_write_y (tmp, ctx); +emit_alu_K (SDIV , dst, imm, ctx); + }else { +emit_write_y (G0 , ctx); +emit_alu_K (DIV , dst, imm, ctx); + } goto do_alu32_trunc; + }case BPF_ALU64 |BPF_DIV |BPF_K :if (imm ==0 )return -EINVAL ; -emit_alu_K (UDIVX , dst, imm, ctx); +emit_alu_K ((off ==1 ) ?SDIVX :UDIVX , dst, imm, ctx);break ;case BPF_ALU64 |BPF_MOD |BPF_K :case BPF_ALU |BPF_MOD |BPF_K : {const u8 tmp = bpf2sparc[TMP_REG_2 ]; +const bool is_signed = (off ==1 );unsigned int div;if (imm ==0 )return -EINVAL ; - div = (BPF_CLASS (code) ==BPF_ALU64 ) ?UDIVX :DIV ; +if (BPF_CLASS (code) ==BPF_ALU64 ) + div = is_signed ?SDIVX :UDIVX ; +else + div = is_signed ?SDIV :DIV ; ctx->tmp_2_used = true; -if (BPF_CLASS (code) !=BPF_ALU64 ) -emit_write_y (G0 , ctx); +if (BPF_CLASS (code) !=BPF_ALU64 ) { +if (is_signed) { +const u8 tmp3 = bpf2sparc[TMP_REG_3 ]; + + ctx->tmp_3_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp3, ctx); +emit_write_y (tmp3, ctx); + }else { +emit_write_y (G0 , ctx); + } + }if (is_simm13 (imm)) {emit (div |IMMED |RS1 (dst) |S13 (imm) |RD (tmp), ctx);emit (MULX |IMMED |RS1 (tmp) |S13 (imm) |RD (tmp), ctx);
base-commit: d1f4b56417a3dc1a0600f960b14f46bd25eda89d
Signature · 1 line
2.34.1
Quoted 13 lines · Nicholas Dudar
build_insn()'s div/mod emit sites pick DIV/UDIVX by ALU width only, never by
insn->off, so the sparc64 JIT does not implement signedBPF_SDIVandBPF_SMOD(off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands.Emit SDIV/SDIVX on the signed path across the ALU32 and ALU64, X-form and K-form div/mod sites. The 32-bit signed divide takes a 64-bit dividend from %y:dst, so sign-fill %y from dst before SDIV where the unsigned path zeroes it; the 64-bit SDIVX skips the %y write. Route the signed ALU32 result through
do_alu32_trunc, since SDIV does not leave bits 63:32 clear the way unsigned DIV does.
Quoted 8 lines
The verifier and
bpf_do_misc_fixups()remove the zero divisor andINT_MIN/ -1 cases before the JIT runs.Signed-off-by Nicholas Dudar <[email protected]>Assisted-by Claude:claude-opus-4-8--- arch/sparc/net/bpf_jit_comp_64.c | 92 +++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 12 deletions(-)
I saw that your bpf jit patches span multiple architectures. Assuming you don't have access to all these different hardware platforms. Just out of curiosity how were these patches tested?
Regards,
Kuan-Wei
Quoted 164 lines
arch/sparc/net/bpf_jit_comp_64.c +80
-12@@ -149,6 +149,8 @@ static u32 WDISP10(u32 off)#define MULX F3(2, 0x09)#define UDIVX F3(2, 0x0d)#define DIV F3(2, 0x0e) +#define SDIV F3(2, 0x0f) +#define SDIVX F3(2, 0x2d)#define SLL F3(2, 0x25)#define SLLX (F3(2, 0x25)|(1<<12))#define SRA F3(2, 0x27)@@ -941,32 +943,69 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)case BPF_ALU64 |BPF_MUL |BPF_X :emit_alu (MULX , src, dst, ctx);break ; -case BPF_ALU |BPF_DIV |BPF_X : +case BPF_ALU |BPF_DIV |BPF_X : { +const bool is_signed = (off ==1 ); + +if (is_signed) { +const u8 tmp = bpf2sparc[TMP_REG_1 ]; + + ctx->tmp_1_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp, ctx); +emit_write_y (tmp, ctx); +emit_alu (SDIV , src, dst, ctx); +/* +* SDIV does not guarantee the zero-extension that +* the unsigned DIV path relies on, so clear bits +* 63:32 through the shared truncation instead of +* consuming the verifier's zext marker, as the +* other ALU32 ops do. +*/ + goto do_alu32_trunc; + }emit_write_y (G0 , ctx);emit_alu (DIV , src, dst, ctx);if (insn_is_zext (&insn[1 ]))return 1 ;break ; + }case BPF_ALU64 |BPF_DIV |BPF_X : -emit_alu (UDIVX , src, dst, ctx); +if (off ==1 ) +emit_alu (SDIVX , src, dst, ctx); +else +emit_alu (UDIVX , src, dst, ctx);break ;case BPF_ALU |BPF_MOD |BPF_X : {const u8 tmp = bpf2sparc[TMP_REG_1 ]; +const bool is_signed = (off ==1 ); ctx->tmp_1_used = true; -emit_write_y (G0 , ctx); -emit_alu3 (DIV , dst, src, tmp, ctx); +if (is_signed) { +const u8 tmp2 = bpf2sparc[TMP_REG_2 ]; + + ctx->tmp_2_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp2, ctx); +emit_write_y (tmp2, ctx); +emit_alu3 (SDIV , dst, src, tmp, ctx); + }else { +emit_write_y (G0 , ctx); +emit_alu3 (DIV , dst, src, tmp, ctx); + }emit_alu3 (MULX , tmp, src, tmp, ctx);emit_alu3 (SUB , dst, tmp, dst, ctx); goto do_alu32_trunc; }case BPF_ALU64 |BPF_MOD |BPF_X : {const u8 tmp = bpf2sparc[TMP_REG_1 ]; +const unsigned int mod = (off ==1 ) ?SDIVX :UDIVX ; ctx->tmp_1_used = true; -emit_alu3 (UDIVX , dst, src, tmp, ctx); +emit_alu3 (mod, dst, src, tmp, ctx);emit_alu3 (MULX , tmp, src, tmp, ctx);emit_alu3 (SUB , dst, tmp, dst, ctx);break ;@@ -1096,33 +1135,62 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)case BPF_ALU64 |BPF_MUL |BPF_K :emit_alu_K (MULX , dst, imm, ctx);break ; -case BPF_ALU |BPF_DIV |BPF_K : +case BPF_ALU |BPF_DIV |BPF_K : { +const bool is_signed = (off ==1 ); +if (imm ==0 )return -EINVAL ; -emit_write_y (G0 , ctx); -emit_alu_K (DIV , dst, imm, ctx); +if (is_signed) { +const u8 tmp = bpf2sparc[TMP_REG_2 ]; + + ctx->tmp_2_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp, ctx); +emit_write_y (tmp, ctx); +emit_alu_K (SDIV , dst, imm, ctx); + }else { +emit_write_y (G0 , ctx); +emit_alu_K (DIV , dst, imm, ctx); + } goto do_alu32_trunc; + }case BPF_ALU64 |BPF_DIV |BPF_K :if (imm ==0 )return -EINVAL ; -emit_alu_K (UDIVX , dst, imm, ctx); +emit_alu_K ((off ==1 ) ?SDIVX :UDIVX , dst, imm, ctx);break ;case BPF_ALU64 |BPF_MOD |BPF_K :case BPF_ALU |BPF_MOD |BPF_K : {const u8 tmp = bpf2sparc[TMP_REG_2 ]; +const bool is_signed = (off ==1 );unsigned int div;if (imm ==0 )return -EINVAL ; - div = (BPF_CLASS (code) ==BPF_ALU64 ) ?UDIVX :DIV ; +if (BPF_CLASS (code) ==BPF_ALU64 ) + div = is_signed ?SDIVX :UDIVX ; +else + div = is_signed ?SDIV :DIV ; ctx->tmp_2_used = true; -if (BPF_CLASS (code) !=BPF_ALU64 ) -emit_write_y (G0 , ctx); +if (BPF_CLASS (code) !=BPF_ALU64 ) { +if (is_signed) { +const u8 tmp3 = bpf2sparc[TMP_REG_3 ]; + + ctx->tmp_3_used = true; + +/* Sign-extend dst into %y for 32-bit sdiv. */ +emit_alu3_K (SRA , dst,31 , tmp3, ctx); +emit_write_y (tmp3, ctx); + }else { +emit_write_y (G0 , ctx); + } + }if (is_simm13 (imm)) {emit (div |IMMED |RS1 (dst) |S13 (imm) |RD (tmp), ctx);emit (MULX |IMMED |RS1 (tmp) |S13 (imm) |RD (tmp), ctx);base-commit:
d1f4b56417a3dc1a0600f960b14f46bd25eda89dSignature · 1 line
2.34.1
Quoted 3 lines · Kuan-Wei Chiu
I saw that your bpf jit patches span multiple architectures. Assuming you don't have access to all these different hardware platforms. Just out of curiosity how were these patches tested?
Correct, I used QEMU for my local testing. For sparc64 I cross-compiled bpf-next 520d7d794202 with sparc64-linux-gnu-gcc 11.4.0, then compared complete lib/test_bpf runs from stock and patched kernels under qemu-system-sparc64 -M sun4u. Ten tests changed from FAIL to PASS.
I also compared stock and patched runs for x86_32, parisc32/64 and MIPS32/64, including MIPS R6.
For LoongArch I reproduced the issue under QEMU and Tiezhu Yang later ran test_bpf with the patch and supplied Tested-by.
Thanks,
Nicholas