riscv, bpf: Add support for signed operations and 32-bit atomics
Subsystems (3)
v3 cover riscv, bpf: Add support for signed operations and 32-bit atomics
Add support for missing signed operations and 32-bit atomics in the RV32 BPF JIT compiler.
The current implementation lacks support for BPF_SDIV, BPF_SMOD, and BPF_MOVSX, ignoring the instruction offset field and treating them as unsigned or zero-extended. Introduce support for these operations by checking the offset field and emitting the corresponding instructions.
Additionally, we leverage the mandatory A extension to natively support 32-bit BPF atomics (and, or, xor, xchg) by mapping them directly to amo*.w instructions. BPF_CMPXCHG continues to fall back to the interpreter.
As a result, test_bpf.ko now runs with 0 failures, and the total number of successfully JIT'ed test cases increases from 843 to 902.
---
Changes in v3:
- Pass insn directly to emit_alu_r32().
- Remove the Fixes: tag and update title.
- Consolidate size, mode, and off into insn for emit_store_r64().
Changes in v2:
- Add missing Fixes tags.
- Fix memory ordering by emitting aq=1, rl=1
Kuan-Wei Chiu (3):
riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT riscv, bpf: Add support for BPF_MOVSX in RV32 JIT riscv, bpf: Add 32 bit atomic operations to RV32 JIT
arch/riscv/net/bpf_jit_comp32.c | 111 +++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 16 deletions(-)
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
PATCH v3 1/3 riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT
The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1.
Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.
Before this patch:
[ 44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times) [ 44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)
After this patch:
[ 16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS [ 16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS [ 16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS [ 16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS
--- arch/riscv/net/bpf_jit_comp32.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +10 -5
@@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, }static void emit_alu_r32 (const s8 *dst,const s8 *src, -struct rv_jit_context *ctx,const u8 op) +struct rv_jit_context *ctx, +const struct bpf_insn *insn) {const s8 *tmp1 = bpf2rv32[TMP_REG_1 ];const s8 *tmp2 = bpf2rv32[TMP_REG_2 ];const s8 *rd =bpf_get_reg32 (dst, tmp1, ctx);const s8 *rs =bpf_get_reg32 (src, tmp2, ctx); +u8 op =BPF_OP (insn->code); +bool is_signed = insn->off ==1 ;switch (op) {case BPF_MOV :
@@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src,emit (rv_mul (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_DIV : -emit (rv_divu (lo (rd),lo (rd),lo (rs)), ctx); +emit (is_signed ?rv_div (lo (rd),lo (rd),lo (rs)) : +rv_divu (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_MOD : -emit (rv_remu (lo (rd),lo (rd),lo (rs)), ctx); +emit (is_signed ?rv_rem (lo (rd),lo (rd),lo (rs)) : +rv_remu (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_LSH :emit (rv_sll (lo (rd),lo (rd),lo (rs)), ctx);
@@ -1041,7 +1046,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,emit_imm32 (tmp2, imm, ctx); src = tmp2; } -emit_alu_r32 (dst, src, ctx,BPF_OP (code)); +emit_alu_r32 (dst, src, ctx, insn);break ;case BPF_ALU |BPF_MOV |BPF_K :
@@ -1065,7 +1070,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,* src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ -emit_alu_r32 (dst, tmp2, ctx,BPF_OP (code)); +emit_alu_r32 (dst, tmp2, ctx, insn);break ;case BPF_ALU |BPF_END |BPF_FROM_LE :
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
[email protected]
AI reviewRe: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT
This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:
Quoted 19 lines
commit
c5a973b7fc82eeb9b84aaf80fc2673a94ab795b2
Author: Kuan-Wei Chiu <[email protected]>riscv, bpf: Add support for
BPF_SDIVandBPF_SMODin RV32 JITThe current rv32 bpf jit compiler incorrectly treats
BPF_SDIVandBPF_SMODas unsigned operations. The BPF instruction set allows signed division and modulo by reusing theBPF_DIVandBPF_MODopcodes with the instruction offset set to 1.Update the
emit_alu_r32()function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.Before this patch:
[ 44.161771]test_bpf: #165ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd!= 0xfffffffd)FAIL (1 times)
[ 44.167385]test_bpf: #166ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd!= 0xfffffffd)FAIL (1 times)
[ 44.171053]test_bpf: #169ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1!= 0xffffffff)FAIL (1 times)
[ 44.172081]test_bpf: #170ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1!= 0xffffffff)FAIL (1 times)
Quoted 7 lines
After this patch:
[ 16.002192]test_bpf: #165ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[ 16.002983]test_bpf: #166ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[ 16.017167]test_bpf: #169ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[ 16.023002]test_bpf: #170ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASSSigned-off-by Kuan-Wei Chiu <[email protected]>
This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/28811594040
Pu Lehui
maintainerRe: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT
Quoted 25 lines · On 2026/7/7 2:12, [email protected] wrote:
Quoted 25 lines
commit
c5a973b7fc82eeb9b84aaf80fc2673a94ab795b2
Author: Kuan-Wei Chiu <[email protected]>riscv, bpf: Add support for
BPF_SDIVandBPF_SMODin RV32 JITThe current rv32 bpf jit compiler incorrectly treats
BPF_SDIVandBPF_SMODas unsigned operations. The BPF instruction set allows signed division and modulo by reusing theBPF_DIVandBPF_MODopcodes with the instruction offset set to 1.Update the
emit_alu_r32()function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.Before this patch:
[ 44.161771]test_bpf: #165ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd!= 0xfffffffd)FAIL (1 times)
[ 44.167385]test_bpf: #166ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd!= 0xfffffffd)FAIL (1 times)
[ 44.171053]test_bpf: #169ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1!= 0xffffffff)FAIL (1 times)
[ 44.172081]test_bpf: #170ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1!= 0xffffffff)FAIL (1 times)After this patch:
[ 16.002192]test_bpf: #165ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[ 16.002983]test_bpf: #166ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[ 16.017167]test_bpf: #169ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[ 16.023002]test_bpf: #170ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS
Quoted 6 lines
Quoted 1 line
Signed-off-by Kuan-Wei Chiu <[email protected]>This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:
Fixes 5f316b65e99f ("riscv, bpf: Add RV32G eBPF JIT")
not need to add fixes tag, as this not a bugfix
Quoted 7 lines
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.mdCI run summary: https://github.com/kernel-patches/bpf/actions/runs/28811594040
Re: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT
In reply to Kuan-Wei ChiuQuoted 71 lines · Kuan-Wei Chiu
The current rv32 bpf jit compiler incorrectly treats
BPF_SDIVandBPF_SMODas unsigned operations. The BPF instruction set allows signed division and modulo by reusing theBPF_DIVandBPF_MODopcodes with the instruction offset set to 1.Update the
emit_alu_r32()function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.Before this patch:
[ 44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times) [ 44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)After this patch:
[ 16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS [ 16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS [ 16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS [ 16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASSSigned-off-by Kuan-Wei Chiu <[email protected]>--- arch/riscv/net/bpf_jit_comp32.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)arch/riscv/net/bpf_jit_comp32.c +9
-4@@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, }static void emit_alu_r32 (const s8 *dst,const s8 *src, -struct rv_jit_context *ctx,const u8 op) +struct rv_jit_context *ctx, +const struct bpf_insn *insn) {const s8 *tmp1 = bpf2rv32[TMP_REG_1 ];const s8 *tmp2 = bpf2rv32[TMP_REG_2 ];const s8 *rd =bpf_get_reg32 (dst, tmp1, ctx);const s8 *rs =bpf_get_reg32 (src, tmp2, ctx); +u8 op =BPF_OP (insn->code); +bool is_signed = insn->off ==1 ;switch (op) {case BPF_MOV :@@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src,emit (rv_mul (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_DIV : -emit (rv_divu (lo (rd),lo (rd),lo (rs)), ctx); +emit (is_signed ?rv_div (lo (rd),lo (rd),lo (rs)) : +rv_divu (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_MOD : -emit (rv_remu (lo (rd),lo (rd),lo (rs)), ctx); +emit (is_signed ?rv_rem (lo (rd),lo (rd),lo (rs)) : +rv_remu (lo (rd),lo (rd),lo (rs)), ctx);break ;case BPF_LSH :emit (rv_sll (lo (rd),lo (rd),lo (rs)), ctx);@@ -1041,7 +1046,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,emit_imm32 (tmp2, imm, ctx); src = tmp2; } -emit_alu_r32 (dst, src, ctx,BPF_OP (code)); +emit_alu_r32 (dst, src, ctx, insn);break ;case BPF_ALU |BPF_MOV |BPF_K :
arch/riscv/net/bpf_jit_comp32.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 1076
Quoted 8 lines
arch/riscv/net/bpf_jit_comp32.c +1
-1@@ -1065,7 +1070,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,* src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ -emit_alu_r32 (dst, tmp2, ctx,BPF_OP (code)); +emit_alu_r32 (dst, tmp2, ctx, insn);break ;case BPF_ALU |BPF_END |BPF_FROM_LE :
lgtm, thanks
PATCH v3 2/3 riscv, bpf: Add support for BPF_MOVSX in RV32 JIT
In reply to Kuan-Wei ChiuThe current rv32 bpf jit compiler incorrectly treats BPF_MOVSX as a standard zero-extended move operation. The bpf instruction set allows sign-extension moves by reusing the BPF_MOV opcode with the instruction offset set to 8, 16, or 32.
Update the bpf_jit_emit_insn() function to check the offset field for both ALU and ALU64 MOV operations. If the offset is non-zero, emit the correct slli and srai instructions to perform the sign extension.
Before this patch:
[ 19.549705] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.551354] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.552576] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.553542] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.554807] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)
After this patch:
[ 17.931172] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 125 PASS [ 17.932198] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 124 PASS [ 17.933039] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 124 PASS [ 17.933918] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 124 PASS [ 17.934751] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 122 PASS
--- arch/riscv/net/bpf_jit_comp32.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
arch/riscv/net/bpf_jit_comp32.c +32 -0
@@ -972,6 +972,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,switch (code) {case BPF_ALU64 |BPF_MOV |BPF_X : +if (insn->off !=0 ) { +const s8 *rd =bpf_get_reg64 (dst, tmp1, ctx); +const s8 *rs =bpf_get_reg64 (src, tmp2, ctx); + +if (insn->off ==8 ) { +emit (rv_slli (lo (rd),lo (rs),24 ), ctx); +emit (rv_srai (lo (rd),lo (rd),24 ), ctx); + }else if (insn->off ==16 ) { +emit (rv_slli (lo (rd),lo (rs),16 ), ctx); +emit (rv_srai (lo (rd),lo (rd),16 ), ctx); + }else { +emit (rv_addi (lo (rd),lo (rs),0 ), ctx); + } +emit (rv_srai (hi (rd),lo (rd),31 ), ctx); +bpf_put_reg64 (dst, rd, ctx); +break ; + } + fallthrough;case BPF_ALU64 |BPF_ADD |BPF_X :case BPF_ALU64 |BPF_ADD |BPF_K :
@@ -1022,6 +1040,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,emit_zext64 (dst, ctx);break ; } +if (insn->off !=0 ) { +const s8 *rd =bpf_get_reg32 (dst, tmp1, ctx); +const s8 *rs =bpf_get_reg32 (src, tmp2, ctx); + +if (insn->off ==8 ) { +emit (rv_slli (lo (rd),lo (rs),24 ), ctx); +emit (rv_srai (lo (rd),lo (rd),24 ), ctx); + }else if (insn->off ==16 ) { +emit (rv_slli (lo (rd),lo (rs),16 ), ctx); +emit (rv_srai (lo (rd),lo (rd),16 ), ctx); + } +bpf_put_reg32 (dst, rd, ctx); +break ; + } fallthrough;case BPF_ALU |BPF_ADD |BPF_X :
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
PATCH v3 3/3 riscv, bpf: Add 32 bit atomic operations to RV32 JIT
In reply to Kuan-Wei ChiuThe RV32 BPF JIT compiler currently only supports the BPF_ADD atomic operation. Other 32 bit atomic operations (and, or, xor, xchg) and their BPF_FETCH variants are not supported and gracefully fall back to the interpreter.
Since the RISC-V A extension is required for Linux on RV32, we can natively support these 32-bit BPF atomic operations by mapping them directly to the corresponding RISC-V amo*.w instructions.
Implement BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, and BPF_XCHG with and without BPF_FETCH. BPF_CMPXCHG requires a more complex lr.w/sc.w loop and is left to fall back to the interpreter.
Before this patch:
[ 138.862161] test_bpf: Summary: 1054 PASSED, 0 FAILED, [843/1042 JIT'ed]
After this patch:
[ 157.024124] test_bpf: Summary: 1054 PASSED, 0 FAILED, [902/1042 JIT'ed]
--- arch/riscv/net/bpf_jit_comp32.c | 64 +++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 11 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +53 -11
@@ -874,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,return 0 ; } -static int emit_store_r64 (const s8 *dst,const s8 *src,s16 off, -struct rv_jit_context *ctx,const u8 size, -const u8 mode) +static int emit_bpf_atomic (const s8 *src,const s8 *rs, +struct rv_jit_context *ctx, +const struct bpf_insn *insn) +{ +s32 imm = insn->imm; +bool is_fetch = (imm &BPF_FETCH ) || (imm ==BPF_XCHG ); +s8 fetch_reg = is_fetch ?lo (rs) :RV_REG_ZERO ; +int aq = is_fetch ?1 :0 ; +int rl = is_fetch ?1 :0 ; + +switch (imm) { +case BPF_ADD : +case BPF_ADD |BPF_FETCH : +emit (rv_amoadd_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_AND : +case BPF_AND |BPF_FETCH : +emit (rv_amoand_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_OR : +case BPF_OR |BPF_FETCH : +emit (rv_amoor_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_XOR : +case BPF_XOR |BPF_FETCH : +emit (rv_amoxor_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_XCHG : +emit (rv_amoswap_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +default : +return -1 ; + } + +if (is_fetch) { +emit (rv_addi (hi (rs),RV_REG_ZERO ,0 ), ctx); +bpf_put_reg64 (src, rs, ctx); + } +return 0 ; +} + +static int emit_store_r64 (const s8 *dst,const s8 *src, +struct rv_jit_context *ctx, +const struct bpf_insn *insn) {const s8 *tmp1 = bpf2rv32[TMP_REG_1 ];const s8 *tmp2 = bpf2rv32[TMP_REG_2 ];const s8 *rd =bpf_get_reg64 (dst, tmp1, ctx);const s8 *rs =bpf_get_reg64 (src, tmp2, ctx); +u8 size =BPF_SIZE (insn->code); +u8 mode =BPF_MODE (insn->code); +s16 off = insn->off;if (mode ==BPF_ATOMIC && size !=BPF_W )return -1 ;
@@ -901,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,case BPF_MEM :emit (rv_sw (RV_REG_T0 ,0 ,lo (rs)), ctx);break ; -case BPF_ATOMIC :/* Only BPF_ADD supported */ -emit (rv_amoadd_w (RV_REG_ZERO ,lo (rs),RV_REG_T0 ,0 ,0 ), - ctx); +case BPF_ATOMIC : +if (emit_bpf_atomic (src, rs, ctx, insn)) +return -1 ;break ; }break ;
@@ -1303,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, src = tmp2; } -if (emit_store_r64 (dst, src, off, ctx,BPF_SIZE (code), -BPF_MODE (code))) +if (emit_store_r64 (dst, src, ctx, insn))return -1 ;break ;case BPF_STX |BPF_ATOMIC |BPF_W : -if (insn->imm !=BPF_ADD ) { +if (insn->imm ==BPF_CMPXCHG ) {pr_info_once ("bpf-jit: not supported: atomic operation %02x ***\n" , insn->imm);return -EFAULT ; } -if (emit_store_r64 (dst, src, off, ctx,BPF_SIZE (code), -BPF_MODE (code))) +if (emit_store_r64 (dst, src, ctx, insn))return -1 ;break ;
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
Pu Lehui
maintainerRe: [PATCH bpf-next v3 3/3] riscv, bpf: Add 32 bit atomic operations to RV32 JIT
Quoted 29 lines · Kuan-Wei Chiu
The RV32 BPF JIT compiler currently only supports the
BPF_ADDatomic operation. Other 32 bit atomic operations (and, or, xor, xchg) and theirBPF_FETCHvariants are not supported and gracefully fall back to the interpreter.Since the RISC-V A extension is required for Linux on RV32, we can natively support these 32-bit BPF atomic operations by mapping them directly to the corresponding RISC-V amo*.w instructions.
Implement
BPF_ADD,BPF_AND,BPF_OR,BPF_XOR, andBPF_XCHGwith and withoutBPF_FETCH.BPF_CMPXCHGrequires a more complex lr.w/sc.w loop and is left to fall back to the interpreter.Before this patch:
[ 138.862161] test_bpf: Summary: 1054 PASSED, 0 FAILED, [843/1042 JIT'ed]After this patch:
[ 157.024124] test_bpf: Summary: 1054 PASSED, 0 FAILED, [902/1042 JIT'ed]Signed-off-by Kuan-Wei Chiu <[email protected]>--- arch/riscv/net/bpf_jit_comp32.c | 64 +++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 11 deletions(-)arch/riscv/net/bpf_jit_comp32.c +0
-0
arch/riscv/net/bpf_jit_comp32.c static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, · line 877
Quoted 7 lines
arch/riscv/net/bpf_jit_comp32.c +1
-3@@ -874,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,return 0 ; } -static int emit_store_r64 (const s8 *dst,const s8 *src,s16 off, -struct rv_jit_context *ctx,const u8 size, -const u8 mode) +static int emit_bpf_atomic(const s8 *src, const s8 *rs,
pls add a dst param to pass in RV_REG_T0. It feels kinda hacky to just use a temp register directly
Quoted 92 lines
arch/riscv/net/bpf_jit_comp32.c +52
-8+struct rv_jit_context *ctx, + const struct bpf_insn *insn) +{ + s32 imm = insn->imm; +bool is_fetch = (imm &BPF_FETCH ) || (imm ==BPF_XCHG ); +s8 fetch_reg = is_fetch ?lo (rs) :RV_REG_ZERO ; +int aq = is_fetch ?1 :0 ; +int rl = is_fetch ?1 :0 ; + +switch (imm) { +case BPF_ADD : +case BPF_ADD |BPF_FETCH : +emit (rv_amoadd_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_AND : +case BPF_AND |BPF_FETCH : +emit (rv_amoand_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_OR : +case BPF_OR |BPF_FETCH : +emit (rv_amoor_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_XOR : +case BPF_XOR |BPF_FETCH : +emit (rv_amoxor_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +case BPF_XCHG : +emit (rv_amoswap_w (fetch_reg,lo (rs),RV_REG_T0 , aq, rl), ctx); +break ; +default : +return -1 ; + } + +if (is_fetch) { +emit (rv_addi (hi (rs),RV_REG_ZERO ,0 ), ctx); +bpf_put_reg64 (src, rs, ctx); + } +return 0 ; +} + +static int emit_store_r64 (const s8 *dst,const s8 *src, +struct rv_jit_context *ctx, +const struct bpf_insn *insn) {const s8 *tmp1 = bpf2rv32[TMP_REG_1 ];const s8 *tmp2 = bpf2rv32[TMP_REG_2 ];const s8 *rd =bpf_get_reg64 (dst, tmp1, ctx);const s8 *rs =bpf_get_reg64 (src, tmp2, ctx); +u8 size =BPF_SIZE (insn->code); +u8 mode =BPF_MODE (insn->code); +s16 off = insn->off;if (mode ==BPF_ATOMIC && size !=BPF_W )return -1 ;@@ -901,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,case BPF_MEM :emit (rv_sw (RV_REG_T0 ,0 ,lo (rs)), ctx);break ; -case BPF_ATOMIC :/* Only BPF_ADD supported */ -emit (rv_amoadd_w (RV_REG_ZERO ,lo (rs),RV_REG_T0 ,0 ,0 ), - ctx); +case BPF_ATOMIC : +if (emit_bpf_atomic (src, rs, ctx, insn)) +return -1 ;break ; }break ;@@ -1303,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, src = tmp2; } -if (emit_store_r64 (dst, src, off, ctx,BPF_SIZE (code), -BPF_MODE (code))) +if (emit_store_r64 (dst, src, ctx, insn))return -1 ;break ;case BPF_STX |BPF_ATOMIC |BPF_W : -if (insn->imm !=BPF_ADD ) { +if (insn->imm ==BPF_CMPXCHG ) {pr_info_once ("bpf-jit: not supported: atomic operation %02x ***\n" , insn->imm);return -EFAULT ; } -if (emit_store_r64 (dst, src, off, ctx,BPF_SIZE (code), -BPF_MODE (code))) +if (emit_store_r64 (dst, src, ctx, insn))return -1 ;break ;