Skip to contentSkip to conversation

riscv, bpf: Add support for signed operations and 32-bit atomics

PATCH Superseded Kuan-Wei Chiu 3 participants 8 messages Jul 6 – Jul 7

Subsystems (3)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v3 cover riscv, bpf: Add support for signed operations and 32-bit atomics

Add support for missing signed operations and 32-bit atomics in the RV32 BPF JIT compiler.

The current implementation lacks support for BPF_SDIV, BPF_SMOD, and BPF_MOVSX, ignoring the instruction offset field and treating them as unsigned or zero-extended. Introduce support for these operations by checking the offset field and emitting the corresponding instructions.

Additionally, we leverage the mandatory A extension to natively support 32-bit BPF atomics (and, or, xor, xchg) by mapping them directly to amo*.w instructions. BPF_CMPXCHG continues to fall back to the interpreter.

As a result, test_bpf.ko now runs with 0 failures, and the total number of successfully JIT'ed test cases increases from 843 to 902.
---
Changes in v3:
- Pass insn directly to emit_alu_r32().
- Remove the Fixes: tag and update title.
- Consolidate size, mode, and off into insn for emit_store_r64().

Changes in v2:
- Add missing Fixes tags.
- Fix memory ordering by emitting aq=1, rl=1

Kuan-Wei Chiu (3):
riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT riscv, bpf: Add support for BPF_MOVSX in RV32 JIT riscv, bpf: Add 32 bit atomic operations to RV32 JIT

 arch/riscv/net/bpf_jit_comp32.c | 111 +++++++++++++++++++++++++++-----
 1 file changed, 95 insertions(+), 16 deletions(-)
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#1 lore raw

PATCH v3 1/3 riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1.

Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.

Before this patch:

[   44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[   44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[   44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)
[   44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)

After this patch:

[   16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[   16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[   16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[   16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS
---
 arch/riscv/net/bpf_jit_comp32.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +10 -5
index 592dd86fbf81..89153946a4e4 100644
@@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src,
 }
 
 static void emit_alu_r32(const s8 *dst, const s8 *src,
-			 struct rv_jit_context *ctx, const u8 op)
+			 struct rv_jit_context *ctx,
+			 const struct bpf_insn *insn)
 {
 	const s8 *tmp1 = bpf2rv32[TMP_REG_1];
 	const s8 *tmp2 = bpf2rv32[TMP_REG_2];
 	const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
 	const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
+	u8 op = BPF_OP(insn->code);
+	bool is_signed = insn->off == 1;
 
 	switch (op) {
 	case BPF_MOV:
@@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src,
 		emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
 		break;
 	case BPF_DIV:
-		emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
+		emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) :
+				 rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
 		break;
 	case BPF_MOD:
-		emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
+		emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) :
+				 rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
 		break;
 	case BPF_LSH:
 		emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
@@ -1041,7 +1046,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 			emit_imm32(tmp2, imm, ctx);
 			src = tmp2;
 		}
-		emit_alu_r32(dst, src, ctx, BPF_OP(code));
+		emit_alu_r32(dst, src, ctx, insn);
 		break;
 
 	case BPF_ALU | BPF_MOV | BPF_K:
@@ -1065,7 +1070,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 		 * src is ignored---choose tmp2 as a dummy register since it
 		 * is not on the stack.
 		 */
-		emit_alu_r32(dst, tmp2, ctx, BPF_OP(code));
+		emit_alu_r32(dst, tmp2, ctx, insn);
 		break;
 
 	case BPF_ALU | BPF_END | BPF_FROM_LE:
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#2 lore raw

Re: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:
Quoted 19 lines

commit c5a973b7fc82eeb9b84aaf80fc2673a94ab795b2
Author: Kuan-Wei Chiu <[email protected]>

riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1.

Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.

Before this patch:
[ 44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[ 44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[ 44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)
[ 44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)

Quoted 7 lines

After this patch:
[ 16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[ 16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[ 16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[ 16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS

This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:

Fixes 5f316b65e99f ("riscv, bpf: Add RV32G eBPF JIT")

---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/28811594040

maintainer
#3 lore raw

Re: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

Quoted 25 lines · On 2026/7/7 2:12, [email protected] wrote:
Quoted 25 lines

commit c5a973b7fc82eeb9b84aaf80fc2673a94ab795b2
Author: Kuan-Wei Chiu <[email protected]>

riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1.

Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.

Before this patch:
[ 44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[ 44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[ 44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)
[ 44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)

After this patch:
[ 16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[ 16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[ 16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[ 16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS

Quoted 6 lines
Quoted 1 line

This looks like a bug fix for the original RV32 BPF JIT implementation. Should this include:

Fixes 5f316b65e99f ("riscv, bpf: Add RV32G eBPF JIT")

not need to add fixes tag, as this not a bugfix

Quoted 7 lines

---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/28811594040

maintainer Reviewed-by on v3 1/3
#4 lore raw

Re: [PATCH bpf-next v3 1/3] riscv, bpf: Add support for BPF_SDIV and BPF_SMOD in RV32 JIT

In reply to Kuan-Wei Chiu
Quoted 71 lines · Kuan-Wei Chiu

The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1.

Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1.

Before this patch:

[   44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[   44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times)
[   44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)
[   44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times)

After this patch:

[   16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS
[   16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS
[   16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS
[   16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS
---
  arch/riscv/net/bpf_jit_comp32.c | 15 ++++++++++-----
  1 file changed, 10 insertions(+), 5 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +9 -4
index 592dd86fbf81..89153946a4e4 100644
@@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src,
  }
 
  static void emit_alu_r32(const s8 *dst, const s8 *src,
-			 struct rv_jit_context *ctx, const u8 op)
+			 struct rv_jit_context *ctx,
+			 const struct bpf_insn *insn)
  {
  	const s8 *tmp1 = bpf2rv32[TMP_REG_1];
  	const s8 *tmp2 = bpf2rv32[TMP_REG_2];
  	const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
  	const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
+	u8 op = BPF_OP(insn->code);
+	bool is_signed = insn->off == 1;
 
  	switch (op) {
  	case BPF_MOV:
@@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src,
  		emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
  		break;
  	case BPF_DIV:
-		emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
+		emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) :
+				 rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
  		break;
  	case BPF_MOD:
-		emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
+		emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) :
+				 rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
  		break;
  	case BPF_LSH:
  		emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
@@ -1041,7 +1046,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
  			emit_imm32(tmp2, imm, ctx);
  			src = tmp2;
  		}
-		emit_alu_r32(dst, src, ctx, BPF_OP(code));
+		emit_alu_r32(dst, src, ctx, insn);
  		break;
 
  	case BPF_ALU | BPF_MOV | BPF_K:

arch/riscv/net/bpf_jit_comp32.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 1076

Quoted 8 lines
arch/riscv/net/bpf_jit_comp32.c +1 -1
@@ -1065,7 +1070,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
  		 * src is ignored---choose tmp2 as a dummy register since it
  		 * is not on the stack.
  		 */
-		emit_alu_r32(dst, tmp2, ctx, BPF_OP(code));
+		emit_alu_r32(dst, tmp2, ctx, insn);
  		break;
 
  	case BPF_ALU | BPF_END | BPF_FROM_LE:

lgtm, thanks

#5 lore raw

PATCH v3 2/3 riscv, bpf: Add support for BPF_MOVSX in RV32 JIT

In reply to Kuan-Wei Chiu

The current rv32 bpf jit compiler incorrectly treats BPF_MOVSX as a standard zero-extended move operation. The bpf instruction set allows sign-extension moves by reusing the BPF_MOV opcode with the instruction offset set to 8, 16, or 32.

Update the bpf_jit_emit_insn() function to check the offset field for both ALU and ALU64 MOV operations. If the offset is non-zero, emit the correct slli and srai instructions to perform the sign extension.

Before this patch:

[   19.549705] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)
[   19.551354] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)
[   19.552576] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)
[   19.553542] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)
[   19.554807] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times)

After this patch:

[   17.931172] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 125 PASS
[   17.932198] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 124 PASS
[   17.933039] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 124 PASS
[   17.933918] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 124 PASS
[   17.934751] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 122 PASS
---
 arch/riscv/net/bpf_jit_comp32.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
arch/riscv/net/bpf_jit_comp32.c +32 -0
index 89153946a4e4..39e2b0b907dc 100644
@@ -972,6 +972,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 
 	switch (code) {
 	case BPF_ALU64 | BPF_MOV | BPF_X:
+		if (insn->off != 0) {
+			const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
+			const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
+
+			if (insn->off == 8) {
+				emit(rv_slli(lo(rd), lo(rs), 24), ctx);
+				emit(rv_srai(lo(rd), lo(rd), 24), ctx);
+			} else if (insn->off == 16) {
+				emit(rv_slli(lo(rd), lo(rs), 16), ctx);
+				emit(rv_srai(lo(rd), lo(rd), 16), ctx);
+			} else {
+				emit(rv_addi(lo(rd), lo(rs), 0), ctx);
+			}
+			emit(rv_srai(hi(rd), lo(rd), 31), ctx);
+			bpf_put_reg64(dst, rd, ctx);
+			break;
+		}
+		fallthrough;
 
 	case BPF_ALU64 | BPF_ADD | BPF_X:
 	case BPF_ALU64 | BPF_ADD | BPF_K:
@@ -1022,6 +1040,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 			emit_zext64(dst, ctx);
 			break;
 		}
+		if (insn->off != 0) {
+			const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
+			const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
+
+			if (insn->off == 8) {
+				emit(rv_slli(lo(rd), lo(rs), 24), ctx);
+				emit(rv_srai(lo(rd), lo(rd), 24), ctx);
+			} else if (insn->off == 16) {
+				emit(rv_slli(lo(rd), lo(rs), 16), ctx);
+				emit(rv_srai(lo(rd), lo(rd), 16), ctx);
+			}
+			bpf_put_reg32(dst, rd, ctx);
+			break;
+		}
 		fallthrough;
 
 	case BPF_ALU | BPF_ADD | BPF_X:
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#6 lore raw

PATCH v3 3/3 riscv, bpf: Add 32 bit atomic operations to RV32 JIT

In reply to Kuan-Wei Chiu

The RV32 BPF JIT compiler currently only supports the BPF_ADD atomic operation. Other 32 bit atomic operations (and, or, xor, xchg) and their BPF_FETCH variants are not supported and gracefully fall back to the interpreter.

Since the RISC-V A extension is required for Linux on RV32, we can natively support these 32-bit BPF atomic operations by mapping them directly to the corresponding RISC-V amo*.w instructions.

Implement BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, and BPF_XCHG with and without BPF_FETCH. BPF_CMPXCHG requires a more complex lr.w/sc.w loop and is left to fall back to the interpreter.

Before this patch:

[  138.862161] test_bpf: Summary: 1054 PASSED, 0 FAILED, [843/1042 JIT'ed]

After this patch:

[  157.024124] test_bpf: Summary: 1054 PASSED, 0 FAILED, [902/1042 JIT'ed]
---
 arch/riscv/net/bpf_jit_comp32.c | 64 +++++++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 11 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +53 -11
index 39e2b0b907dc..bf3fb971294c 100644
@@ -874,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,
 	return 0;
 }
 
-static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,
-			  struct rv_jit_context *ctx, const u8 size,
-			  const u8 mode)
+static int emit_bpf_atomic(const s8 *src, const s8 *rs,
+			   struct rv_jit_context *ctx,
+			   const struct bpf_insn *insn)
+{
+	s32 imm = insn->imm;
+	bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG);
+	s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO;
+	int aq = is_fetch ? 1 : 0;
+	int rl = is_fetch ? 1 : 0;
+
+	switch (imm) {
+	case BPF_ADD:
+	case BPF_ADD | BPF_FETCH:
+		emit(rv_amoadd_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_AND:
+	case BPF_AND | BPF_FETCH:
+		emit(rv_amoand_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_OR:
+	case BPF_OR | BPF_FETCH:
+		emit(rv_amoor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_XOR:
+	case BPF_XOR | BPF_FETCH:
+		emit(rv_amoxor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_XCHG:
+		emit(rv_amoswap_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	default:
+		return -1;
+	}
+
+	if (is_fetch) {
+		emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx);
+		bpf_put_reg64(src, rs, ctx);
+	}
+	return 0;
+}
+
+static int emit_store_r64(const s8 *dst, const s8 *src,
+			  struct rv_jit_context *ctx,
+			  const struct bpf_insn *insn)
 {
 	const s8 *tmp1 = bpf2rv32[TMP_REG_1];
 	const s8 *tmp2 = bpf2rv32[TMP_REG_2];
 	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
 	const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
+	u8 size = BPF_SIZE(insn->code);
+	u8 mode = BPF_MODE(insn->code);
+	s16 off = insn->off;
 
 	if (mode == BPF_ATOMIC && size != BPF_W)
 		return -1;
@@ -901,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,
 		case BPF_MEM:
 			emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
 			break;
-		case BPF_ATOMIC: /* Only BPF_ADD supported */
-			emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0),
-			     ctx);
+		case BPF_ATOMIC:
+			if (emit_bpf_atomic(src, rs, ctx, insn))
+				return -1;
 			break;
 		}
 		break;
@@ -1303,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 			src = tmp2;
 		}
 
-		if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
-				   BPF_MODE(code)))
+		if (emit_store_r64(dst, src, ctx, insn))
 			return -1;
 		break;
 
 	case BPF_STX | BPF_ATOMIC | BPF_W:
-		if (insn->imm != BPF_ADD) {
+		if (insn->imm == BPF_CMPXCHG) {
 			pr_info_once(
 				"bpf-jit: not supported: atomic operation %02x ***\n",
 				insn->imm);
 			return -EFAULT;
 		}
 
-		if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
-				   BPF_MODE(code)))
+		if (emit_store_r64(dst, src, ctx, insn))
 			return -1;
 		break;
 
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
maintainer
#7 lore raw

Re: [PATCH bpf-next v3 3/3] riscv, bpf: Add 32 bit atomic operations to RV32 JIT

Quoted 29 lines · Kuan-Wei Chiu

The RV32 BPF JIT compiler currently only supports the BPF_ADD atomic operation. Other 32 bit atomic operations (and, or, xor, xchg) and their BPF_FETCH variants are not supported and gracefully fall back to the interpreter.

Since the RISC-V A extension is required for Linux on RV32, we can natively support these 32-bit BPF atomic operations by mapping them directly to the corresponding RISC-V amo*.w instructions.

Implement BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, and BPF_XCHG with and without BPF_FETCH. BPF_CMPXCHG requires a more complex lr.w/sc.w loop and is left to fall back to the interpreter.

Before this patch:

[  138.862161] test_bpf: Summary: 1054 PASSED, 0 FAILED, [843/1042 JIT'ed]

After this patch:

[  157.024124] test_bpf: Summary: 1054 PASSED, 0 FAILED, [902/1042 JIT'ed]
---
  arch/riscv/net/bpf_jit_comp32.c | 64 +++++++++++++++++++++++++++------
  1 file changed, 53 insertions(+), 11 deletions(-)
arch/riscv/net/bpf_jit_comp32.c +0 -0
index 39e2b0b907dc..bf3fb971294c 100644

arch/riscv/net/bpf_jit_comp32.c static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, · line 877

Quoted 7 lines
arch/riscv/net/bpf_jit_comp32.c +1 -3
@@ -874,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,
  	return 0;
  }
 
-static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,
-			  struct rv_jit_context *ctx, const u8 size,
-			  const u8 mode)
+static int emit_bpf_atomic(const s8 *src, const s8 *rs,

pls add a dst param to pass in RV_REG_T0. It feels kinda hacky to just use a temp register directly

Quoted 92 lines
arch/riscv/net/bpf_jit_comp32.c +52 -8
+			   struct rv_jit_context *ctx,
+			   const struct bpf_insn *insn)
+{
+	s32 imm = insn->imm;
+	bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG);
+	s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO;
+	int aq = is_fetch ? 1 : 0;
+	int rl = is_fetch ? 1 : 0;
+
+	switch (imm) {
+	case BPF_ADD:
+	case BPF_ADD | BPF_FETCH:
+		emit(rv_amoadd_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_AND:
+	case BPF_AND | BPF_FETCH:
+		emit(rv_amoand_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_OR:
+	case BPF_OR | BPF_FETCH:
+		emit(rv_amoor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_XOR:
+	case BPF_XOR | BPF_FETCH:
+		emit(rv_amoxor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	case BPF_XCHG:
+		emit(rv_amoswap_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx);
+		break;
+	default:
+		return -1;
+	}
+
+	if (is_fetch) {
+		emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx);
+		bpf_put_reg64(src, rs, ctx);
+	}
+	return 0;
+}
+
+static int emit_store_r64(const s8 *dst, const s8 *src,
+			  struct rv_jit_context *ctx,
+			  const struct bpf_insn *insn)
  {
  	const s8 *tmp1 = bpf2rv32[TMP_REG_1];
  	const s8 *tmp2 = bpf2rv32[TMP_REG_2];
  	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
  	const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
+	u8 size = BPF_SIZE(insn->code);
+	u8 mode = BPF_MODE(insn->code);
+	s16 off = insn->off;
 
  	if (mode == BPF_ATOMIC && size != BPF_W)
  		return -1;
@@ -901,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,
  		case BPF_MEM:
  			emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
  			break;
-		case BPF_ATOMIC: /* Only BPF_ADD supported */
-			emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0),
-			     ctx);
+		case BPF_ATOMIC:
+			if (emit_bpf_atomic(src, rs, ctx, insn))
+				return -1;
  			break;
  		}
  		break;
@@ -1303,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
  			src = tmp2;
  		}
 
-		if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
-				   BPF_MODE(code)))
+		if (emit_store_r64(dst, src, ctx, insn))
  			return -1;
  		break;
 
  	case BPF_STX | BPF_ATOMIC | BPF_W:
-		if (insn->imm != BPF_ADD) {
+		if (insn->imm == BPF_CMPXCHG) {
  			pr_info_once(
  				"bpf-jit: not supported: atomic operation %02x ***\n",
  				insn->imm);
  			return -EFAULT;
  		}
 
-		if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
-				   BPF_MODE(code)))
+		if (emit_store_r64(dst, src, ctx, insn))
  			return -1;
  		break;