bpf, riscv64: Support load-acquire and store-release instructions
Subsystems (7)
cover bpf, riscv64: Support load-acquire and store-release instructions
Hi all!
Patchset [1] introduced BPF load-acquire (BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64 JIT compiler support. As a follow-up, this patchset supports load-acquire and store-release instructions for the riscv64 JIT compiler, and introduces some related selftests/ changes.
Specifically:
* PATCH 1 makes insn_def_regno() handle load-acquires properly for bpf_jit_needs_zext() (true for riscv64) architectures
* PATCH 2, 3 from Andrea Parri add the actual support to the riscv64 JIT compiler
* PATCH 4 optimizes code emission by skipping redundant zext instructions inserted by the verifier
* PATCH 5, 6 and 7 are minor selftest/ improvements
* PATCH 8 enables (non-arena) load-acquire/store-release selftests for riscv64
Please refer to individual patches for details. Thanks!
[1] https://lore.kernel.org/all/[email protected]/
Andrea Parri (2):
bpf, riscv64: Introduce emit_load_*() and emit_store_*() bpf, riscv64: Support load-acquire and store-release instructions
Peilin Ye (6):
bpf/verifier: Handle BPF_LOAD_ACQ instructions in insn_def_regno() bpf, riscv64: Skip redundant zext instruction after load-acquire selftests/bpf: Use CAN_USE_LOAD_ACQ_STORE_REL when appropriate selftests/bpf: Avoid passing out-of-range values to __retval() selftests/bpf: Verify zero-extension behavior in load-acquire tests selftests/bpf: Enable non-arena load-acquire/store-release selftests
for riscv64
arch/riscv/net/bpf_jit.h | 15 + arch/riscv/net/bpf_jit_comp64.c | 334 ++++++++++++------ arch/riscv/net/bpf_jit_core.c | 3 +- kernel/bpf/verifier.c | 11 +- tools/testing/selftests/bpf/progs/bpf_misc.h | 5 +- .../bpf/progs/verifier_load_acquire.c | 48 ++- .../selftests/bpf/progs/verifier_precision.c | 5 +- .../bpf/progs/verifier_store_release.c | 39 +- 8 files changed, 314 insertions(+), 146 deletions(-)
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 1/8 bpf/verifier: Handle BPF_LOAD_ACQ instructions in insn_def_regno()
In preparation for supporting BPF load-acquire and store-release instructions for architectures where bpf_jit_needs_zext() returns true (e.g. riscv64), make insn_def_regno() handle load-acquires properly.
--- kernel/bpf/verifier.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
kernel/bpf/verifier.c +7 -4
@@ -3649,13 +3649,16 @@ static int insn_def_regno(const struct bpf_insn *insn)case BPF_ST :return -1 ;case BPF_STX : -if ((BPF_MODE (insn->code) ==BPF_ATOMIC || -BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) && - (insn->imm &BPF_FETCH )) { +if (BPF_MODE (insn->code) ==BPF_ATOMIC || +BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) {if (insn->imm ==BPF_CMPXCHG )return BPF_REG_0 ; -else +else if (insn->imm ==BPF_LOAD_ACQ ) +return insn->dst_reg; +else if (insn->imm &BPF_FETCH )return insn->src_reg; +else +return -1 ; }else {return -1 ; }
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 2/8 bpf, riscv64: Introduce emit_load_*() and emit_store_*()
In reply to Peilin YeFrom: Andrea Parri <[email protected]>
We're planning to add support for the load-acquire and store-release BPF instructions. Define emit_load_<size>() and emit_store_<size>() to enable/facilitate the (re)use of their code.
--- arch/riscv/net/bpf_jit_comp64.c | 242 +++++++++++++++++++------------- 1 file changed, 143 insertions(+), 99 deletions(-)
arch/riscv/net/bpf_jit_comp64.c +143 -99
@@ -473,6 +473,140 @@ static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx)emit (hash, ctx); } +static int emit_load_8 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lb (rd, off, rs), ctx); +else +emit (rv_lbu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lb (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lbu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_16 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lh (rd, off, rs), ctx); +else +emit (rv_lhu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lh (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lhu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_32 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lw (rd, off, rs), ctx); +else +emit (rv_lwu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lw (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lwu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_64 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +emit_ld (rd, off, rs, ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +emit_ld (rd,0 ,RV_REG_T1 , ctx); +return ctx->ninsns - insns_start; +} + +static void emit_store_8 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit (rv_sb (rd, off, rs), ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit (rv_sb (RV_REG_T1 ,0 , rs), ctx); +} + +static void emit_store_16 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit (rv_sh (rd, off, rs), ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit (rv_sh (RV_REG_T1 ,0 , rs), ctx); +} + +static void emit_store_32 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit_sw (rd, off, rs, ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit_sw (RV_REG_T1 ,0 , rs, ctx); +} + +static void emit_store_64 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit_sd (rd, off, rs, ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit_sd (RV_REG_T1 ,0 , rs, ctx); +} +static void emit_atomic (u8 rd,u8 rs,s16 off,s32 imm,bool is64,struct rv_jit_context *ctx) {
@@ -1650,8 +1784,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_LDX |BPF_PROBE_MEM32 |BPF_W :case BPF_LDX |BPF_PROBE_MEM32 |BPF_DW : { -int insn_len, insns_start;bool sign_ext; +int insn_len; sign_ext =BPF_MODE (insn->code) ==BPF_MEMSX ||BPF_MODE (insn->code) ==BPF_PROBE_MEMSX ;
@@ -1663,78 +1797,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,switch (BPF_SIZE (code)) {case BPF_B : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lb (rd, off, rs), ctx); -else -emit (rv_lbu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lb (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lbu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_8 (sign_ext, rd, off, rs, ctx);break ;case BPF_H : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lh (rd, off, rs), ctx); -else -emit (rv_lhu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lh (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lhu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_16 (sign_ext, rd, off, rs, ctx);break ;case BPF_W : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lw (rd, off, rs), ctx); -else -emit (rv_lwu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lw (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lwu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_32 (sign_ext, rd, off, rs, ctx);break ;case BPF_DW : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -emit_ld (rd, off, rs, ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -emit_ld (rd,0 ,RV_REG_T1 , ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_64 (sign_ext, rd, off, rs, ctx);break ; }
@@ -1879,44 +1951,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,/* STX: *(size *)(dst + off) = src */ case BPF_STX |BPF_MEM |BPF_B : -if (is_12b_int (off)) { -emit (rv_sb (rd, off, rs), ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit (rv_sb (RV_REG_T1 ,0 , rs), ctx); +emit_store_8 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_H : -if (is_12b_int (off)) { -emit (rv_sh (rd, off, rs), ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit (rv_sh (RV_REG_T1 ,0 , rs), ctx); +emit_store_16 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_W : -if (is_12b_int (off)) { -emit_sw (rd, off, rs, ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit_sw (RV_REG_T1 ,0 , rs, ctx); +emit_store_32 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_DW : -if (is_12b_int (off)) { -emit_sd (rd, off, rs, ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit_sd (RV_REG_T1 ,0 , rs, ctx); +emit_store_64 (rd, off, rs, ctx);break ;case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW :
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 3/8 bpf, riscv64: Support load-acquire and store-release instructions
In reply to Peilin YeFrom: Andrea Parri <[email protected]>
Support BPF load-acquire (BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions in the riscv64 JIT compiler. For example, consider the following 64-bit load-acquire (assuming little-endian):
db 10 00 00 00 01 00 00 r1 = load_acquire((u64 *)(r1 + 0x0)) 95 00 00 00 00 00 00 00 exit
opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX
imm (0x00000100): BPF_LOAD_ACQ
The JIT compiler will emit an LD instruction followed by a FENCE R,RW instruction for the above, e.g.:
ld x7,0(x6)
fence r,rw
Similarly, consider the following 16-bit store-release:
cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) 95 00 00 00 00 00 00 00 exit
opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX
imm (0x00000110): BPF_STORE_REL
A FENCE RW,W instruction followed by an SH instruction will be emitted, e.g.:
fence rw,w
sh x2,0(x4)
8-bit and 16-bit load-acquires are zero-extending (cf., LBU, LHU). The verifier always rejects misaligned load-acquires/store-releases (even if BPF_F_ANY_ALIGNMENT is set), so the emitted load and store instructions are guaranteed to be single-copy atomic.
Introduce primitives to emit the relevant (and the most common/used in the kernel) fences, i.e. fences with R -> RW, RW -> W and RW -> RW.
Rename emit_atomic() to emit_atomic_rmw() to make it clear that it only handles RMW atomics, and replace its is64 parameter to allow to perform the required checks on the opsize (BPF_SIZE(code)).
[[email protected]: whitespace changes; cosmetic changes to commit
title and message]
--- arch/riscv/net/bpf_jit.h | 15 +++++++ arch/riscv/net/bpf_jit_comp64.c | 77 ++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-)
arch/riscv/net/bpf_jit.h +15 -0
@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ)return rv_i_insn (imm11_0,0 ,0 ,0 ,0xf ); } +static inline void emit_fence_r_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x2 ,0x3 ), ctx); +} + +static inline void emit_fence_rw_w (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x1 ), ctx); +} + +static inline void emit_fence_rw_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x3 ), ctx); +} +static inline u32 rv_nop (void ) {return rv_i_insn (0 ,0 ,0 ,0 ,0x13 );
arch/riscv/net/bpf_jit_comp64.c +70 -7
114 lines not on this page · raw message
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 4/8 bpf, riscv64: Skip redundant zext instruction after load-acquire
In reply to Peilin YeCurrently, the verifier inserts a zext instruction right after every 8-, 16- or 32-bit load-acquire, which is already zero-extending. Skip such redundant zext instructions.
While we are here, update that already-obsolete comment about "skip the next instruction" in build_body(). Also change emit_atomic_rmw()'s parameters to keep it consistent with emit_atomic_ld_st().
Note that checking 'insn[1]' relies on 'insn' not being the last instruction, which should have been guaranteed by the verifier; we already use 'insn[1]' elsewhere in the file for similar purposes. Additionally, we don't check if 'insn[1]' is actually a zext for our load-acquire's dst_reg, or some other registers - in other words, here we are relying on the verifier to always insert a redundant zext right after a 8/16/32-bit load-acquire, for its dst_reg.
--- arch/riscv/net/bpf_jit_comp64.c | 23 ++++++++++++++++++----- arch/riscv/net/bpf_jit_core.c | 3 +-- 2 files changed, 19 insertions(+), 7 deletions(-)
arch/riscv/net/bpf_jit_comp64.c +18 -5
@@ -607,8 +607,13 @@ static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)emit_sd (RV_REG_T1 ,0 , rs, ctx); } -static int emit_atomic_ld_st (u8 rd,u8 rs,s16 off,s32 imm,u8 code,struct rv_jit_context *ctx) +static int emit_atomic_ld_st (u8 rd,u8 rs,const struct bpf_insn *insn, +struct rv_jit_context *ctx) { +u8 code = insn->code; +s32 imm = insn->imm; +s16 off = insn->off; +switch (imm) {/* dst_reg = load_acquire(src_reg + off16) */ case BPF_LOAD_ACQ :
@@ -627,6 +632,12 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, s16 off, s32 imm, u8 code, struct rv_break ; }emit_fence_r_rw (ctx); + +/* If our next insn is a redundant zext, return 1 to tell +* build_body() to skip it. +*/ +if (BPF_SIZE (code) !=BPF_DW &&insn_is_zext (&insn[1 ])) +return 1 ;break ;/* store_release(dst_reg + off16, src_reg) */ case BPF_STORE_REL :
@@ -654,10 +665,12 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, s16 off, s32 imm, u8 code, struct rv_return 0 ; } -static int emit_atomic_rmw (u8 rd,u8 rs,s16 off,s32 imm,u8 code, +static int emit_atomic_rmw (u8 rd,u8 rs,const struct bpf_insn *insn,struct rv_jit_context *ctx) { -u8 r0; +u8 r0, code = insn->code; +s16 off = insn->off; +s32 imm = insn->imm;int jmp_offset;bool is64;
@@ -2026,9 +2039,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW :if (bpf_atomic_is_load_store (insn)) - ret =emit_atomic_ld_st (rd, rs, off, imm, code, ctx); + ret =emit_atomic_ld_st (rd, rs, insn, ctx);else - ret =emit_atomic_rmw (rd, rs, off, imm, code, ctx); + ret =emit_atomic_rmw (rd, rs, insn, ctx);break ;case BPF_STX |BPF_PROBE_MEM32 |BPF_B :
arch/riscv/net/bpf_jit_core.c +1 -2
@@ -26,9 +26,8 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)int ret; ret =bpf_jit_emit_insn (insn, ctx, extra_pass); -/* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ if (ret >0 ) - i++; + i++;/* skip the next instruction */ if (offset) offset[i] = ctx->ninsns;if (ret <0 )
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 5/8 selftests/bpf: Use CAN_USE_LOAD_ACQ_STORE_REL when appropriate
In reply to Peilin YeInstead of open-coding the conditions, use
'#ifdef CAN_USE_LOAD_ACQ_STORE_REL' to guard the following tests:
verifier_precision/bpf_load_acquire
verifier_precision/bpf_store_release
verifier_store_release/*
Note that, for the first two tests in verifier_precision.c, switching to '#ifdef CAN_USE_LOAD_ACQ_STORE_REL' means also checking if '__clang_major__ >= 18', which has already been guaranteed by the outer '#if' check.
--- tools/testing/selftests/bpf/progs/verifier_precision.c | 5 ++--- tools/testing/selftests/bpf/progs/verifier_store_release.c | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-)
tools/testing/selftests/bpf/progs/verifier_precision.c +2 -3
@@ -91,8 +91,7 @@ __naked int bpf_end_bswap(void) ::: __clobber_all); } -#if defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#ifdef CAN_USE_LOAD_ACQ_STORE_REL SEC("?raw_tp") __success __log_level(2)
@@ -138,7 +137,7 @@ __naked int bpf_store_release(void) : __clobber_all); } -#endif /* load-acquire, store-release */ +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ #endif /* v4 instruction */ SEC ("?raw_tp" )
tools/testing/selftests/bpf/progs/verifier_store_release.c +3 -4
@@ -6,8 +6,7 @@#include "../../../include/linux/filter.h" #include "bpf_misc.h" -#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#ifdef CAN_USE_LOAD_ACQ_STORE_REL SEC("socket")__description ("store-release, 8-bit" )
@@ -271,7 +270,7 @@ __naked void store_release_with_invalid_reg(void) : __clobber_all); } -#else +#else /* CAN_USE_LOAD_ACQ_STORE_REL */ SEC("socket")__description ("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test" )
@@ -281,6 +280,6 @@ int dummy_test(void)return 0 ; } -#endif +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ char _license[] SEC("license") = "GPL";
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 6/8 selftests/bpf: Avoid passing out-of-range values to __retval()
In reply to Peilin YeCurrently, we pass 0x1234567890abcdef to __retval() for the following two tests:
verifier_load_acquire/load_acquire_64
verifier_store_release/store_release_64
However, the upper 32 bits of that value are being ignored, since __retval() expects an int. Actually, the tests would still pass even if I change '__retval(0x1234567890abcdef)' to e.g. '__retval(0x90abcdef)'.
Restructure the tests a bit to test the entire 64-bit values properly. Do the same to their 8-, 16- and 32-bit variants as well to keep the style consistent.
--- .../bpf/progs/verifier_load_acquire.c | 40 +++++++++++++------ .../bpf/progs/verifier_store_release.c | 32 +++++++++++---- 2 files changed, 52 insertions(+), 20 deletions(-)
tools/testing/selftests/bpf/progs/verifier_load_acquire.c +28 -12
@@ -10,65 +10,81 @@ SEC("socket") __description("load-acquire, 8-bit") -__success __success_unpriv __retval(0x12) +__success __success_unpriv __retval(0) __naked void load_acquire_8(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x12;" "*(u8 *)(r10 - 1) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1)) + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1)) : __clobber_all); } SEC("socket") __description("load-acquire, 16-bit") -__success __success_unpriv __retval(0x1234) +__success __success_unpriv __retval(0) __naked void load_acquire_16(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x1234;" "*(u16 *)(r10 - 2) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2)) + BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2)) : __clobber_all); } SEC("socket") __description("load-acquire, 32-bit") -__success __success_unpriv __retval(0x12345678) +__success __success_unpriv __retval(0) __naked void load_acquire_32(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x12345678;" "*(u32 *)(r10 - 4) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4)) + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4)) : __clobber_all); } SEC("socket") __description("load-acquire, 64-bit") -__success __success_unpriv __retval(0x1234567890abcdef) +__success __success_unpriv __retval(0) __naked void load_acquire_64(void) { asmvolatile ( +"r0 = 0;" "r1 = 0x1234567890abcdef ll;" "*(u64 *)(r10 - 8) = r1;" -".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8)); +".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)) + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) : __clobber_all); }
tools/testing/selftests/bpf/progs/verifier_store_release.c +24 -8
@@ -10,13 +10,17 @@ SEC("socket") __description("store-release, 8-bit") -__success __success_unpriv __retval(0x12) +__success __success_unpriv __retval(0) __naked void store_release_8(void) { asm volatile ( + "r0 = 0;" "w1 = 0x12;" ".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1); - "w0 = *(u8 *)(r10 - 1);" + "w2 = *(u8 *)(r10 - 1);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,
@@ -26,13 +30,17 @@ __naked void store_release_8(void) SEC("socket") __description("store-release, 16-bit") -__success __success_unpriv __retval(0x1234) +__success __success_unpriv __retval(0) __naked void store_release_16(void) { asm volatile ( + "r0 = 0;" "w1 = 0x1234;" ".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1); - "w0 = *(u16 *)(r10 - 2);" + "w2 = *(u16 *)(r10 - 2);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,
@@ -42,13 +50,17 @@ __naked void store_release_16(void) SEC("socket") __description("store-release, 32-bit") -__success __success_unpriv __retval(0x12345678) +__success __success_unpriv __retval(0) __naked void store_release_32(void) { asm volatile ( + "r0 = 0;" "w1 = 0x12345678;" ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1); - "w0 = *(u32 *)(r10 - 4);" + "w2 = *(u32 *)(r10 - 4);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,
@@ -58,13 +70,17 @@ __naked void store_release_32(void) SEC("socket") __description("store-release, 64-bit") -__success __success_unpriv __retval(0x1234567890abcdef) +__success __success_unpriv __retval(0) __naked void store_release_64(void) { asm volatile ( + "r0 = 0;" "r1 = 0x1234567890abcdef ll;" ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1); - "r0 = *(u64 *)(r10 - 8);" + "r2 = *(u64 *)(r10 - 8);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 7/8 selftests/bpf: Verify zero-extension behavior in load-acquire tests
In reply to Peilin YeVerify that 8-, 16- and 32-bit load-acquires are zero-extending by using immediate values with their highest bit set. Do the same for the 64-bit variant to keep the style consistent.
--- tools/testing/selftests/bpf/progs/verifier_load_acquire.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
tools/testing/selftests/bpf/progs/verifier_load_acquire.c +4 -4
@@ -15,7 +15,7 @@ __naked void load_acquire_8(void) { asmvolatile ("r0 = 0;" -"w1 = 0x12;" +"w1 = 0xfe;" "*(u8 *)(r10 - 1) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); "if r2 == r1 goto 1f;"
@@ -35,7 +35,7 @@ __naked void load_acquire_16(void) { asmvolatile ("r0 = 0;" -"w1 = 0x1234;" +"w1 = 0xfedc;" "*(u16 *)(r10 - 2) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); "if r2 == r1 goto 1f;"
@@ -55,7 +55,7 @@ __naked void load_acquire_32(void) { asmvolatile ("r0 = 0;" -"w1 = 0x12345678;" +"w1 = 0xfedcba09;" "*(u32 *)(r10 - 4) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); "if r2 == r1 goto 1f;"
@@ -75,7 +75,7 @@ __naked void load_acquire_64(void) { asmvolatile ("r0 = 0;" -"r1 = 0x1234567890abcdef ll;" +"r1 = 0xfedcba0987654321 ll;" "*(u64 *)(r10 - 8) = r1;" ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); "if r2 == r1 goto 1f;"
Signature · 1 line
2.49.0.901.g37484f566f-goog
PATCH 8/8 selftests/bpf: Enable non-arena load-acquire/store-release selftests for riscv64
In reply to Peilin YeFor riscv64, enable all BPF_{LOAD_ACQ,STORE_REL} selftests except the arena_atomics/* ones (not guarded behind CAN_USE_LOAD_ACQ_STORE_REL), since arena access is not yet supported.
--- tools/testing/selftests/bpf/progs/bpf_misc.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
tools/testing/selftests/bpf/progs/bpf_misc.h +3 -2
@@ -225,8 +225,9 @@#define CAN_USE_BPF_ST #endif -#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 ))#define CAN_USE_LOAD_ACQ_STORE_REL #endif
Signature · 1 line
2.49.0.901.g37484f566f-goog
Re: [PATCH bpf-next 2/8] bpf, riscv64: Introduce emit_load_*() and emit_store_*()
In reply to Peilin YeQuoted 291 lines · Peilin Ye
From: Andrea Parri <[email protected]>
We're planning to add support for the load-acquire and store-release BPF instructions. Define emit_load_<size>() and emit_store_<size>() to enable/facilitate the (re)use of their code.
Tested-by Peilin Ye <[email protected]>Signed-off-by Andrea Parri <[email protected]>[[email protected]: cosmetic change to commit title]Signed-off-by Peilin Ye <[email protected]>--- arch/riscv/net/bpf_jit_comp64.c | 242 +++++++++++++++++++------------- 1 file changed, 143 insertions(+), 99 deletions(-)arch/riscv/net/bpf_jit_comp64.c +142
-93@@ -473,6 +473,140 @@ static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx)emit (hash, ctx); } +static int emit_load_8 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lb (rd, off, rs), ctx); +else +emit (rv_lbu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lb (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lbu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_16 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lh (rd, off, rs), ctx); +else +emit (rv_lhu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lh (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lhu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_32 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lw (rd, off, rs), ctx); +else +emit (rv_lwu (rd, off, rs), ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +if (sign_ext) +emit (rv_lw (rd,0 ,RV_REG_T1 ), ctx); +else +emit (rv_lwu (rd,0 ,RV_REG_T1 ), ctx); +return ctx->ninsns - insns_start; +} + +static int emit_load_64 (bool sign_ext,u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +int insns_start; + +if (is_12b_int (off)) { + insns_start = ctx->ninsns; +emit_ld (rd, off, rs, ctx); +return ctx->ninsns - insns_start; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); + insns_start = ctx->ninsns; +emit_ld (rd,0 ,RV_REG_T1 , ctx); +return ctx->ninsns - insns_start; +} + +static void emit_store_8 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit (rv_sb (rd, off, rs), ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit (rv_sb (RV_REG_T1 ,0 , rs), ctx); +} + +static void emit_store_16 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit (rv_sh (rd, off, rs), ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit (rv_sh (RV_REG_T1 ,0 , rs), ctx); +} + +static void emit_store_32 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit_sw (rd, off, rs, ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit_sw (RV_REG_T1 ,0 , rs, ctx); +} + +static void emit_store_64 (u8 rd,s32 off,u8 rs,struct rv_jit_context *ctx) +{ +if (is_12b_int (off)) { +emit_sd (rd, off, rs, ctx); +return ; + } + +emit_imm (RV_REG_T1 , off, ctx); +emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); +emit_sd (RV_REG_T1 ,0 , rs, ctx); +} +static void emit_atomic (u8 rd,u8 rs,s16 off,s32 imm,bool is64,struct rv_jit_context *ctx) {@@ -1650,8 +1784,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_LDX |BPF_PROBE_MEM32 |BPF_W :case BPF_LDX |BPF_PROBE_MEM32 |BPF_DW : { -int insn_len, insns_start;bool sign_ext; +int insn_len; sign_ext =BPF_MODE (insn->code) ==BPF_MEMSX ||BPF_MODE (insn->code) ==BPF_PROBE_MEMSX ;@@ -1663,78 +1797,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,switch (BPF_SIZE (code)) {case BPF_B : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lb (rd, off, rs), ctx); -else -emit (rv_lbu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lb (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lbu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_8 (sign_ext, rd, off, rs, ctx);break ;case BPF_H : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lh (rd, off, rs), ctx); -else -emit (rv_lhu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lh (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lhu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_16 (sign_ext, rd, off, rs, ctx);break ;case BPF_W : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lw (rd, off, rs), ctx); -else -emit (rv_lwu (rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -if (sign_ext) -emit (rv_lw (rd,0 ,RV_REG_T1 ), ctx); -else -emit (rv_lwu (rd,0 ,RV_REG_T1 ), ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_32 (sign_ext, rd, off, rs, ctx);break ;case BPF_DW : -if (is_12b_int (off)) { - insns_start = ctx->ninsns; -emit_ld (rd, off, rs, ctx); - insn_len = ctx->ninsns - insns_start; -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rs, ctx); - insns_start = ctx->ninsns; -emit_ld (rd,0 ,RV_REG_T1 , ctx); - insn_len = ctx->ninsns - insns_start; + insn_len =emit_load_64 (sign_ext, rd, off, rs, ctx);break ; }@@ -1879,44 +1951,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,/* STX: *(size *)(dst + off) = src */ case BPF_STX |BPF_MEM |BPF_B : -if (is_12b_int (off)) { -emit (rv_sb (rd, off, rs), ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit (rv_sb (RV_REG_T1 ,0 , rs), ctx); +emit_store_8 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_H : -if (is_12b_int (off)) { -emit (rv_sh (rd, off, rs), ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit (rv_sh (RV_REG_T1 ,0 , rs), ctx); +emit_store_16 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_W : -if (is_12b_int (off)) { -emit_sw (rd, off, rs, ctx); -break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit_sw (RV_REG_T1 ,0 , rs, ctx); +emit_store_32 (rd, off, rs, ctx);break ;case BPF_STX |BPF_MEM |BPF_DW : -if (is_12b_int (off)) { -emit_sd (rd, off, rs, ctx);
arch/riscv/net/bpf_jit_comp64.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 1966
Quoted 10 lines
arch/riscv/net/bpf_jit_comp64.c +1
-6-break ; - } - -emit_imm (RV_REG_T1 , off, ctx); -emit_add (RV_REG_T1 ,RV_REG_T1 , rd, ctx); -emit_sd (RV_REG_T1 ,0 , rs, ctx); +emit_store_64 (rd, off, rs, ctx);break ;case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW :
Quoted 195 lines · Peilin Ye
From: Andrea Parri <[email protected]>
Support BPF load-acquire (
BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions in the riscv64 JIT compiler. For example, consider the following 64-bit load-acquire (assuming little-endian):db 10 00 00 00 01 00 00 r1 = load_acquire((u64 *)(r1 + 0x0)) 95 00 00 00 00 00 00 00 exit opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX imm (0x00000100): BPF_LOAD_ACQThe JIT compiler will emit an LD instruction followed by a FENCE R,RW instruction for the above, e.g.:
ld x7,0(x6) fence r,rwSimilarly, consider the following 16-bit store-release:
cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) 95 00 00 00 00 00 00 00 exit opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX imm (0x00000110): BPF_STORE_RELA FENCE RW,W instruction followed by an SH instruction will be emitted, e.g.:
fence rw,w sh x2,0(x4)8-bit and 16-bit load-acquires are zero-extending (cf., LBU, LHU). The verifier always rejects misaligned load-acquires/store-releases (even if
BPF_F_ANY_ALIGNMENTis set), so the emitted load and store instructions are guaranteed to be single-copy atomic.Introduce primitives to emit the relevant (and the most common/used in the kernel) fences, i.e. fences with R -> RW, RW -> W and RW -> RW.
Rename
emit_atomic()toemit_atomic_rmw()to make it clear that it only handles RMW atomics, and replace its is64 parameter to allow to perform the required checks on the opsize (BPF_SIZE(code)).Tested-by Peilin Ye <[email protected]>Signed-off-by Andrea Parri <[email protected]>[[email protected]: whitespace changes; cosmetic changes to commit
title and message]Signed-off-by Peilin Ye <[email protected]>--- arch/riscv/net/bpf_jit.h | 15 +++++++ arch/riscv/net/bpf_jit_comp64.c | 77 ++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-)arch/riscv/net/bpf_jit.h +15
-0@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ)return rv_i_insn (imm11_0,0 ,0 ,0 ,0xf ); } +static inline void emit_fence_r_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x2 ,0x3 ), ctx); +} + +static inline void emit_fence_rw_w (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x1 ), ctx); +} + +static inline void emit_fence_rw_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x3 ), ctx); +} +static inline u32 rv_nop (void ) {return rv_i_insn (0 ,0 ,0 ,0 ,0x13 );arch/riscv/net/bpf_jit_comp64.c +69
-6@@ -607,11 +607,65 @@ static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)emit_sd (RV_REG_T1 ,0 , rs, ctx); } -static void emit_atomic (u8 rd,u8 rs,s16 off,s32 imm,bool is64, -struct rv_jit_context *ctx) +static int emit_atomic_ld_st (u8 rd,u8 rs,s16 off,s32 imm,u8 code,struct rv_jit_context *ctx) +{ +switch (imm) { +/* dst_reg = load_acquire(src_reg + off16) */ +case BPF_LOAD_ACQ : +switch (BPF_SIZE (code)) { +case BPF_B : +emit_load_8 (false, rd, off, rs, ctx); +break ; +case BPF_H : +emit_load_16 (false, rd, off, rs, ctx); +break ; +case BPF_W : +emit_load_32 (false, rd, off, rs, ctx); +break ; +case BPF_DW : +emit_load_64 (false, rd, off, rs, ctx); +break ; + } +emit_fence_r_rw (ctx); +break ; +/* store_release(dst_reg + off16, src_reg) */ +case BPF_STORE_REL : +emit_fence_rw_w (ctx); +switch (BPF_SIZE (code)) { +case BPF_B : +emit_store_8 (rd, off, rs, ctx); +break ; +case BPF_H : +emit_store_16 (rd, off, rs, ctx); +break ; +case BPF_W : +emit_store_32 (rd, off, rs, ctx); +break ; +case BPF_DW : +emit_store_64 (rd, off, rs, ctx); +break ; + } +break ; +default : +pr_err_once ("bpf-jit: invalid atomic load/store opcode %02x\n" , imm); +return -EINVAL ; + } + +return 0 ; +} + +static int emit_atomic_rmw (u8 rd,u8 rs,s16 off,s32 imm,u8 code, +struct rv_jit_context *ctx) {u8 r0;int jmp_offset; +bool is64; + +if (BPF_SIZE (code) !=BPF_W &&BPF_SIZE (code) !=BPF_DW ) { +pr_err_once ("bpf-jit: 1- and 2-byte RMW atomics are not supported\n" ); +return -EINVAL ; + } + is64 =BPF_SIZE (code) ==BPF_DW ;if (off) {if (is_12b_int (off)) {@@ -688,9 +742,14 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); jmp_offset =ninsns_rvoff (-6 );emit (rv_bne (RV_REG_T3 ,0 , jmp_offset >>1 ), ctx); -emit (rv_fence (0x3 ,0x3 ), ctx); +emit_fence_rw_rw (ctx);break ; +default : +pr_err_once ("bpf-jit: invalid atomic RMW opcode %02x\n" , imm); +return -EINVAL ; } + +return 0 ; }#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)@@ -1259,7 +1318,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, {bool is64 =BPF_CLASS (insn->code) ==BPF_ALU64 ||BPF_CLASS (insn->code) ==BPF_JMP ; -int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; +int s, e, rvoff, ret =0 , i = insn - ctx->prog->insnsi;struct bpf_prog_aux *aux = ctx->prog->aux;u8 rd =-1 , rs =-1 , code = insn->code;s16 off = insn->off;@@ -1962,10 +2021,14 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_STX |BPF_MEM |BPF_DW :emit_store_64 (rd, off, rs, ctx);break ; +case BPF_STX |BPF_ATOMIC |BPF_B : +case BPF_STX |BPF_ATOMIC |BPF_H :case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW : -emit_atomic (rd, rs, off, imm, -BPF_SIZE (code) ==BPF_DW , ctx); +if (bpf_atomic_is_load_store (insn)) + ret =emit_atomic_ld_st (rd, rs, off, imm, code, ctx); +else + ret =emit_atomic_rmw (rd, rs, off, imm, code, ctx);break ;case BPF_STX |BPF_PROBE_MEM32 |BPF_B :
arch/riscv/net/bpf_jit_comp64.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 2119
Quoted 8 lines
arch/riscv/net/bpf_jit_comp64.c +1
-1@@ -2050,7 +2113,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,return -EINVAL ; } -return 0 ; +return ret; }void bpf_jit_build_prologue (struct rv_jit_context *ctx,bool is_subprog)
Re: [PATCH bpf-next 4/8] bpf, riscv64: Skip redundant zext instruction after load-acquire
In reply to Peilin YeQuoted 86 lines · Peilin Ye
Currently, the verifier inserts a zext instruction right after every 8-, 16- or 32-bit load-acquire, which is already zero-extending. Skip such redundant zext instructions.
While we are here, update that already-obsolete comment about "skip the next instruction" in
build_body(). Also change emit_atomic_rmw()'s parameters to keep it consistent withemit_atomic_ld_st().Note that checking 'insn[1]' relies on 'insn' not being the last instruction, which should have been guaranteed by the verifier; we already use 'insn[1]' elsewhere in the file for similar purposes. Additionally, we don't check if 'insn[1]' is actually a zext for our load-acquire's
dst_reg, or some other registers - in other words, here we are relying on the verifier to always insert a redundant zext right after a 8/16/32-bit load-acquire, for itsdst_reg.Signed-off-by Peilin Ye <[email protected]>--- arch/riscv/net/bpf_jit_comp64.c | 23 ++++++++++++++++++----- arch/riscv/net/bpf_jit_core.c | 3 +-- 2 files changed, 19 insertions(+), 7 deletions(-)arch/riscv/net/bpf_jit_comp64.c +18
-5@@ -607,8 +607,13 @@ static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)emit_sd (RV_REG_T1 ,0 , rs, ctx); } -static int emit_atomic_ld_st (u8 rd,u8 rs,s16 off,s32 imm,u8 code,struct rv_jit_context *ctx) +static int emit_atomic_ld_st (u8 rd,u8 rs,const struct bpf_insn *insn, +struct rv_jit_context *ctx) { +u8 code = insn->code; +s32 imm = insn->imm; +s16 off = insn->off; +switch (imm) {/* dst_reg = load_acquire(src_reg + off16) */ case BPF_LOAD_ACQ :@@ -627,6 +632,12 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, s16 off, s32 imm, u8 code, struct rv_break ; }emit_fence_r_rw (ctx); + +/* If our next insn is a redundant zext, return 1 to tell +* build_body() to skip it. +*/ +if (BPF_SIZE (code) !=BPF_DW &&insn_is_zext (&insn[1 ])) +return 1 ;break ;/* store_release(dst_reg + off16, src_reg) */ case BPF_STORE_REL :@@ -654,10 +665,12 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, s16 off, s32 imm, u8 code, struct rv_return 0 ; } -static int emit_atomic_rmw (u8 rd,u8 rs,s16 off,s32 imm,u8 code, +static int emit_atomic_rmw (u8 rd,u8 rs,const struct bpf_insn *insn,struct rv_jit_context *ctx) { -u8 r0; +u8 r0, code = insn->code; +s16 off = insn->off; +s32 imm = insn->imm;int jmp_offset;bool is64;@@ -2026,9 +2039,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW :if (bpf_atomic_is_load_store (insn)) - ret =emit_atomic_ld_st (rd, rs, off, imm, code, ctx); + ret =emit_atomic_ld_st (rd, rs, insn, ctx);else - ret =emit_atomic_rmw (rd, rs, off, imm, code, ctx); + ret =emit_atomic_rmw (rd, rs, insn, ctx);break ;case BPF_STX |BPF_PROBE_MEM32 |BPF_B :arch/riscv/net/bpf_jit_core.c +0
-0
arch/riscv/net/bpf_jit_core.c static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) · line 33
Quoted 10 lines
arch/riscv/net/bpf_jit_core.c +1
-2@@ -26,9 +26,8 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)int ret; ret =bpf_jit_emit_insn (insn, ctx, extra_pass); -/* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ if (ret >0 ) - i++; + i++;/* skip the next instruction */ if (offset) offset[i] = ctx->ninsns;if (ret <0 )
Pu Lehui
maintainerQuoted 37 lines · Peilin Ye
Hi all!
Patchset [1] introduced BPF load-acquire (
BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64 JIT compiler support. As a follow-up, this patchset supports load-acquire and store-release instructions for the riscv64 JIT compiler, and introduces some related selftests/ changes.Specifically:
* PATCH 1 makes
insn_def_regno()handle load-acquires properly forbpf_jit_needs_zext()(true for riscv64) architectures
* PATCH 2, 3 from Andrea Parri add the actual support to the riscv64 JIT compiler
* PATCH 4 optimizes code emission by skipping redundant zext instructions inserted by the verifier
* PATCH 5, 6 and 7 are minor selftest/ improvements
* PATCH 8 enables (non-arena) load-acquire/store-release selftests forriscv64Please refer to individual patches for details. Thanks!
[1] https://lore.kernel.org/all/[email protected]/
Andrea Parri (2):
bpf, riscv64: Introduce emit_load_*() and emit_store_*() bpf, riscv64: Support load-acquire and store-release instructionsPeilin Ye (6):
bpf/verifier: HandleBPF_LOAD_ACQinstructions ininsn_def_regno()bpf, riscv64: Skip redundant zext instruction after load-acquire selftests/bpf: UseCAN_USE_LOAD_ACQ_STORE_RELwhen appropriate selftests/bpf: Avoid passing out-of-range values to__retval()selftests/bpf: Verify zero-extension behavior in load-acquire tests selftests/bpf: Enable non-arena load-acquire/store-release selftests
for riscv64
Quoted 10 lines
arch/riscv/net/bpf_jit.h | 15 + arch/riscv/net/bpf_jit_comp64.c | 334 ++++++++++++------ arch/riscv/net/bpf_jit_core.c | 3 +- kernel/bpf/verifier.c | 11 +- tools/testing/selftests/bpf/progs/bpf_misc.h | 5 +- .../bpf/progs/verifier_load_acquire.c | 48 ++- .../selftests/bpf/progs/verifier_precision.c | 5 +- .../bpf/progs/verifier_store_release.c | 39 +- 8 files changed, 314 insertions(+), 146 deletions(-)
Hi Peilin, good to see the implementation of load-acquire and store-release instructions on RV64! But I'm about to start my vacation, so I'll test it once I'm back.
Hi Lehui,
Quoted 24 lines · Pu Lehui
Quoted 13 lines · Peilin Ye
Andrea Parri (2):
bpf, riscv64: Introduce emit_load_*() and emit_store_*() bpf, riscv64: Support load-acquire and store-release instructionsPeilin Ye (6):
bpf/verifier: HandleBPF_LOAD_ACQinstructions ininsn_def_regno()bpf, riscv64: Skip redundant zext instruction after load-acquire selftests/bpf: UseCAN_USE_LOAD_ACQ_STORE_RELwhen appropriate selftests/bpf: Avoid passing out-of-range values to__retval()selftests/bpf: Verify zero-extension behavior in load-acquire tests selftests/bpf: Enable non-arena load-acquire/store-release selftests
for riscv64Quoted 9 lines
arch/riscv/net/bpf_jit.h | 15 + arch/riscv/net/bpf_jit_comp64.c | 334 ++++++++++++------ arch/riscv/net/bpf_jit_core.c | 3 +- kernel/bpf/verifier.c | 11 +- tools/testing/selftests/bpf/progs/bpf_misc.h | 5 +- .../bpf/progs/verifier_load_acquire.c | 48 ++- .../selftests/bpf/progs/verifier_precision.c | 5 +- .../bpf/progs/verifier_store_release.c | 39 +- 8 files changed, 314 insertions(+), 146 deletions(-)
Quoted 3 lines
Hi Peilin, good to see the implementation of load-acquire and store-release instructions on RV64! But I'm about to start my vacation, so I'll test it once I'm back.
Thanks for the reviews, and have a great vacation!
Peilin Ye
Andrea/Peilin!
Quoted 7 lines · Peilin Ye
Hi all!
Patchset [1] introduced BPF load-acquire (
BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64 JIT compiler support. As a follow-up, this patchset supports load-acquire and store-release instructions for the riscv64 JIT compiler, and introduces some related selftests/ changes.
Thanks a bunch for working on this!
For the series:
Hi Björn,
Quoted 7 lines · Björn Töpel
Quoted 5 lines · Peilin Ye
Patchset [1] introduced BPF load-acquire (
BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions, and added x86-64 and arm64 JIT compiler support. As a follow-up, this patchset supports load-acquire and store-release instructions for the riscv64 JIT compiler, and introduces some related selftests/ changes.
Quoted 5 lines
Thanks a bunch for working on this!
For the series:
Acked-by Björn Töpel <[email protected]>
Thanks for testing this!
Peilin Ye
Pu Lehui
maintainerRe: [PATCH bpf-next 1/8] bpf/verifier: Handle BPF_LOAD_ACQ instructions in insn_def_regno()
In reply to Peilin YeQuoted 24 lines · Peilin Ye
In preparation for supporting BPF load-acquire and store-release instructions for architectures where
bpf_jit_needs_zext()returns true (e.g. riscv64), makeinsn_def_regno()handle load-acquires properly.Signed-off-by Peilin Ye <[email protected]>--- kernel/bpf/verifier.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)kernel/bpf/verifier.c +2
-3@@ -3649,13 +3649,16 @@ static int insn_def_regno(const struct bpf_insn *insn)case BPF_ST :return -1 ;case BPF_STX : -if ((BPF_MODE (insn->code) ==BPF_ATOMIC || -BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) && - (insn->imm &BPF_FETCH )) { +if (BPF_MODE (insn->code) ==BPF_ATOMIC || +BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) {if (insn->imm ==BPF_CMPXCHG )return BPF_REG_0 ;
kernel/bpf/verifier.c static int insn_def_regno(const struct bpf_insn *insn) · line 3664
Quoted 10 lines
kernel/bpf/verifier.c +5
-1-else +else if (insn->imm ==BPF_LOAD_ACQ ) +return insn->dst_reg; +else if (insn->imm &BPF_FETCH )return insn->src_reg; +else +return -1 ; }else {return -1 ; }
How about simplify like this:
```
static int insn_def_regno(const struct bpf_insn *insn)
{
switch (BPF_CLASS(insn->code)) {
case BPF_JMP:
case BPF_JMP32:
case BPF_ST:
return -1;
case BPF_STX:
if (BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
if (insn->imm == BPF_CMPXCHG)
return BPF_REG_0;
else if (insn->imm == BPF_LOAD_ACQ)
return insn->dst_reg;
else if (insn->imm & BPF_FETCH)
return insn->src_reg;
}
return -1;
default:
return insn->dst_reg;
}
}
```Pu Lehui
maintainerQuoted 195 lines · Peilin Ye
From: Andrea Parri <[email protected]>
Support BPF load-acquire (
BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions in the riscv64 JIT compiler. For example, consider the following 64-bit load-acquire (assuming little-endian):db 10 00 00 00 01 00 00 r1 = load_acquire((u64 *)(r1 + 0x0)) 95 00 00 00 00 00 00 00 exit opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX imm (0x00000100): BPF_LOAD_ACQThe JIT compiler will emit an LD instruction followed by a FENCE R,RW instruction for the above, e.g.:
ld x7,0(x6) fence r,rwSimilarly, consider the following 16-bit store-release:
cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) 95 00 00 00 00 00 00 00 exit opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX imm (0x00000110): BPF_STORE_RELA FENCE RW,W instruction followed by an SH instruction will be emitted, e.g.:
fence rw,w sh x2,0(x4)8-bit and 16-bit load-acquires are zero-extending (cf., LBU, LHU). The verifier always rejects misaligned load-acquires/store-releases (even if
BPF_F_ANY_ALIGNMENTis set), so the emitted load and store instructions are guaranteed to be single-copy atomic.Introduce primitives to emit the relevant (and the most common/used in the kernel) fences, i.e. fences with R -> RW, RW -> W and RW -> RW.
Rename
emit_atomic()toemit_atomic_rmw()to make it clear that it only handles RMW atomics, and replace its is64 parameter to allow to perform the required checks on the opsize (BPF_SIZE(code)).Tested-by Peilin Ye <[email protected]>Signed-off-by Andrea Parri <[email protected]>[[email protected]: whitespace changes; cosmetic changes to commit
title and message]Signed-off-by Peilin Ye <[email protected]>--- arch/riscv/net/bpf_jit.h | 15 +++++++ arch/riscv/net/bpf_jit_comp64.c | 77 ++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-)arch/riscv/net/bpf_jit.h +15
-0@@ -608,6 +608,21 @@ static inline u32 rv_fence(u8 pred, u8 succ)return rv_i_insn (imm11_0,0 ,0 ,0 ,0xf ); } +static inline void emit_fence_r_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x2 ,0x3 ), ctx); +} + +static inline void emit_fence_rw_w (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x1 ), ctx); +} + +static inline void emit_fence_rw_rw (struct rv_jit_context *ctx) +{ +emit (rv_fence (0x3 ,0x3 ), ctx); +} +static inline u32 rv_nop (void ) {return rv_i_insn (0 ,0 ,0 ,0 ,0x13 );arch/riscv/net/bpf_jit_comp64.c +69
-6@@ -607,11 +607,65 @@ static void emit_store_64(u8 rd, s32 off, u8 rs, struct rv_jit_context *ctx)emit_sd (RV_REG_T1 ,0 , rs, ctx); } -static void emit_atomic (u8 rd,u8 rs,s16 off,s32 imm,bool is64, -struct rv_jit_context *ctx) +static int emit_atomic_ld_st (u8 rd,u8 rs,s16 off,s32 imm,u8 code,struct rv_jit_context *ctx) +{ +switch (imm) { +/* dst_reg = load_acquire(src_reg + off16) */ +case BPF_LOAD_ACQ : +switch (BPF_SIZE (code)) { +case BPF_B : +emit_load_8 (false, rd, off, rs, ctx); +break ; +case BPF_H : +emit_load_16 (false, rd, off, rs, ctx); +break ; +case BPF_W : +emit_load_32 (false, rd, off, rs, ctx); +break ; +case BPF_DW : +emit_load_64 (false, rd, off, rs, ctx); +break ; + } +emit_fence_r_rw (ctx); +break ; +/* store_release(dst_reg + off16, src_reg) */ +case BPF_STORE_REL : +emit_fence_rw_w (ctx); +switch (BPF_SIZE (code)) { +case BPF_B : +emit_store_8 (rd, off, rs, ctx); +break ; +case BPF_H : +emit_store_16 (rd, off, rs, ctx); +break ; +case BPF_W : +emit_store_32 (rd, off, rs, ctx); +break ; +case BPF_DW : +emit_store_64 (rd, off, rs, ctx); +break ; + } +break ; +default : +pr_err_once ("bpf-jit: invalid atomic load/store opcode %02x\n" , imm); +return -EINVAL ; + } + +return 0 ; +} + +static int emit_atomic_rmw (u8 rd,u8 rs,s16 off,s32 imm,u8 code, +struct rv_jit_context *ctx) {u8 r0;int jmp_offset; +bool is64; + +if (BPF_SIZE (code) !=BPF_W &&BPF_SIZE (code) !=BPF_DW ) { +pr_err_once ("bpf-jit: 1- and 2-byte RMW atomics are not supported\n" ); +return -EINVAL ; + } + is64 =BPF_SIZE (code) ==BPF_DW ;if (off) {if (is_12b_int (off)) {@@ -688,9 +742,14 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); jmp_offset =ninsns_rvoff (-6 );emit (rv_bne (RV_REG_T3 ,0 , jmp_offset >>1 ), ctx); -emit (rv_fence (0x3 ,0x3 ), ctx); +emit_fence_rw_rw (ctx);break ; +default : +pr_err_once ("bpf-jit: invalid atomic RMW opcode %02x\n" , imm); +return -EINVAL ; } + +return 0 ; }#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)@@ -1259,7 +1318,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, {bool is64 =BPF_CLASS (insn->code) ==BPF_ALU64 ||BPF_CLASS (insn->code) ==BPF_JMP ; -int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; +int s, e, rvoff, ret =0 , i = insn - ctx->prog->insnsi;struct bpf_prog_aux *aux = ctx->prog->aux;u8 rd =-1 , rs =-1 , code = insn->code;s16 off = insn->off;@@ -1962,10 +2021,14 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_STX |BPF_MEM |BPF_DW :emit_store_64 (rd, off, rs, ctx);break ; +case BPF_STX |BPF_ATOMIC |BPF_B : +case BPF_STX |BPF_ATOMIC |BPF_H :case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW : -emit_atomic (rd, rs, off, imm, -BPF_SIZE (code) ==BPF_DW , ctx); +if (bpf_atomic_is_load_store (insn)) + ret =emit_atomic_ld_st (rd, rs, off, imm, code, ctx); +else + ret =emit_atomic_rmw (rd, rs, off, imm, code, ctx);break ;case BPF_STX |BPF_PROBE_MEM32 |BPF_B :
arch/riscv/net/bpf_jit_comp64.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 2116
Quoted 5 lines
arch/riscv/net/bpf_jit_comp64.c +1
-1@@ -2050,7 +2113,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,return -EINVAL ; } -return 0 ; +return ret;
ret may be a value greater than zero, which will potentially cause build_body to skip the next instruction. Let's return 0 here, and return ret if the above fails.
Quoted 3 lines
arch/riscv/net/bpf_jit_comp64.c +0
-0}void bpf_jit_build_prologue (struct rv_jit_context *ctx,bool is_subprog)
Re: [PATCH bpf-next 5/8] selftests/bpf: Use CAN_USE_LOAD_ACQ_STORE_REL when appropriate
In reply to Peilin YeQuoted 65 lines · Peilin Ye
Instead of open-coding the conditions, use
'#ifdefCAN_USE_LOAD_ACQ_STORE_REL' to guard the following tests:verifier_precision/bpf_load_acquire
verifier_precision/bpf_store_release
verifier_store_release/*Note that, for the first two tests in
verifier_precision.c, switching to '#ifdefCAN_USE_LOAD_ACQ_STORE_REL' means also checking if '__clang_major__>= 18', which has already been guaranteed by the outer '#if' check.Signed-off-by Peilin Ye <[email protected]>--- tools/testing/selftests/bpf/progs/verifier_precision.c | 5 ++--- tools/testing/selftests/bpf/progs/verifier_store_release.c | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-)tools/testing/selftests/bpf/progs/verifier_precision.c +2
-3@@ -91,8 +91,7 @@ __naked int bpf_end_bswap(void) ::: __clobber_all); } -#if defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#ifdef CAN_USE_LOAD_ACQ_STORE_REL SEC("?raw_tp") __success __log_level(2)@@ -138,7 +137,7 @@ __naked int bpf_store_release(void) : __clobber_all); } -#endif /* load-acquire, store-release */ +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ #endif /* v4 instruction */ SEC ("?raw_tp" )tools/testing/selftests/bpf/progs/verifier_store_release.c +2
-3@@ -6,8 +6,7 @@#include "../../../include/linux/filter.h" #include "bpf_misc.h" -#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#ifdef CAN_USE_LOAD_ACQ_STORE_REL SEC("socket")__description ("store-release, 8-bit" )@@ -271,7 +270,7 @@ __naked void store_release_with_invalid_reg(void) : __clobber_all); } -#else +#else /* CAN_USE_LOAD_ACQ_STORE_REL */ SEC("socket")__description ("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test" )
tools/testing/selftests/bpf/progs/verifier_store_release.c int dummy_test(void) · line 285
Quoted 7 lines
tools/testing/selftests/bpf/progs/verifier_store_release.c +1
-1@@ -281,6 +280,6 @@ int dummy_test(void)return 0 ; } -#endif +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ char _license[] SEC("license") = "GPL";
Re: [PATCH bpf-next 6/8] selftests/bpf: Avoid passing out-of-range values to __retval()
In reply to Peilin YeQuoted 193 lines · Peilin Ye
Currently, we pass
0x1234567890abcdefto__retval()for the following two tests:verifier_load_acquire/load_acquire_64
verifier_store_release/store_release_64However, the upper 32 bits of that value are being ignored, since
__retval()expects an int. Actually, the tests would still pass even if I change '__retval(0x1234567890abcdef)' to e.g. '__retval(0x90abcdef)'.Restructure the tests a bit to test the entire 64-bit values properly. Do the same to their 8-, 16- and 32-bit variants as well to keep the style consistent.
Fixes ff3afe5da998 ("selftests/bpf: Add selftests for load-acquire and store-release instructions")Signed-off-by Peilin Ye <[email protected]>--- .../bpf/progs/verifier_load_acquire.c | 40 +++++++++++++------ .../bpf/progs/verifier_store_release.c | 32 +++++++++++---- 2 files changed, 52 insertions(+), 20 deletions(-)tools/testing/selftests/bpf/progs/verifier_load_acquire.c +28
-12@@ -10,65 +10,81 @@ SEC("socket") __description("load-acquire, 8-bit") -__success __success_unpriv __retval(0x12) +__success __success_unpriv __retval(0) __naked void load_acquire_8(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x12;" "*(u8 *)(r10 - 1) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1)) + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1)) : __clobber_all); } SEC("socket") __description("load-acquire, 16-bit") -__success __success_unpriv __retval(0x1234) +__success __success_unpriv __retval(0) __naked void load_acquire_16(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x1234;" "*(u16 *)(r10 - 2) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2)) + BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2)) : __clobber_all); } SEC("socket") __description("load-acquire, 32-bit") -__success __success_unpriv __retval(0x12345678) +__success __success_unpriv __retval(0) __naked void load_acquire_32(void) { asmvolatile ( +"r0 = 0;" "w1 = 0x12345678;" "*(u32 *)(r10 - 4) = w1;" -".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4)); +".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4)) + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4)) : __clobber_all); } SEC("socket") __description("load-acquire, 64-bit") -__success __success_unpriv __retval(0x1234567890abcdef) +__success __success_unpriv __retval(0) __naked void load_acquire_64(void) { asmvolatile ( +"r0 = 0;" "r1 = 0x1234567890abcdef ll;" "*(u64 *)(r10 - 8) = r1;" -".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8)); +".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)) + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)) : __clobber_all); }tools/testing/selftests/bpf/progs/verifier_store_release.c +20
-7@@ -10,13 +10,17 @@ SEC("socket") __description("store-release, 8-bit") -__success __success_unpriv __retval(0x12) +__success __success_unpriv __retval(0) __naked void store_release_8(void) { asm volatile ( + "r0 = 0;" "w1 = 0x12;" ".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1); - "w0 = *(u8 *)(r10 - 1);" + "w2 = *(u8 *)(r10 - 1);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,@@ -26,13 +30,17 @@ __naked void store_release_8(void) SEC("socket") __description("store-release, 16-bit") -__success __success_unpriv __retval(0x1234) +__success __success_unpriv __retval(0) __naked void store_release_16(void) { asm volatile ( + "r0 = 0;" "w1 = 0x1234;" ".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1); - "w0 = *(u16 *)(r10 - 2);" + "w2 = *(u16 *)(r10 - 2);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,@@ -42,13 +50,17 @@ __naked void store_release_16(void) SEC("socket") __description("store-release, 32-bit") -__success __success_unpriv __retval(0x12345678) +__success __success_unpriv __retval(0) __naked void store_release_32(void) { asm volatile ( + "r0 = 0;" "w1 = 0x12345678;" ".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1); - "w0 = *(u32 *)(r10 - 4);" + "w2 = *(u32 *)(r10 - 4);" + "if r2 == r1 goto 1f;" + "r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,@@ -58,13 +70,17 @@ __naked void store_release_32(void) SEC("socket") __description("store-release, 64-bit") -__success __success_unpriv __retval(0x1234567890abcdef) +__success __success_unpriv __retval(0) __naked void store_release_64(void) { asmvolatile ( + "r0 = 0;"
tools/testing/selftests/bpf/progs/verifier_store_release.c __naked void store_release_32(void) · line 86
Quoted 10 lines
tools/testing/selftests/bpf/progs/verifier_store_release.c +4
-1"r1 = 0x1234567890abcdef ll;" ".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1); -"r0 = *(u64 *)(r10 - 8);" +"r2 = *(u64 *)(r10 - 8);" +"if r2 == r1 goto 1f;" +"r0 = 1;" +"1:" "exit;" : : __imm_insn(store_release_insn,
Re: [PATCH bpf-next 7/8] selftests/bpf: Verify zero-extension behavior in load-acquire tests
In reply to Peilin YeQuoted 41 lines · Peilin Ye
Verify that 8-, 16- and 32-bit load-acquires are zero-extending by using immediate values with their highest bit set. Do the same for the 64-bit variant to keep the style consistent.
Signed-off-by Peilin Ye <[email protected]>--- tools/testing/selftests/bpf/progs/verifier_load_acquire.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)tools/testing/selftests/bpf/progs/verifier_load_acquire.c +3
-3@@ -15,7 +15,7 @@ __naked void load_acquire_8(void) { asmvolatile ("r0 = 0;" -"w1 = 0x12;" +"w1 = 0xfe;" "*(u8 *)(r10 - 1) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1)); "if r2 == r1 goto 1f;"@@ -35,7 +35,7 @@ __naked void load_acquire_16(void) { asmvolatile ("r0 = 0;" -"w1 = 0x1234;" +"w1 = 0xfedc;" "*(u16 *)(r10 - 2) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2)); "if r2 == r1 goto 1f;"@@ -55,7 +55,7 @@ __naked void load_acquire_32(void) { asmvolatile ("r0 = 0;" -"w1 = 0x12345678;" +"w1 = 0xfedcba09;" "*(u32 *)(r10 - 4) = w1;" ".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4)); "if r2 == r1 goto 1f;"
tools/testing/selftests/bpf/progs/verifier_load_acquire.c __naked void load_acquire_64(void) · line 81
Quoted 8 lines
tools/testing/selftests/bpf/progs/verifier_load_acquire.c +1
-1@@ -75,7 +75,7 @@ __naked void load_acquire_64(void) { asmvolatile ("r0 = 0;" -"r1 = 0x1234567890abcdef ll;" +"r1 = 0xfedcba0987654321 ll;" "*(u64 *)(r10 - 8) = r1;" ".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8)); "if r2 == r1 goto 1f;"
Re: [PATCH bpf-next 8/8] selftests/bpf: Enable non-arena load-acquire/store-release selftests for riscv64
In reply to Peilin YeQuoted 15 lines · Peilin Ye
For riscv64, enable all BPF_{LOAD_ACQ,STORE_REL} selftests except the arena_atomics/* ones (not guarded behind
CAN_USE_LOAD_ACQ_STORE_REL), since arena access is not yet supported.Signed-off-by Peilin Ye <[email protected]>--- tools/testing/selftests/bpf/progs/bpf_misc.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)tools/testing/selftests/bpf/progs/bpf_misc.h +0
-0@@ -225,8 +225,9 @@#define CAN_USE_BPF_ST
tools/testing/selftests/bpf/progs/bpf_misc.h line 233
Quoted 10 lines
tools/testing/selftests/bpf/progs/bpf_misc.h +3
-2#endif -#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) +#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 ))#define CAN_USE_LOAD_ACQ_STORE_REL #endif
Re: [PATCH bpf-next 1/8] bpf/verifier: Handle BPF_LOAD_ACQ instructions in insn_def_regno()
In reply to Pu LehuiHi Lehui,
Quoted 42 lines · Pu Lehui
Quoted 13 lines · Peilin Ye
kernel/bpf/verifier.c +2
-3@@ -3649,13 +3649,16 @@ static int insn_def_regno(const struct bpf_insn *insn)case BPF_ST :return -1 ;case BPF_STX : -if ((BPF_MODE (insn->code) ==BPF_ATOMIC || -BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) && - (insn->imm &BPF_FETCH )) { +if (BPF_MODE (insn->code) ==BPF_ATOMIC || +BPF_MODE (insn->code) ==BPF_PROBE_ATOMIC ) {if (insn->imm ==BPF_CMPXCHG )return BPF_REG_0 ;kernel/bpf/verifier.c static int insn_def_regno(const struct bpf_insn *insn) · line 3664
Quoted 10 lines
kernel/bpf/verifier.c +5
-1-else +else if (insn->imm ==BPF_LOAD_ACQ ) +return insn->dst_reg; +else if (insn->imm &BPF_FETCH )return insn->src_reg; +else +return -1 ; }else {return -1 ; }How about simplify like this:
``` static int insn_def_regno(const struct bpf_insn *insn) { switch (BPF_CLASS(insn->code)) { case BPF_JMP: case BPF_JMP32: case BPF_ST: return -1; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC || BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { if (insn->imm == BPF_CMPXCHG) return BPF_REG_0; else if (insn->imm == BPF_LOAD_ACQ) return insn->dst_reg; else if (insn->imm & BPF_FETCH)
Quoted 8 lines
return insn->src_reg;
}
return -1;
default:
return insn->dst_reg;
}
}
```I see, I'll change it in v2. Thanks for the suggestion!
Peilin Ye
Quoted 27 lines · Pu Lehui
Quoted 26 lines · Peilin Ye
arch/riscv/net/bpf_jit_comp64.c +7
-3@@ -1259,7 +1318,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, {bool is64 =BPF_CLASS (insn->code) ==BPF_ALU64 ||BPF_CLASS (insn->code) ==BPF_JMP ; -int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; +int s, e, rvoff, ret =0 , i = insn - ctx->prog->insnsi;struct bpf_prog_aux *aux = ctx->prog->aux;u8 rd =-1 , rs =-1 , code = insn->code;s16 off = insn->off;@@ -1962,10 +2021,14 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,case BPF_STX |BPF_MEM |BPF_DW :emit_store_64 (rd, off, rs, ctx);break ; +case BPF_STX |BPF_ATOMIC |BPF_B : +case BPF_STX |BPF_ATOMIC |BPF_H :case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW : -emit_atomic (rd, rs, off, imm, -BPF_SIZE (code) ==BPF_DW , ctx); +if (bpf_atomic_is_load_store (insn)) + ret =emit_atomic_ld_st (rd, rs, off, imm, code, ctx); +else + ret =emit_atomic_rmw (rd, rs, off, imm, code, ctx);break ;case BPF_STX |BPF_PROBE_MEM32 |BPF_B :@@ -2050,7 +2113,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
Quoted 8 lines
arch/riscv/net/bpf_jit_comp64.c int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, · line 2116
Quoted 4 lines
arch/riscv/net/bpf_jit_comp64.c +1
-1return -EINVAL ; } -return 0 ; +return ret;
retmay be a value greater than zero, which will potentially causebuild_bodyto skip the next instruction. Let'sreturn 0here, andreturn retif the above fails.
Sure, I'll change this in v2.
Thanks,
Peilin Ye