bpf: bug fixes
A message that is not in the archive (sent off-list or not yet received): [email protected]
v7 cover bpf: bug fixes
BPF related fixes mostly related to JIT.
The arm64 BPF JIT did not implement BPF_LD | BPF_ABS or BPF_LD | BPF_IND, so cBPF filters converted by rte_bpf_convert() could not be JIT compiled and silently fell back to the interpreter on arm64.
The first patch fixes a latent bug in emit_return_zero_if_src_zero():
the offset of the branch to the epilogue was held in an unsigned. A backward branch wrapped around. Existing JIT tests were never being run on ARM.
The next two patches make the bpf tests assert that,
on an architecture with a JITbackend, code is actually generated, so a missing or failed JIT is reported rather than skipped.
The final patch adds the ABS/IND opcodes, mirroring the x86 JIT with a fast path for data in the first
mbuf segment and a __rte_pktmbuf_read() slow path for the rest.
Christophe Fontaine (1):
bpf/arm64: fix offset type to allow a negative jump
Stephen Hemminger (9):
bpf/x86: fix JIT encoding of fixed-width immediates test/bpf: add JSET test with small immediate
bpf: mask shift count in interpreter per RFC 9669 bpf/arm64: mask shift count per RFC 9669 test/bpf: add test for large shift bpf/arm64: add BPF_ABS/BPF_IND packet load support test/bpf: check that JIT was generated test/bpf: check that bpf_convert can be JIT'd
bpf: fix uninitialized warning
app/test/test_bpf.c | 320 +++++++++++++++++++++++++++++++--------- lib/bpf/bpf_exec.c | 31 ++-- lib/bpf/bpf_jit_arm64.c | 185 ++++++++++++++++++++++- lib/bpf/bpf_jit_x86.c | 6 +- lib/bpf/bpf_validate.c | 2 +- lib/bpf/meson.build | 2 + 6 files changed, 457 insertions(+), 89 deletions(-)
Signature · 1 line
2.53.0
PATCH v7 1/10 bpf/x86: fix JIT encoding of fixed-width immediates
Several places in the x86 JIT size an immediate with imm_size(), which returns 1 or 4 bytes depending on the value. That is wrong for opcodes whose immediate width is fixed by the encoding, and it breaks in both directions.
TEST (0xF7 /0, used for BPF_JSET) has no imm8 form; the immediate is always 32 bits. For a small mask such as BPF_JSET | BPF_K #0x1, imm_size() returns 1, so the JIT emits a 1-byte immediate. The CPU still consumes 4, swallowing 3 bytes of the following Jcc. The instruction stream desyncs and the program crashes.
ROR and the shifts (0xC1 group) have the opposite problem: their immediate is always imm8. For a count >= 128, imm_size() returns 4 and the JIT emits 3 stray bytes, again desyncing the stream.
Size each immediate by its encoding: 32 bits for TEST, 8 bits for ROR and the shifts.
Bugzilla ID: 1959
--- lib/bpf/bpf_jit_x86.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/bpf/bpf_jit_x86.c +3 -3
@@ -300,7 +300,7 @@ emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)emit_rex (st,BPF_ALU ,0 , dreg);emit_bytes (st, &ops,sizeof (ops));emit_modregrm (st,MOD_DIRECT , mods, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (uint8_t )); }/*
@@ -441,7 +441,7 @@ emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) {emit_shift (st, op, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (uint8_t )); }/*
@@ -921,7 +921,7 @@ emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)emit_rex (st, op,0 , dreg);emit_bytes (st, &ops,sizeof (ops));emit_modregrm (st,MOD_DIRECT , mods, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (int32_t )); } static void
Signature · 1 line
2.53.0
PATCH v7 2/10 test/bpf: add JSET test with small immediate
In reply to Stephen HemmingerThe existing jump test only used a 32-bit JSET mask,
so the broken imm8 encoding of TEST in the x86 JIT was never exercised. Add a case with a byte-sized mask;run_test() runs it through the interpreter and the JIT.
--- app/test/test_bpf.c | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+)
app/test/test_bpf.c +82 -0
@@ -3229,7 +3229,89 @@ test_int64min_umod_uint64max_check(uint64_t rc, const void *arg) }/* all bpf test cases */ +/* +* JSET with a byte-sized mask: exercises the imm8 path of the TEST +* encoding in the x86 JIT (a 32-bit mask takes a different path). +*/ +static const struct ebpf_insn test_jset1_prog[] = { + { + .code = (BPF_ALU |EBPF_MOV |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_1 , + .off = offsetof(struct dummy_offset , u8), + }, +/* bit 0 is set in the input: branch is taken */ + { + .code = (BPF_JMP |BPF_JSET |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0x1 , + .off =1 , + }, + { + .code = (BPF_JMP |BPF_JA ), + .off =1 , + }, + { + .code = (EBPF_ALU64 |BPF_OR |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0x1 , + }, +/* bit 1 is clear in the input: branch is not taken */ + { + .code = (BPF_JMP |BPF_JSET |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0x2 , + .off =1 , + }, + { + .code = (BPF_JMP |BPF_JA ), + .off =1 , + }, + { + .code = (EBPF_ALU64 |BPF_OR |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0x2 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; + +static void +test_jset1_prepare (void *arg) +{ +struct dummy_offset *df = arg; + +memset (df,0 ,sizeof (*df)); + df->u8 =0x1 ;/* bit 0 set, bit 1 clear */ +} + +static int +test_jset1_check (uint64_t rc,const void *arg) +{ +return cmp_res (__func__,0x1 , rc, arg, arg,0 ); +} +static const struct bpf_test tests[] = { + { + .name ="test_jset1" , + .arg_sz =sizeof (struct dummy_offset ), + .prm = { + .ins = test_jset1_prog, + .nb_ins =RTE_DIM (test_jset1_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR , + .size =sizeof (struct dummy_offset ), + }, + }, + .prepare = test_jset1_prepare, + .check_result = test_jset1_check, + }, { .name ="test_store1" , .arg_sz =sizeof (struct dummy_offset ),
Signature · 1 line
2.53.0
PATCH v7 3/10 bpf: mask shift count in interpreter per RFC 9669
In reply to Stephen HemmingerThe interpreter shifted by the raw immediate or register value, which is undefined behavior in C when the count is >= the operand width and trips UBSan. RFC 9669 masks shift counts (0x3f for 64-bit, 0x1f for 32-bit); mask the count in the LSH/RSH/ARSH cases.
--- lib/bpf/bpf_exec.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-)
lib/bpf/bpf_exec.c +21 -10
@@ -4,6 +4,7 @@#include <stdio.h> #include <stdint.h> +#include <limits.h> #include <eal_export.h> #include <rte_common.h>
@@ -43,6 +44,16 @@ ((reg)[(ins)->dst_reg] = \ (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) +#define BPF_OP_SHIFT_IMM (reg, ins, op, type) \ + ((reg)[(ins)->dst_reg] = \ + (type)(reg)[(ins)->dst_reg] op \ + ((ins)->imm & (sizeof(type) * CHAR_BIT - 1))) + +#define BPF_OP_SHIFT_REG (reg, ins, op, type) \ + ((reg)[(ins)->dst_reg] = \ + (type)(reg)[(ins)->dst_reg] op \ + ((reg)[(ins)->src_reg] & (sizeof(type) * CHAR_BIT - 1))) +#define BPF_DIV_ZERO_CHECK (bpf, reg, ins, type) do { \ if ((type)(reg)[(ins)->src_reg] == 0) { \ RTE_BPF_LOG_LINE(ERR, \
@@ -183,10 +194,10 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_IMM(reg, ins, |, uint32_t);break ;case (BPF_ALU |BPF_LSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, <<, uint32_t); + BPF_OP_SHIFT_IMM(reg, ins, <<, uint32_t);break ;case (BPF_ALU |BPF_RSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, uint32_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, uint32_t);break ;case (BPF_ALU |BPF_XOR |BPF_K ): BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
@@ -217,10 +228,10 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_REG(reg, ins, |, uint32_t);break ;case (BPF_ALU |BPF_LSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, <<, uint32_t); + BPF_OP_SHIFT_REG(reg, ins, <<, uint32_t);break ;case (BPF_ALU |BPF_RSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, uint32_t); + BPF_OP_SHIFT_REG(reg, ins, >>, uint32_t);break ;case (BPF_ALU |BPF_XOR |BPF_X ): BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
@@ -262,13 +273,13 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_IMM(reg, ins, |, uint64_t);break ;case (EBPF_ALU64 |BPF_LSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, <<, uint64_t); + BPF_OP_SHIFT_IMM(reg, ins, <<, uint64_t);break ;case (EBPF_ALU64 |BPF_RSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, uint64_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, uint64_t);break ;case (EBPF_ALU64 |EBPF_ARSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, int64_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, int64_t);break ;case (EBPF_ALU64 |BPF_XOR |BPF_K ): BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
@@ -299,13 +310,13 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_REG(reg, ins, |, uint64_t);break ;case (EBPF_ALU64 |BPF_LSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, <<, uint64_t); + BPF_OP_SHIFT_REG(reg, ins, <<, uint64_t);break ;case (EBPF_ALU64 |BPF_RSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, uint64_t); + BPF_OP_SHIFT_REG(reg, ins, >>, uint64_t);break ;case (EBPF_ALU64 |EBPF_ARSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, int64_t); + BPF_OP_SHIFT_REG(reg, ins, >>, int64_t);break ;case (EBPF_ALU64 |BPF_XOR |BPF_X ): BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
Signature · 1 line
2.53.0
PATCH v7 4/10 bpf/arm64: mask shift count per RFC 9669
In reply to Stephen HemmingerThe ARM JIT was not masking the shift count as required by RFC 9669 (0x3f for 64-bit, 0x1f for 32-bit), so large immediate shift counts overflowed the UBFM/SBFM encoding and failed the JIT. Mask the immediate in emit_lsl/emit_lsr/emit_asr.
--- lib/bpf/bpf_jit_arm64.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
lib/bpf/bpf_jit_arm64.c +10 -2
@@ -545,12 +545,14 @@ emit_bitfield(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,emit_insn (ctx, insn,check_reg (rd) ||check_reg (rn) ||check_immr_imms (is64, immr, imms)); } +static void emit_lsl (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) {const unsigned int width = is64 ?64 :32 ;uint8_t imms, immr; + imm &= width -1 ; immr = (width - imm) & (width -1 ); imms = width -1 - imm;
@@ -560,13 +562,19 @@ emit_lsl(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)static void emit_lsr (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) { -emit_bitfield (ctx, is64, rd, rd, imm, is64 ?63 :31 ,A64_UBFM ); +const unsigned int width = is64 ?64 :32 ; + + imm &= width -1 ; +emit_bitfield (ctx, is64, rd, rd, imm, width -1 ,A64_UBFM ); }static void emit_asr (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) { -emit_bitfield (ctx, is64, rd, rd, imm, is64 ?63 :31 ,A64_SBFM ); +const unsigned int width = is64 ?64 :32 ; + + imm &= width -1 ; +emit_bitfield (ctx, is64, rd, rd, imm, width -1 ,A64_SBFM ); }#define A64_AND 0
Signature · 1 line
2.53.0
PATCH v7 5/10 test/bpf: add test for large shift
In reply to Stephen HemmingerThere were multiple bugs with immediate values in shift instructions. The code was not masking as required by RFC.
Add new tests that cover these instructions.
--- app/test/test_bpf.c | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
app/test/test_bpf.c +59 -0
@@ -2012,6 +2012,51 @@ test_div1_check(uint64_t rc, const void *arg)return cmp_res (__func__,0 , rc, dve.out, dvt->out,sizeof (dve.out)); } +/* +* Shift counts are masked to the operand width (RFC 9669: 0x3f for 64-bit, +* 0x1f for 32-bit). Counts >= 128 also exercise the x86 imm_size() path that +* used to desync the stream, and the arm64 UBFM/SBFM immediate encoding. +*/ +static const struct ebpf_insn test_shift_big_imm_prog[] = { + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =1 + }, + { + .code = (EBPF_ALU64 |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =191 + }, + { + .code = (EBPF_ALU64 |EBPF_ARSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =200 + }, + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =130 + }, + { + .code = (BPF_JMP |EBPF_EXIT ) + }, +}; + +static void +test_shift_big_imm_prepare (void *arg) +{ +memset (arg,0 ,sizeof (struct dummy_offset )); +} + +static int +test_shift_big_imm_check (uint64_t rc,const void *arg) +{ +uint64_t expect =0x3FE0000000000000ULL ; + +return cmp_res (__func__, expect, rc, arg, arg,0 ); +} +/* call test-cases */ static const struct ebpf_insn test_call1_prog[] = {
@@ -3480,6 +3525,20 @@ static const struct bpf_test tests[] = { .prepare = test_mul1_prepare, .check_result = test_div1_check, }, + { + .name = "test_shift_big_imm", + .arg_sz = sizeof(struct dummy_offset), + .prm = { + .ins = test_shift_big_imm_prog, + .nb_ins = RTE_DIM(test_shift_big_imm_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR, + .size =sizeof (struct dummy_offset ), + }, + }, + .prepare = test_shift_big_imm_prepare, + .check_result = test_shift_big_imm_check, + }, { .name ="test_call1" , .arg_sz = sizeof(struct dummy_offset),
Signature · 1 line
2.53.0
PATCH v7 6/10 bpf/arm64: fix offset type to allow a negative jump
In reply to Stephen HemmingerFrom: Christophe Fontaine <[email protected]>
The DPDK BPF JIT standalone test test_ld_mbuf1 fails on arm64. It does:
r6 = r1 // mbuf r0 = *(u8 *)pkt[0] // BPF_ABS if ((r0 & 0xf0) == 0x40) goto parse r0 = 0 exit // epilogue E0 parse: r0 = *(u8 *)pkt[r0 + 3] // BPF_IND ... exit
emit_return_zero_if_src_zero() returns 0 by branching to a function epilogue. The target may be a previous epilogue so branch might be backwards; therefore the offset needs to be negative.
The offset was stored in a uint16_t, so a negative value wrapped to a large positive number; emit_b() then branched past the end of the program and faulted at run time.
--- lib/bpf/bpf_jit_arm64.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/bpf/bpf_jit_arm64.c +3 -1
@@ -965,10 +965,12 @@ static void emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src) { uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); -uint16_t jump_to_epilogue; +int32_t jump_to_epilogue;emit_cbnz (ctx, is64, src,3 );emit_mov_imm (ctx, is64, r0,0 ); + +/* maybe backwards branch to earlier epilogue */ jump_to_epilogue = (ctx->program_start + ctx->program_sz) - ctx->idx;emit_b (ctx, jump_to_epilogue); }
Signature · 1 line
2.53.0
PATCH v7 7/10 bpf/arm64: add BPF_ABS/BPF_IND packet load support
In reply to Stephen HemmingerThe arm64 JIT rejected BPF_LD | BPF_ABS and BPF_LD | BPF_IND with "invalid opcode", so cBPF programs converted by rte_bpf_convert() could not be JITed. Add these opcodes, mirroring the x86 JIT: a fast path for data held in the first mbuf segment, and a __rte_pktmbuf_read() slow path for everything else.
The forward branches over the call cannot use fixed distances:emit_call() materializes the helper address with a variable number of mov/movk instructions, so the block sizes are not known up front. Size the three blocks (fast path, slow path, common tail) in a dry run, then emit for real with the branches resolved from the measured offsets.
The effective offset is validated before use: src is a runtime value for BPF_IND, so a negative offset is routed to the slow path rather than read from the first segment, and the offset is bounded to UINT32_MAX before __rte_pktmbuf_read(), whose off argument is uint32_t.
Programs using these opcodes use the call register layout, since the slow path makes a function call.
For example, BPF_LD | BPF_IND | BPF_W (4-byte indirect load, mbuf in R6/x19, effective offset kept in x9) emits:
mov x9, #imm // off = imm add x9, x9, src // off += src (BPF_IND) cmp x9, xzr // reject negative b.mi slow // effective offset mov x10, #data_len_ofs ldrh w10, [x19, x10] // mbuf->data_len sub x10, x10, x9 // data_len - off mov x11, #sz cmp x10, x11 b.lt slow // not in first segment mov x10, #data_off_ofs ldrh w10, [x19, x10] // mbuf->data_off mov x7, #buf_addr_ofs ldr x7, [x19, x7] // mbuf->buf_addr add x7, x7, x10 add x7, x7, x9 // ptr = buf_addr + data_off + off b load slow: mov x10, #UINT32_MAX cmp x9, x10 b.ls 1f // off fits uint32_t ... mov x7, #0 // else return 0 b epilogue 1: mov x1, x9 // __rte_pktmbuf_read(mbuf, off, sz, buf) mov x0, x19 mov w2, #sz sub x3, x25, #stack_ofs mov x9, #<helper lo> movk x9, #<helper hi> blr x9 mov x7, x0 // ptr = return value cbnz x7, load // non-NULL -> common tail mov x7, #0 // else return 0 b epilogue load: ldr w7, [x7, xzr] // *(uint32_t *)ptr (size varies) rev32 x7, x7 // ntoh (size varies; omitted for BPF_B)
For BPF_ABS the "add x9, x9, src" is omitted; the final load/byte-swap vary with the access size.
Bugzilla ID: 1427
--- lib/bpf/bpf_jit_arm64.c | 169 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 1 deletion(-)
lib/bpf/bpf_jit_arm64.c +168 -1
@@ -1133,6 +1133,155 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off)emit_b_cond (ctx,ebpf_to_a64_cond (op),jump_offset_get (ctx, i, off)); } +/* LD_ABS/LD_IND code block offsets (in arm64 instructions) */ +enum { +LDMB_FAST_OFS ,/* fast path */ +LDMB_SLOW_OFS ,/* slow path */ +LDMB_FIN_OFS ,/* common tail */ +LDMB_OFS_NUM +}; + +/* +* Helper for emit_ld_mbuf(): fast path. +* Compute the packet offset; if it lies inside the first segment leave the +* data pointer in R0, otherwise branch to the slow path. +*/ +static void +emit_ldmb_fast_path (struct a64_jit_ctx *ctx,uint8_t src,uint8_t mode, +uint32_t sz,int32_t imm,const uint32_t ofs[LDMB_OFS_NUM ]) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); +uint8_t r6 =ebpf_to_a64_reg (ctx,EBPF_REG_6 ); +uint8_t tmp1 =ebpf_to_a64_reg (ctx,TMP_REG_1 ); +uint8_t tmp2 =ebpf_to_a64_reg (ctx,TMP_REG_2 ); +uint8_t tmp3 =ebpf_to_a64_reg (ctx,TMP_REG_3 ); + +/* off = imm (+ src for BPF_IND) */ +emit_mov_imm (ctx,1 , tmp1, imm); +if (mode ==BPF_IND ) +emit_add (ctx,1 , tmp1, src); + +/* +* A negative effective offset (src can be < 0 for BPF_IND) would pass +* the signed check below and read before the segment, so route it to +* the slow path, which rejects it via the uint32_t bound on off. +*/ +emit_cmp (ctx,1 , tmp1,A64_ZR ); +emit_b_cond (ctx,A64_MI , (int32_t )(ofs[LDMB_SLOW_OFS ] - ctx->idx)); + +/* if ((int64_t)(mbuf->data_len - off) < sz) goto slow_path */ +emit_mov_imm (ctx,1 , tmp2, offsetof(struct rte_mbuf , data_len)); +emit_ldr (ctx,BPF_H , tmp2, r6, tmp2); +emit_sub (ctx,1 , tmp2, tmp1); +emit_mov_imm (ctx,1 , tmp3, sz); +emit_cmp (ctx,1 , tmp2, tmp3); +emit_b_cond (ctx,A64_LT , (int32_t )(ofs[LDMB_SLOW_OFS ] - ctx->idx)); + +/* R0 = mbuf->buf_addr + mbuf->data_off + off */ +emit_mov_imm (ctx,1 , tmp2, offsetof(struct rte_mbuf , data_off)); +emit_ldr (ctx,BPF_H , tmp2, r6, tmp2); +emit_mov_imm (ctx,1 , r0, offsetof(struct rte_mbuf , buf_addr)); +emit_ldr (ctx,EBPF_DW , r0, r6, r0); +emit_add (ctx,1 , r0, tmp2); +emit_add (ctx,1 , r0, tmp1); + +emit_b (ctx, (int32_t )(ofs[LDMB_FIN_OFS ] - ctx->idx)); +} + +/* +* Helper for emit_ld_mbuf(): slow path. +* R0 = __rte_pktmbuf_read(mbuf, off, sz, buf); return 0 if NULL. +* The scratch buffer is the space reserved by __rte_bpf_validate() at the +* bottom of the eBPF stack frame, i.e. (frame_pointer - stack_ofs). +*/ +static void +emit_ldmb_slow_path (struct a64_jit_ctx *ctx,uint32_t sz,uint32_t stack_ofs) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); +uint8_t r6 =ebpf_to_a64_reg (ctx,EBPF_REG_6 ); +uint8_t fp =ebpf_to_a64_reg (ctx,EBPF_FP ); +uint8_t tmp1 =ebpf_to_a64_reg (ctx,TMP_REG_1 ); +uint8_t tmp2 =ebpf_to_a64_reg (ctx,TMP_REG_2 ); + +/* +* __rte_pktmbuf_read() takes a uint32_t off, so a 64-bit off that does +* not fit would be silently truncated. Return 0 if it is out of range; +* this also catches the negative off routed here by the fast path. +*/ +emit_mov_imm (ctx,1 , tmp2,UINT32_MAX ); +emit_cmp (ctx,1 , tmp1, tmp2); +emit_b_cond (ctx,A64_LS ,3 );/* off <= UINT32_MAX: do the call */ +emit_mov_imm (ctx,1 , r0,0 ); +emit_b (ctx, (ctx->program_start + ctx->program_sz) - ctx->idx); + +/* arguments of __rte_pktmbuf_read(mbuf, off, len, buf) */ +emit_mov_64 (ctx,A64_R (1 ), tmp1);/* off (held in tmp1) */ +emit_mov_64 (ctx,A64_R (0 ), r6);/* mbuf */ +emit_mov_imm (ctx,0 ,A64_R (2 ), sz);/* len */ +emit_sub_imm_64 (ctx,A64_R (3 ), fp, stack_ofs);/* buf */ + +emit_call (ctx, tmp1, (void *)(uintptr_t )__rte_pktmbuf_read); +emit_return_zero_if_src_zero (ctx,1 , r0); +} + +/* +* Helper for emit_ld_mbuf(): common tail. +* Load the value pointed to by R0 and convert from network byte order. +*/ +static void +emit_ldmb_fin (struct a64_jit_ctx *ctx,uint8_t opsz,uint32_t sz) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); + +emit_ldr (ctx, opsz, r0, r0,A64_ZR ); +if (opsz !=BPF_B ) +emit_be (ctx, r0, sz *8 ); +} + +/* +* Emit code for BPF_LD | BPF_ABS and BPF_LD | BPF_IND packet loads: +* +* off = imm (+ src for BPF_IND) +* if (off >= 0 && mbuf->data_len - off >= sz) -- fast path +* ptr = mbuf->buf_addr + mbuf->data_off + off; +* else -- slow path +* if ((uint64_t)off > UINT32_MAX) +* return 0; +* ptr = __rte_pktmbuf_read(mbuf, off, sz, buf); +* if (ptr == NULL) +* return 0; +* R0 = ntoh(*(size *)ptr); -- common tail +* +* The three blocks are sized in a dry run so the forward branches can be +* resolved, then emitted for real (arm64 instructions are fixed width, so +* the dry run reproduces the real instruction count exactly). +*/ +static void +emit_ld_mbuf (struct a64_jit_ctx *ctx,uint8_t op,uint8_t src,int32_t imm, +uint32_t stack_ofs) +{ +uint8_t mode =BPF_MODE (op); +uint8_t opsz =BPF_SIZE (op); +uint32_t sz =bpf_size (opsz); +uint32_t ofs[LDMB_OFS_NUM ]; + +/* seed offsets so the dry-run branches stay in range */ + ofs[LDMB_FAST_OFS ] = ofs[LDMB_SLOW_OFS ] = ofs[LDMB_FIN_OFS ] = ctx->idx; + +/* dry run to record block offsets */ +emit_ldmb_fast_path (ctx, src, mode, sz, imm, ofs); + ofs[LDMB_SLOW_OFS ] = ctx->idx; +emit_ldmb_slow_path (ctx, sz, stack_ofs); + ofs[LDMB_FIN_OFS ] = ctx->idx; +emit_ldmb_fin (ctx, opsz, sz); + +/* rewind and emit for real with resolved offsets */ + ctx->idx = ofs[LDMB_FAST_OFS ]; +emit_ldmb_fast_path (ctx, src, mode, sz, imm, ofs); +emit_ldmb_slow_path (ctx, sz, stack_ofs); +emit_ldmb_fin (ctx, opsz, sz); +} +static void check_program_has_call (struct a64_jit_ctx *ctx,struct rte_bpf *bpf) {
@@ -1145,8 +1294,17 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) op = ins->code;switch (op) { -/* Call imm */ +/* +* BPF_ABS/BPF_IND can fall through to __rte_pktmbuf_read(), +* so they need the call-clobbered register layout as well. +*/ case (BPF_JMP |EBPF_CALL ): +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): ctx->foundcall =1 ;return ; }
@@ -1348,6 +1506,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)emit_mov_imm (ctx,1 , dst, u64); i++;break ; +/* R0 = ntoh(*(size *)(mbuf data + (src) + imm)) */ +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): +emit_ld_mbuf (ctx, op, src, imm, bpf->stack_sz); +break ;/* *(size *)(dst + off) = src */ case (BPF_STX |BPF_MEM |BPF_B ):case (BPF_STX |BPF_MEM |BPF_H ):
Signature · 1 line
2.53.0
PATCH v7 8/10 test/bpf: check that JIT was generated
In reply to Stephen HemmingerAvoid silently ignoring JIT failures. The test cases should all succeed JIT compilation; if not it is a bug in the JIT implementation and should be reported.
Introduce a configuration setting RTE_BPF_JIT_SUPPORTED which is cleaner than using an ARCH specific #ifdef.
--- app/test/test_bpf.c | 8 ++++++++ lib/bpf/meson.build | 2 ++ 2 files changed, 10 insertions(+)
app/test/test_bpf.c +8 -0
@@ -3748,6 +3748,14 @@ run_test(const struct bpf_test *tst) rv, strerror(rv)); } } +#ifdef RTE_BPF_JIT_SUPPORTED + else { +/* a JIT backend exists for this arch, so it must compile */ +printf ("%s@%d: %s: no JIT code generated;\n" , + __func__, __LINE__, tst->name); + ret =-1 ; + } +#endif rte_bpf_destroy (bpf);return ret;
lib/bpf/meson.build +2 -0
@@ -29,8 +29,10 @@ sources = files( ) if arch_subdir == 'x86' and dpdk_conf.get('RTE_ARCH_64') + dpdk_conf.set('RTE_BPF_JIT_SUPPORTED', 1) sources += files('bpf_jit_x86.c') elif dpdk_conf.has('RTE_ARCH_ARM64') + dpdk_conf.set('RTE_BPF_JIT_SUPPORTED', 1) sources += files('bpf_jit_arm64.c') endif
Signature · 1 line
2.53.0
PATCH v7 9/10 test/bpf: check that bpf_convert can be JIT'd
In reply to Stephen HemmingerRun each converted filter through both the interpreter and the JIT and check they agree, catching JIT miscompiles.
test_bpf_filter and test_bpf_match did nearly the same thing: compile, load and run a filter against the dummy packet. Combine them into test_bpf_match, which now builds the packet itself and returns whether the filter matched. Callers run it for both load methods.
The dummy packet is a UDP packet to a fixed destination MAC, source and destination ports, so the filter results are deterministic. None of the sample filters should match it, so assert that; a convert or JIT bug that flips a result is then caught. The destination MAC and source port are chosen so the negative ethernet and port filters do not match, and "port not 53 and not arp" is dropped as it matches any non-ARP packet that lacks port 53.
Reduce log output to make it easier to match which expression might be causing issues.
--- app/test/test_bpf.c | 171 ++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 71 deletions(-)
app/test/test_bpf.c +100 -71
@@ -32,6 +32,7 @@ test_bpf(void)#include <rte_bpf.h> #include <rte_ether.h> #include <rte_ip.h> +#include <rte_udp.h> /* Tests of most simple BPF programs (no instructions, one instruction etc.) */
@@ -4855,11 +4856,13 @@ load_cbpf_program_convert(struct bpf_program *cbpf_program, const char *str)return NULL ; } +#ifdef DEBUG printf ("bpf convert(\"%s\") produced:\n" , str);rte_bpf_dump (stdout, prm->ins, prm->nb_ins);printf ("%s \"%s\"\n" , __func__, str);test_bpf_dump (cbpf_program, prm); +#endif bpf =rte_bpf_load (prm);rte_free (prm);
@@ -4884,18 +4887,65 @@ load_cbpf_program_direct(struct bpf_program *cbpf_program, const char *str __rte }); } +static const load_cbpf_program_t cbpf_program_loaders[] = { + load_cbpf_program_convert, + load_cbpf_program_direct, +}; + +/* Setup Ethernet/IP/UDP headers in a dummy packet buffer for filter tests */ +static void +dummy_ip_prep (void *data,uint16_t plen) +{ +struct { +struct rte_ether_hdr eth_hdr; +struct rte_ipv4_hdr ip_hdr; +struct rte_udp_hdr udp_hdr; + } *hdr = data; + + hdr->eth_hdr = (struct rte_ether_hdr ) { + .dst_addr.addr_bytes = {0x01 ,0x80 ,0xc2 ,0x00 ,0x00 ,0x0e }, + .ether_type =rte_cpu_to_be_16 (RTE_ETHER_TYPE_IPV4 ), + }; + hdr->ip_hdr = (struct rte_ipv4_hdr ) { + .version_ihl =RTE_IPV4_VHL_DEF , + .total_length =rte_cpu_to_be_16 (plen), + .time_to_live =IPDEFTTL , + .next_proto_id =IPPROTO_UDP , + .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), + .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), + }; + hdr->udp_hdr = (struct rte_udp_hdr ) { + .src_port =rte_cpu_to_be_16 (49152 ),/* fixed, avoids filter ports */ + .dst_port =rte_cpu_to_be_16 (9 ),/* discard port */ + .dgram_len =rte_cpu_to_be_16 (plen -sizeof (struct rte_ipv4_hdr )), + .dgram_cksum =0 , + }; +} + +/* +* Compile a pcap filter, load it with the given loader, then run it against +* a standard dummy packet with both the interpreter and (when available) the +* JIT, checking the two agree. +* +* Returns 1 if the filter matched, 0 if it did not, and -1 on any error +* (compile, load, or interpreter/JIT mismatch). +*/ static int -test_bpf_match (pcap_t *pcap,const char *str,struct rte_mbuf *mb, +test_bpf_match (pcap_t *pcap,const char *str,load_cbpf_program_t load_cbpf_program) { +uint8_t tbuf[RTE_MBUF_DEFAULT_BUF_SIZE ]; +const uint32_t plen =100 ;struct bpf_program fcode; -struct rte_bpf *bpf; +struct rte_mbuf mb = {0 }; +struct rte_bpf *bpf =NULL ;int ret =-1 ;uint64_t rc; +printf ("%s '%s'\n" , __func__, str);if (pcap_compile (pcap, &fcode, str,1 ,PCAP_NETMASK_UNKNOWN )) {printf ("%s@%d: pcap_compile(\"%s\") failed: %s;\n" , - __func__, __LINE__, str,pcap_geterr (pcap)); + __func__, __LINE__, str,pcap_geterr (pcap));return -1 ; }
@@ -4903,15 +4953,41 @@ test_bpf_match(pcap_t *pcap, const char *str, struct rte_mbuf *mb,if (bpf ==NULL ) {printf ("%s@%d: failed to load cbpf program for \"%s\", error=%d(%s);\n" , __func__, __LINE__, str, rte_errno,strerror (rte_errno)); +test_bpf_dump (&fcode,NULL ); goto error; } - rc =rte_bpf_exec (bpf, mb); -/* The return code from bpf capture filter is non-zero if matched */ - ret = (rc ==0 ); +dummy_mbuf_prep (&mb, tbuf,sizeof (tbuf), plen); + dummy_ip_prep(rte_pktmbuf_mtod(&mb, void *), plen); + + rc =rte_bpf_exec (bpf, &mb); + +/* Verify the JIT, when available, produces the same result. */ + { +struct rte_bpf_jit jit; + +rte_bpf_get_jit (bpf, &jit); +if (jit.func !=NULL ) { +fflush (stdout); +if (jit.func (&mb) != rc) { +printf ("%s@%d: JIT return code does not match\n" , + __func__, __LINE__); + goto error; + } + } +#ifdef RTE_BPF_JIT_SUPPORTED + else { +printf ("%s@%d: no JIT code generated\n" , + __func__, __LINE__); + goto error; + } +#endif + } + +/* The return code from a bpf capture filter is non-zero if matched. */ + ret = (rc !=0 ); error: -if (bpf) -rte_bpf_destroy (bpf); +rte_bpf_destroy (bpf);pcap_freecode (&fcode);return ret; }
@@ -4920,44 +4996,13 @@ test_bpf_match(pcap_t *pcap, const char *str, struct rte_mbuf *mb,static int test_bpf_filter_sanity (pcap_t *pcap) { -static const load_cbpf_program_t cbpf_program_loaders[] = { - load_cbpf_program_convert, - load_cbpf_program_direct, - }; - -const uint32_t plen =100 ; -struct rte_mbuf mb, *m; -uint8_t tbuf[RTE_MBUF_DEFAULT_BUF_SIZE ]; -struct { -struct rte_ether_hdr eth_hdr; -struct rte_ipv4_hdr ip_hdr; - } *hdr; - -memset (&mb,0 ,sizeof (mb)); -dummy_mbuf_prep (&mb, tbuf,sizeof (tbuf), plen); - m = &mb; - - hdr =rte_pktmbuf_mtod (m,typeof (hdr)); - hdr->eth_hdr = (struct rte_ether_hdr ) { - .dst_addr.addr_bytes = {0xff ,0xff ,0xff ,0xff ,0xff ,0xff }, - .ether_type =rte_cpu_to_be_16 (RTE_ETHER_TYPE_IPV4 ), - }; - hdr->ip_hdr = (struct rte_ipv4_hdr ) { - .version_ihl =RTE_IPV4_VHL_DEF , - .total_length =rte_cpu_to_be_16 (plen), - .time_to_live =IPDEFTTL , - .next_proto_id =IPPROTO_RAW , - .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), - .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), - }; - -for (int li =0 ; li !=RTE_DIM (cbpf_program_loaders); ++li) { -if (test_bpf_match (pcap,"ip" , m, cbpf_program_loaders[li]) !=0 ) { +for (unsigned int li =0 ; li !=RTE_DIM (cbpf_program_loaders); ++li) { +if (test_bpf_match (pcap,"ip" , cbpf_program_loaders[li]) !=1 ) {printf ("%s@%d: filter \"ip\" doesn't match test data\n" , __func__, __LINE__);return -1 ; } -if (test_bpf_match (pcap,"not ip" , m, cbpf_program_loaders[li]) ==0 ) { +if (test_bpf_match (pcap,"not ip" , cbpf_program_loaders[li]) !=0 ) {printf ("%s@%d: filter \"not ip\" does match test data\n" , __func__, __LINE__);return -1 ;
@@ -4981,7 +5026,6 @@ static const char * const sample_filters[] = {"port 53" ,"host 192.0.2.1 and not (port 80 or port 25)" ,"host 2001:4b98:db0::8 and not port 80 and not port 25" , -"port not 53 and not arp" ,"(tcp[0:2] > 1500 and tcp[0:2] < 1550) or (tcp[2:2] > 1500 and tcp[2:2] < 1550)" ,"ether proto 0x888e" , "ether[0] & 1 = 0 and ip[16] >= 224",
@@ -5008,35 +5052,10 @@ static const char * const sample_filters[] = { "or host 192.0.2.1 or host 192.0.2.100 or host 192.0.2.200"), }; -static int -test_bpf_filter (pcap_t *pcap,const char *s,load_cbpf_program_t load_cbpf_program) -{ -struct bpf_program fcode; -struct rte_bpf *bpf; - -if (pcap_compile (pcap, &fcode, s,1 ,PCAP_NETMASK_UNKNOWN )) { -printf ("%s@%d: pcap_compile(\"%s\") failed: %s;\n" , - __func__, __LINE__, s,pcap_geterr (pcap)); -return -1 ; - } - - bpf =load_cbpf_program (&fcode, s); -if (bpf ==NULL ) { -printf ("%s@%d: failed to load cbpf program for \"%s\", error=%d(%s);\n" , - __func__, __LINE__, s, rte_errno,strerror (rte_errno)); -test_bpf_dump (&fcode,NULL ); - } - -rte_bpf_destroy (bpf); - -pcap_freecode (&fcode); -return (bpf ==NULL ) ?-1 :0 ; -} -static int test_bpf_convert (void ) { -unsigned int i; +unsigned int i, li;pcap_t *pcap;int rc;
@@ -5048,8 +5067,18 @@ test_bpf_convert(void) rc =test_bpf_filter_sanity (pcap);for (i =0 ; i <RTE_DIM (sample_filters); i++) { - rc |=test_bpf_filter (pcap, sample_filters[i], load_cbpf_program_convert); - rc |=test_bpf_filter (pcap, sample_filters[i], load_cbpf_program_direct); +for (li =0 ; li <RTE_DIM (cbpf_program_loaders); li++) { +int m =test_bpf_match (pcap, sample_filters[i], + cbpf_program_loaders[li]); + +/* None of the sample filters match the dummy packet. */ +if (m !=0 ) { +if (m >0 ) +printf ("%s@%d: filter \"%s\" unexpectedly matched\n" , + __func__, __LINE__, sample_filters[i]); + rc =-1 ; + } + } }pcap_close (pcap);
Signature · 1 line
2.53.0
PATCH v7 10/10 bpf: fix uninitialized warning
In reply to Stephen HemmingerCoverity complains uninitialized use of structure.
Coverity issue: 504611
--- lib/bpf/bpf_validate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bpf/bpf_validate.c +1 -1
@@ -659,7 +659,7 @@ eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)static void eval_add (struct bpf_reg_val *rd,const struct bpf_reg_val *rs,uint64_t msk) { -struct bpf_reg_val rs_buf; +struct bpf_reg_val rs_buf = {0 };struct bpf_reg_val rv;if (RTE_BPF_ARG_PTR_TYPE (rs->v.type) !=0 ) {
Signature · 1 line
2.53.0
RE: [PATCH v7 10/10] bpf: fix uninitialized warning
Quoted 33 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Wednesday 22 July 2026 17:05
To: [email protected]
Cc: Stephen Hemminger <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Marat Khalili <[email protected]>
Subject: [PATCH v7 10/10] bpf: fix uninitialized warningCoverity complains uninitialized use of structure.
Coverity issue: 504611
Fixes 17509d474226 ("bpf/validate: fix BPF_ADD of pointer to a scalar")Signed-off-by Stephen Hemminger <[email protected]>--- lib/bpf/bpf_validate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)lib/bpf/bpf_validate.c +1
-2@@ -659,7 +659,7 @@ eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)static void eval_add (struct bpf_reg_val *rd,const struct bpf_reg_val *rs,uint64_t msk) { -struct bpf_reg_val rs_buf; +struct bpf_reg_val rs_buf = {0 };struct bpf_reg_val rv;if (RTE_BPF_ARG_PTR_TYPE (rs->v.type) !=0 ) { --
Quoted 1 line
2.53.0
v8 cover bpf: JIT and interpreter fixes
In reply to a message not in the archiveBPF fixes, mostly in the JIT backends. Bugzilla 1959 and 1427.
The existing JIT tests were never run on arm64, which is how the gaps here went unnoticed. The tests now fail rather than skip when a JIT backend exists but produces nothing.
Tested on x86_64 and arm64.
Christophe Fontaine (1):
bpf/arm64: fix offset type to allow a negative jump
Stephen Hemminger (9):
bpf/x86: fix JIT encoding of fixed-width immediates test/bpf: add JSET test with small immediate
bpf: mask shift count in interpreter per RFC 9669 bpf/arm64: mask shift count per RFC 9669
test/bpf: add test for large shift
Subject: [PATCH 07/10] bpf/arm64: add BPF_ABS/BPF_IND packet load
support
test/bpf: check that JIT was generated
test/bpf: check that bpf_convert can be JIT'd
bpf: fix uninitialized warning
app/test/test_bpf.c | 320 +++++++++++++++++++++++++++++++--------- lib/bpf/bpf_exec.c | 31 ++-- lib/bpf/bpf_jit_arm64.c | 204 ++++++++++++++++++++++++- lib/bpf/bpf_jit_x86.c | 6 +- lib/bpf/bpf_validate.c | 2 +- lib/bpf/meson.build | 2 + 6 files changed, 476 insertions(+), 89 deletions(-)
Signature · 1 line
2.53.0
PATCH v8 1/10 bpf/x86: fix JIT encoding of fixed-width immediates
Several places in the x86 JIT size an immediate with imm_size(), which returns 1 or 4 bytes depending on the value. That is wrong for opcodes whose immediate width is fixed by the encoding, and it breaks in both directions.
TEST (0xF7 /0, used for BPF_JSET) has no imm8 form; the immediate is always 32 bits. For a small mask such as BPF_JSET | BPF_K #0x1, imm_size() returns 1, so the JIT emits a 1-byte immediate. The CPU still consumes 4, swallowing 3 bytes of the following Jcc. The instruction stream desyncs and the program crashes.
ROR and the shifts (0xC1 group) have the opposite problem: their immediate is always imm8. For a count >= 128, imm_size() returns 4 and the JIT emits 3 stray bytes, again desyncing the stream.
Size each immediate by its encoding: 32 bits for TEST, 8 bits for ROR and the shifts.
Bugzilla ID: 1959
--- lib/bpf/bpf_jit_x86.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/bpf/bpf_jit_x86.c +3 -3
@@ -300,7 +300,7 @@ emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)emit_rex (st,BPF_ALU ,0 , dreg);emit_bytes (st, &ops,sizeof (ops));emit_modregrm (st,MOD_DIRECT , mods, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (uint8_t )); }/*
@@ -441,7 +441,7 @@ emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) {emit_shift (st, op, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (uint8_t )); }/*
@@ -921,7 +921,7 @@ emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)emit_rex (st, op,0 , dreg);emit_bytes (st, &ops,sizeof (ops));emit_modregrm (st,MOD_DIRECT , mods, dreg); -emit_imm (st, imm,imm_size (imm)); +emit_imm (st, imm,sizeof (int32_t )); } static void
Signature · 1 line
2.53.0
PATCH v8 2/10 test/bpf: add JSET test with small immediate
In reply to Stephen HemmingerThe existing jump test only used a 32-bit JSET mask,
so the broken imm8 encoding of TEST in the x86 JIT was never exercised. Add a case with a byte-sized mask;run_test() runs it through the interpreter and the JIT.
--- app/test/test_bpf.c | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+)
app/test/test_bpf.c +82 -0
@@ -3229,7 +3229,89 @@ test_int64min_umod_uint64max_check(uint64_t rc, const void *arg) }/* all bpf test cases */ +/* +* JSET with a byte-sized mask: exercises the imm8 path of the TEST +* encoding in the x86 JIT (a 32-bit mask takes a different path). +*/ +static const struct ebpf_insn test_jset1_prog[] = { + { + .code = (BPF_ALU |EBPF_MOV |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0 , + }, + { + .code = (BPF_LDX |BPF_MEM |BPF_B ), + .dst_reg =EBPF_REG_2 , + .src_reg =EBPF_REG_1 , + .off = offsetof(struct dummy_offset , u8), + }, +/* bit 0 is set in the input: branch is taken */ + { + .code = (BPF_JMP |BPF_JSET |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0x1 , + .off =1 , + }, + { + .code = (BPF_JMP |BPF_JA ), + .off =1 , + }, + { + .code = (EBPF_ALU64 |BPF_OR |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0x1 , + }, +/* bit 1 is clear in the input: branch is not taken */ + { + .code = (BPF_JMP |BPF_JSET |BPF_K ), + .dst_reg =EBPF_REG_2 , + .imm =0x2 , + .off =1 , + }, + { + .code = (BPF_JMP |BPF_JA ), + .off =1 , + }, + { + .code = (EBPF_ALU64 |BPF_OR |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =0x2 , + }, + { + .code = (BPF_JMP |EBPF_EXIT ), + }, +}; + +static void +test_jset1_prepare (void *arg) +{ +struct dummy_offset *df = arg; + +memset (df,0 ,sizeof (*df)); + df->u8 =0x1 ;/* bit 0 set, bit 1 clear */ +} + +static int +test_jset1_check (uint64_t rc,const void *arg) +{ +return cmp_res (__func__,0x1 , rc, arg, arg,0 ); +} +static const struct bpf_test tests[] = { + { + .name ="test_jset1" , + .arg_sz =sizeof (struct dummy_offset ), + .prm = { + .ins = test_jset1_prog, + .nb_ins =RTE_DIM (test_jset1_prog), + .prog_arg = { + .type =RTE_BPF_ARG_PTR , + .size =sizeof (struct dummy_offset ), + }, + }, + .prepare = test_jset1_prepare, + .check_result = test_jset1_check, + }, { .name ="test_store1" , .arg_sz =sizeof (struct dummy_offset ),
Signature · 1 line
2.53.0
PATCH v8 3/10 bpf: mask shift count in interpreter per RFC 9669
In reply to Stephen HemmingerThe interpreter shifted by the raw immediate or register value, which is undefined behavior in C when the count is >= the operand width and trips UBSan. RFC 9669 masks shift counts (0x3f for 64-bit, 0x1f for 32-bit); mask the count in the LSH/RSH/ARSH cases.
--- lib/bpf/bpf_exec.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-)
lib/bpf/bpf_exec.c +21 -10
@@ -4,6 +4,7 @@#include <stdio.h> #include <stdint.h> +#include <limits.h> #include <eal_export.h> #include <rte_common.h>
@@ -43,6 +44,16 @@ ((reg)[(ins)->dst_reg] = \ (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) +#define BPF_OP_SHIFT_IMM (reg, ins, op, type) \ + ((reg)[(ins)->dst_reg] = \ + (type)(reg)[(ins)->dst_reg] op \ + ((ins)->imm & (sizeof(type) * CHAR_BIT - 1))) + +#define BPF_OP_SHIFT_REG (reg, ins, op, type) \ + ((reg)[(ins)->dst_reg] = \ + (type)(reg)[(ins)->dst_reg] op \ + ((reg)[(ins)->src_reg] & (sizeof(type) * CHAR_BIT - 1))) +#define BPF_DIV_ZERO_CHECK (bpf, reg, ins, type) do { \ if ((type)(reg)[(ins)->src_reg] == 0) { \ RTE_BPF_LOG_LINE(ERR, \
@@ -183,10 +194,10 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_IMM(reg, ins, |, uint32_t);break ;case (BPF_ALU |BPF_LSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, <<, uint32_t); + BPF_OP_SHIFT_IMM(reg, ins, <<, uint32_t);break ;case (BPF_ALU |BPF_RSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, uint32_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, uint32_t);break ;case (BPF_ALU |BPF_XOR |BPF_K ): BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
@@ -217,10 +228,10 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_REG(reg, ins, |, uint32_t);break ;case (BPF_ALU |BPF_LSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, <<, uint32_t); + BPF_OP_SHIFT_REG(reg, ins, <<, uint32_t);break ;case (BPF_ALU |BPF_RSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, uint32_t); + BPF_OP_SHIFT_REG(reg, ins, >>, uint32_t);break ;case (BPF_ALU |BPF_XOR |BPF_X ): BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
@@ -262,13 +273,13 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_IMM(reg, ins, |, uint64_t);break ;case (EBPF_ALU64 |BPF_LSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, <<, uint64_t); + BPF_OP_SHIFT_IMM(reg, ins, <<, uint64_t);break ;case (EBPF_ALU64 |BPF_RSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, uint64_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, uint64_t);break ;case (EBPF_ALU64 |EBPF_ARSH |BPF_K ): - BPF_OP_ALU_IMM(reg, ins, >>, int64_t); + BPF_OP_SHIFT_IMM(reg, ins, >>, int64_t);break ;case (EBPF_ALU64 |BPF_XOR |BPF_K ): BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
@@ -299,13 +310,13 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) BPF_OP_ALU_REG(reg, ins, |, uint64_t);break ;case (EBPF_ALU64 |BPF_LSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, <<, uint64_t); + BPF_OP_SHIFT_REG(reg, ins, <<, uint64_t);break ;case (EBPF_ALU64 |BPF_RSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, uint64_t); + BPF_OP_SHIFT_REG(reg, ins, >>, uint64_t);break ;case (EBPF_ALU64 |EBPF_ARSH |BPF_X ): - BPF_OP_ALU_REG(reg, ins, >>, int64_t); + BPF_OP_SHIFT_REG(reg, ins, >>, int64_t);break ;case (EBPF_ALU64 |BPF_XOR |BPF_X ): BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
Signature · 1 line
2.53.0
PATCH v8 4/10 bpf/arm64: mask shift count per RFC 9669
In reply to Stephen HemmingerThe ARM JIT was not masking the shift count as required by RFC 9669 (0x3f for 64-bit, 0x1f for 32-bit), so large immediate shift counts overflowed the UBFM/SBFM encoding and failed the JIT. Mask the immediate in emit_lsl/emit_lsr/emit_asr.
--- lib/bpf/bpf_jit_arm64.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
lib/bpf/bpf_jit_arm64.c +10 -2
@@ -545,12 +545,14 @@ emit_bitfield(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,emit_insn (ctx, insn,check_reg (rd) ||check_reg (rn) ||check_immr_imms (is64, immr, imms)); } +static void emit_lsl (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) {const unsigned int width = is64 ?64 :32 ;uint8_t imms, immr; + imm &= width -1 ; immr = (width - imm) & (width -1 ); imms = width -1 - imm;
@@ -560,13 +562,19 @@ emit_lsl(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)static void emit_lsr (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) { -emit_bitfield (ctx, is64, rd, rd, imm, is64 ?63 :31 ,A64_UBFM ); +const unsigned int width = is64 ?64 :32 ; + + imm &= width -1 ; +emit_bitfield (ctx, is64, rd, rd, imm, width -1 ,A64_UBFM ); }static void emit_asr (struct a64_jit_ctx *ctx,bool is64,uint8_t rd,uint8_t imm) { -emit_bitfield (ctx, is64, rd, rd, imm, is64 ?63 :31 ,A64_SBFM ); +const unsigned int width = is64 ?64 :32 ; + + imm &= width -1 ; +emit_bitfield (ctx, is64, rd, rd, imm, width -1 ,A64_SBFM ); }#define A64_AND 0
Signature · 1 line
2.53.0
PATCH v8 5/10 test/bpf: add test for large shift
In reply to Stephen HemmingerThere were multiple bugs with immediate values in shift instructions. The code was not masking as required by RFC.
Add new tests that cover these instructions.
--- app/test/test_bpf.c | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
app/test/test_bpf.c +59 -0
@@ -2012,6 +2012,51 @@ test_div1_check(uint64_t rc, const void *arg)return cmp_res (__func__,0 , rc, dve.out, dvt->out,sizeof (dve.out)); } +/* +* Shift counts are masked to the operand width (RFC 9669: 0x3f for 64-bit, +* 0x1f for 32-bit). Counts >= 128 also exercise the x86 imm_size() path that +* used to desync the stream, and the arm64 UBFM/SBFM immediate encoding. +*/ +static const struct ebpf_insn test_shift_big_imm_prog[] = { + { + .code = (EBPF_ALU64 |EBPF_MOV |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =1 + }, + { + .code = (EBPF_ALU64 |BPF_LSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =191 + }, + { + .code = (EBPF_ALU64 |EBPF_ARSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =200 + }, + { + .code = (EBPF_ALU64 |BPF_RSH |BPF_K ), + .dst_reg =EBPF_REG_0 , + .imm =130 + }, + { + .code = (BPF_JMP |EBPF_EXIT ) + }, +}; + +static void +test_shift_big_imm_prepare (void *arg) +{ +memset (arg,0 ,sizeof (struct dummy_offset )); +} + +static int +test_shift_big_imm_check (uint64_t rc,const void *arg) +{ +uint64_t expect =0x3FE0000000000000ULL ; + +return cmp_res (__func__, expect, rc, arg, arg,0 ); +} +/* call test-cases */ static const struct ebpf_insn test_call1_prog[] = {
@@ -3480,6 +3525,20 @@ static const struct bpf_test tests[] = { .prepare = test_mul1_prepare, .check_result = test_div1_check, }, + { + .name = "test_shift_big_imm", + .arg_sz = sizeof(struct dummy_offset), + .prm = { + .ins = test_shift_big_imm_prog, + .nb_ins = RTE_DIM(test_shift_big_imm_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR, + .size =sizeof (struct dummy_offset ), + }, + }, + .prepare = test_shift_big_imm_prepare, + .check_result = test_shift_big_imm_check, + }, { .name ="test_call1" , .arg_sz = sizeof(struct dummy_offset),
Signature · 1 line
2.53.0
PATCH v8 6/10 bpf/arm64: fix offset type to allow a negative jump
In reply to Stephen HemmingerFrom: Christophe Fontaine <[email protected]>
The DPDK BPF JIT standalone test test_ld_mbuf1 fails on arm64. It does:
r6 = r1 // mbuf r0 = *(u8 *)pkt[0] // BPF_ABS if ((r0 & 0xf0) == 0x40) goto parse r0 = 0 exit // epilogue E0 parse: r0 = *(u8 *)pkt[r0 + 3] // BPF_IND ... exit
emit_return_zero_if_src_zero() returns 0 by branching to a function epilogue. The target may be a previous epilogue so branch might be backwards; therefore the offset needs to be negative.
The offset was stored in a uint16_t, so a negative value wrapped to a large positive number; emit_b() then branched past the end of the program and faulted at run time.
--- lib/bpf/bpf_jit_arm64.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/bpf/bpf_jit_arm64.c +3 -1
@@ -965,10 +965,12 @@ static void emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src) { uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); -uint16_t jump_to_epilogue; +int32_t jump_to_epilogue;emit_cbnz (ctx, is64, src,3 );emit_mov_imm (ctx, is64, r0,0 ); + +/* maybe backwards branch to earlier epilogue */ jump_to_epilogue = (ctx->program_start + ctx->program_sz) - ctx->idx;emit_b (ctx, jump_to_epilogue); }
Signature · 1 line
2.53.0
PATCH v8 7/10 Subject: [PATCH 07/10] bpf/arm64: add BPF_ABS/BPF_IND packet load support
In reply to Stephen HemmingerThe arm64 JIT rejected BPF_LD | BPF_ABS and BPF_LD | BPF_IND with "invalid opcode", so cBPF programs converted by rte_bpf_convert() could not be JITed. Add these opcodes, mirroring the x86 JIT: a fast path for data held in the first mbuf segment, and a __rte_pktmbuf_read() slow path for everything else.
emit_call() materializes the helper address with a variable number of instructions, so the block sizes are not known up front. Size the three blocks in a dry run, then emit for real with the branches resolved.
For BPF_IND the offset is a runtime value, so it is checked for being negative or too large for the uint32_t off argument of __rte_pktmbuf_read(). For BPF_ABS the offset is the instruction immediate, so those checks are dropped and a negative immediate compiles to a return of zero.
Programs using these opcodes use the call register layout, since the slow path makes a function call.
Bugzilla ID: 1427
--- lib/bpf/bpf_jit_arm64.c | 188 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 1 deletion(-)
lib/bpf/bpf_jit_arm64.c +187 -1
@@ -8,6 +8,7 @@#include <rte_common.h> #include <rte_byteorder.h> +#include <rte_debug.h> #include "bpf_impl.h"
@@ -1133,6 +1134,173 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off)emit_b_cond (ctx,ebpf_to_a64_cond (op),jump_offset_get (ctx, i, off)); } +/* LD_ABS/LD_IND code block offsets (in arm64 instructions) */ +enum { +LDMB_FAST_OFS ,/* fast path */ +LDMB_SLOW_OFS ,/* slow path */ +LDMB_FIN_OFS ,/* common tail */ +LDMB_OFS_NUM +}; +/* +* Helper for emit_ld_mbuf(): fast path. +* Compute the packet offset; if it lies inside the first segment leave the +* data pointer in R0, otherwise branch to the slow path. +*/ +static void +emit_ldmb_fast_path (struct a64_jit_ctx *ctx,uint8_t src,uint8_t mode, +uint32_t sz,int32_t imm,const uint32_t ofs[LDMB_OFS_NUM ]) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); +uint8_t r6 =ebpf_to_a64_reg (ctx,EBPF_REG_6 ); +uint8_t tmp1 =ebpf_to_a64_reg (ctx,TMP_REG_1 ); +uint8_t tmp2 =ebpf_to_a64_reg (ctx,TMP_REG_2 ); +uint8_t tmp3 =ebpf_to_a64_reg (ctx,TMP_REG_3 ); + +emit_mov_imm (ctx,1 , tmp1, imm); + +if (mode ==BPF_IND ) { +/* off = imm + src */ +emit_add (ctx,1 , tmp1, src); + +/* +* src is a runtime value, so off may be negative. +* That would pass the signed check below and read before the segment, +* so route it to the slow path which rejects it. +*/ +emit_cmp (ctx,1 , tmp1,A64_ZR ); +emit_b_cond (ctx,A64_MI , (int32_t )(ofs[LDMB_SLOW_OFS ] - ctx->idx)); + } + +/* if ((int64_t)(mbuf->data_len - off) < sz) goto slow_path */ +emit_mov_imm (ctx,1 , tmp2, offsetof(struct rte_mbuf , data_len)); +emit_ldr (ctx,BPF_H , tmp2, r6, tmp2); +emit_sub (ctx,1 , tmp2, tmp1); +emit_mov_imm (ctx,1 , tmp3, sz); +emit_cmp (ctx,1 , tmp2, tmp3); +emit_b_cond (ctx,A64_LT , (int32_t )(ofs[LDMB_SLOW_OFS ] - ctx->idx)); + +/* R0 = mbuf->buf_addr + mbuf->data_off + off */ +emit_mov_imm (ctx,1 , tmp2, offsetof(struct rte_mbuf , data_off)); +emit_ldr (ctx,BPF_H , tmp2, r6, tmp2); +emit_mov_imm (ctx,1 , r0, offsetof(struct rte_mbuf , buf_addr)); +emit_ldr (ctx,EBPF_DW , r0, r6, r0); +emit_add (ctx,1 , r0, tmp2); +emit_add (ctx,1 , r0, tmp1); + +emit_b (ctx, (int32_t )(ofs[LDMB_FIN_OFS ] - ctx->idx)); +} + +/* +* Helper for emit_ld_mbuf(): common tail. +* Load the value pointed to by R0 and convert from network byte order. +*/ +static void +emit_ldmb_fin (struct a64_jit_ctx *ctx,uint8_t opsz,uint32_t sz) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); + +emit_ldr (ctx, opsz, r0, r0,A64_ZR ); +if (opsz !=BPF_B ) +emit_be (ctx, r0, sz *8 ); +} + +/* +* Helper for emit_ld_mbuf(): slow path. +* R0 = __rte_pktmbuf_read(mbuf, off, sz, buf); return 0 if NULL. +* The scratch buffer is the space reserved by __rte_bpf_validate() at the +* bottom of the eBPF stack frame, i.e. (frame_pointer - stack_ofs). +*/ +static void +emit_ldmb_slow_path (struct a64_jit_ctx *ctx,uint8_t mode,uint32_t sz, +uint32_t stack_ofs) +{ +uint8_t r0 =ebpf_to_a64_reg (ctx,EBPF_REG_0 ); +uint8_t r6 =ebpf_to_a64_reg (ctx,EBPF_REG_6 ); +uint8_t fp =ebpf_to_a64_reg (ctx,EBPF_FP ); +uint8_t tmp1 =ebpf_to_a64_reg (ctx,TMP_REG_1 ); +uint8_t tmp2 =ebpf_to_a64_reg (ctx,TMP_REG_2 ); + +if (mode ==BPF_IND ) { +/* +* __rte_pktmbuf_read() takes a uint32_t off, so a 64-bit off +* that does not fit would be silently truncated. Return 0 if +* out of range; this also catches the negative off routed here +* by the fast path. +*/ +emit_mov_imm (ctx,1 , tmp2,UINT32_MAX ); +emit_cmp (ctx,1 , tmp1, tmp2); +emit_b_cond (ctx,A64_LS ,3 );/* in range: do the call */ +emit_mov_imm (ctx,1 , r0,0 ); +emit_b (ctx, (ctx->program_start + ctx->program_sz) - ctx->idx); + } + +/* arguments of __rte_pktmbuf_read(mbuf, off, len, buf) */ +emit_mov_64 (ctx,A64_R (1 ), tmp1);/* off (held in tmp1) */ +emit_mov_64 (ctx,A64_R (0 ), r6);/* mbuf */ +emit_mov_imm (ctx,0 ,A64_R (2 ), sz);/* len */ +emit_sub_imm_64 (ctx,A64_R (3 ), fp, stack_ofs);/* buf */ + +emit_call (ctx, tmp1, (void *)(uintptr_t )__rte_pktmbuf_read); +emit_return_zero_if_src_zero (ctx,1 , r0); +} + +/* +* Emit code for BPF_LD | BPF_ABS and BPF_LD | BPF_IND packet loads: +* +* off = imm (+ src for BPF_IND) +* if (off >= 0 && mbuf->data_len - off >= sz) -- fast path +* ptr = mbuf->buf_addr + mbuf->data_off + off; +* else -- slow path +* if ((uint64_t)off > UINT32_MAX) +* return 0; +* ptr = __rte_pktmbuf_read(mbuf, off, sz, buf); +* if (ptr == NULL) +* return 0; +* R0 = ntoh(*(size *)ptr); -- common tail +* +* For BPF_ABS the offset is a constant, so the range checks on off are only +* emitted for BPF_IND. +* +* The three blocks are sized in a dry run so the forward branches can be +* resolved, then emitted for real. Both runs see the same immediates, so +* they emit the same instructions; the recorded offsets are only valid +* while that holds, hence the checks. +*/ +static void +emit_ld_mbuf (struct a64_jit_ctx *ctx,uint8_t op,uint8_t src,int32_t imm, +uint32_t stack_ofs) +{ +uint8_t mode =BPF_MODE (op); +uint8_t opsz =BPF_SIZE (op); +uint32_t sz =bpf_size (opsz); +uint32_t ofs[LDMB_OFS_NUM ]; + +/* a constant negative offset is never inside the packet */ +if (mode ==BPF_ABS && imm <0 ) { +emit_mov_imm (ctx,1 ,ebpf_to_a64_reg (ctx,EBPF_REG_0 ),0 ); +emit_b (ctx, (ctx->program_start + ctx->program_sz) - ctx->idx); +return ; + } + +/* seed offsets so the dry-run branches stay in range */ + ofs[LDMB_FAST_OFS ] = ofs[LDMB_SLOW_OFS ] = ofs[LDMB_FIN_OFS ] = ctx->idx; + +/* dry run to record block offsets */ +emit_ldmb_fast_path (ctx, src, mode, sz, imm, ofs); + ofs[LDMB_SLOW_OFS ] = ctx->idx; +emit_ldmb_slow_path (ctx, mode, sz, stack_ofs); + ofs[LDMB_FIN_OFS ] = ctx->idx; +emit_ldmb_fin (ctx, opsz, sz); + +/* rewind and emit for real with resolved offsets */ + ctx->idx = ofs[LDMB_FAST_OFS ]; +emit_ldmb_fast_path (ctx, src, mode, sz, imm, ofs); +RTE_VERIFY (ctx->idx == ofs[LDMB_SLOW_OFS ]); +emit_ldmb_slow_path (ctx, mode, sz, stack_ofs); +RTE_VERIFY (ctx->idx == ofs[LDMB_FIN_OFS ]); +emit_ldmb_fin (ctx, opsz, sz); +} +static void check_program_has_call (struct a64_jit_ctx *ctx,struct rte_bpf *bpf) {
@@ -1145,8 +1313,17 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) op = ins->code;switch (op) { -/* Call imm */ +/* +* BPF_ABS/BPF_IND can fall through to __rte_pktmbuf_read(), +* so they need the call-clobbered register layout as well. +*/ case (BPF_JMP |EBPF_CALL ): +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): ctx->foundcall =1 ;return ; }
@@ -1348,6 +1525,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)emit_mov_imm (ctx,1 , dst, u64); i++;break ; +/* R0 = ntoh(*(size *)(mbuf data + (src) + imm)) */ +case (BPF_LD |BPF_ABS |BPF_B ): +case (BPF_LD |BPF_ABS |BPF_H ): +case (BPF_LD |BPF_ABS |BPF_W ): +case (BPF_LD |BPF_IND |BPF_B ): +case (BPF_LD |BPF_IND |BPF_H ): +case (BPF_LD |BPF_IND |BPF_W ): +emit_ld_mbuf (ctx, op, src, imm, bpf->stack_sz); +break ;/* *(size *)(dst + off) = src */ case (BPF_STX |BPF_MEM |BPF_B ):case (BPF_STX |BPF_MEM |BPF_H ):
Signature · 1 line
2.53.0
PATCH v8 8/10 test/bpf: check that JIT was generated
In reply to Stephen HemmingerAvoid silently ignoring JIT failures. The test cases should all succeed JIT compilation; if not it is a bug in the JIT implementation and should be reported.
Introduce a configuration setting RTE_BPF_JIT_SUPPORTED which is cleaner than using an ARCH specific #ifdef.
--- app/test/test_bpf.c | 8 ++++++++ lib/bpf/meson.build | 2 ++ 2 files changed, 10 insertions(+)
app/test/test_bpf.c +8 -0
@@ -3748,6 +3748,14 @@ run_test(const struct bpf_test *tst) rv, strerror(rv)); } } +#ifdef RTE_BPF_JIT_SUPPORTED + else { +/* a JIT backend exists for this arch, so it must compile */ +printf ("%s@%d: %s: no JIT code generated;\n" , + __func__, __LINE__, tst->name); + ret =-1 ; + } +#endif rte_bpf_destroy (bpf);return ret;
lib/bpf/meson.build +2 -0
@@ -29,8 +29,10 @@ sources = files( ) if arch_subdir == 'x86' and dpdk_conf.get('RTE_ARCH_64') + dpdk_conf.set('RTE_BPF_JIT_SUPPORTED', 1) sources += files('bpf_jit_x86.c') elif dpdk_conf.has('RTE_ARCH_ARM64') + dpdk_conf.set('RTE_BPF_JIT_SUPPORTED', 1) sources += files('bpf_jit_arm64.c') endif
Signature · 1 line
2.53.0
PATCH v8 9/10 test/bpf: check that bpf_convert can be JIT'd
In reply to Stephen HemmingerRun each converted filter through both the interpreter and the JIT and check they agree, catching JIT miscompiles.
test_bpf_filter and test_bpf_match did nearly the same thing: compile, load and run a filter against the dummy packet. Combine them into test_bpf_match, which now builds the packet itself and returns whether the filter matched. Callers run it for both load methods.
The dummy packet is a UDP packet to a fixed destination MAC, source and destination ports, so the filter results are deterministic. None of the sample filters should match it, so assert that; a convert or JIT bug that flips a result is then caught. The destination MAC and source port are chosen so the negative ethernet and port filters do not match, and "port not 53 and not arp" is dropped as it matches any non-ARP packet that lacks port 53.
Reduce log output to make it easier to match which expression might be causing issues.
--- app/test/test_bpf.c | 171 ++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 71 deletions(-)
app/test/test_bpf.c +100 -71
@@ -32,6 +32,7 @@ test_bpf(void)#include <rte_bpf.h> #include <rte_ether.h> #include <rte_ip.h> +#include <rte_udp.h> /* Tests of most simple BPF programs (no instructions, one instruction etc.) */
@@ -4855,11 +4856,13 @@ load_cbpf_program_convert(struct bpf_program *cbpf_program, const char *str)return NULL ; } +#ifdef DEBUG printf ("bpf convert(\"%s\") produced:\n" , str);rte_bpf_dump (stdout, prm->ins, prm->nb_ins);printf ("%s \"%s\"\n" , __func__, str);test_bpf_dump (cbpf_program, prm); +#endif bpf =rte_bpf_load (prm);rte_free (prm);
@@ -4884,18 +4887,65 @@ load_cbpf_program_direct(struct bpf_program *cbpf_program, const char *str __rte }); } +static const load_cbpf_program_t cbpf_program_loaders[] = { + load_cbpf_program_convert, + load_cbpf_program_direct, +}; + +/* Setup Ethernet/IP/UDP headers in a dummy packet buffer for filter tests */ +static void +dummy_ip_prep (void *data,uint16_t plen) +{ +struct { +struct rte_ether_hdr eth_hdr; +struct rte_ipv4_hdr ip_hdr; +struct rte_udp_hdr udp_hdr; + } *hdr = data; + + hdr->eth_hdr = (struct rte_ether_hdr ) { + .dst_addr.addr_bytes = {0x01 ,0x80 ,0xc2 ,0x00 ,0x00 ,0x0e }, + .ether_type =rte_cpu_to_be_16 (RTE_ETHER_TYPE_IPV4 ), + }; + hdr->ip_hdr = (struct rte_ipv4_hdr ) { + .version_ihl =RTE_IPV4_VHL_DEF , + .total_length =rte_cpu_to_be_16 (plen), + .time_to_live =IPDEFTTL , + .next_proto_id =IPPROTO_UDP , + .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), + .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), + }; + hdr->udp_hdr = (struct rte_udp_hdr ) { + .src_port =rte_cpu_to_be_16 (49152 ),/* fixed, avoids filter ports */ + .dst_port =rte_cpu_to_be_16 (9 ),/* discard port */ + .dgram_len =rte_cpu_to_be_16 (plen -sizeof (struct rte_ipv4_hdr )), + .dgram_cksum =0 , + }; +} + +/* +* Compile a pcap filter, load it with the given loader, then run it against +* a standard dummy packet with both the interpreter and (when available) the +* JIT, checking the two agree. +* +* Returns 1 if the filter matched, 0 if it did not, and -1 on any error +* (compile, load, or interpreter/JIT mismatch). +*/ static int -test_bpf_match (pcap_t *pcap,const char *str,struct rte_mbuf *mb, +test_bpf_match (pcap_t *pcap,const char *str,load_cbpf_program_t load_cbpf_program) { +uint8_t tbuf[RTE_MBUF_DEFAULT_BUF_SIZE ]; +const uint32_t plen =100 ;struct bpf_program fcode; -struct rte_bpf *bpf; +struct rte_mbuf mb = {0 }; +struct rte_bpf *bpf =NULL ;int ret =-1 ;uint64_t rc; +printf ("%s '%s'\n" , __func__, str);if (pcap_compile (pcap, &fcode, str,1 ,PCAP_NETMASK_UNKNOWN )) {printf ("%s@%d: pcap_compile(\"%s\") failed: %s;\n" , - __func__, __LINE__, str,pcap_geterr (pcap)); + __func__, __LINE__, str,pcap_geterr (pcap));return -1 ; }
@@ -4903,15 +4953,41 @@ test_bpf_match(pcap_t *pcap, const char *str, struct rte_mbuf *mb,if (bpf ==NULL ) {printf ("%s@%d: failed to load cbpf program for \"%s\", error=%d(%s);\n" , __func__, __LINE__, str, rte_errno,strerror (rte_errno)); +test_bpf_dump (&fcode,NULL ); goto error; } - rc =rte_bpf_exec (bpf, mb); -/* The return code from bpf capture filter is non-zero if matched */ - ret = (rc ==0 ); +dummy_mbuf_prep (&mb, tbuf,sizeof (tbuf), plen); + dummy_ip_prep(rte_pktmbuf_mtod(&mb, void *), plen); + + rc =rte_bpf_exec (bpf, &mb); + +/* Verify the JIT, when available, produces the same result. */ + { +struct rte_bpf_jit jit; + +rte_bpf_get_jit (bpf, &jit); +if (jit.func !=NULL ) { +fflush (stdout); +if (jit.func (&mb) != rc) { +printf ("%s@%d: JIT return code does not match\n" , + __func__, __LINE__); + goto error; + } + } +#ifdef RTE_BPF_JIT_SUPPORTED + else { +printf ("%s@%d: no JIT code generated\n" , + __func__, __LINE__); + goto error; + } +#endif + } + +/* The return code from a bpf capture filter is non-zero if matched. */ + ret = (rc !=0 ); error: -if (bpf) -rte_bpf_destroy (bpf); +rte_bpf_destroy (bpf);pcap_freecode (&fcode);return ret; }
@@ -4920,44 +4996,13 @@ test_bpf_match(pcap_t *pcap, const char *str, struct rte_mbuf *mb,static int test_bpf_filter_sanity (pcap_t *pcap) { -static const load_cbpf_program_t cbpf_program_loaders[] = { - load_cbpf_program_convert, - load_cbpf_program_direct, - }; - -const uint32_t plen =100 ; -struct rte_mbuf mb, *m; -uint8_t tbuf[RTE_MBUF_DEFAULT_BUF_SIZE ]; -struct { -struct rte_ether_hdr eth_hdr; -struct rte_ipv4_hdr ip_hdr; - } *hdr; - -memset (&mb,0 ,sizeof (mb)); -dummy_mbuf_prep (&mb, tbuf,sizeof (tbuf), plen); - m = &mb; - - hdr =rte_pktmbuf_mtod (m,typeof (hdr)); - hdr->eth_hdr = (struct rte_ether_hdr ) { - .dst_addr.addr_bytes = {0xff ,0xff ,0xff ,0xff ,0xff ,0xff }, - .ether_type =rte_cpu_to_be_16 (RTE_ETHER_TYPE_IPV4 ), - }; - hdr->ip_hdr = (struct rte_ipv4_hdr ) { - .version_ihl =RTE_IPV4_VHL_DEF , - .total_length =rte_cpu_to_be_16 (plen), - .time_to_live =IPDEFTTL , - .next_proto_id =IPPROTO_RAW , - .src_addr =rte_cpu_to_be_32 (RTE_IPV4_LOOPBACK ), - .dst_addr =rte_cpu_to_be_32 (RTE_IPV4_BROADCAST ), - }; - -for (int li =0 ; li !=RTE_DIM (cbpf_program_loaders); ++li) { -if (test_bpf_match (pcap,"ip" , m, cbpf_program_loaders[li]) !=0 ) { +for (unsigned int li =0 ; li !=RTE_DIM (cbpf_program_loaders); ++li) { +if (test_bpf_match (pcap,"ip" , cbpf_program_loaders[li]) !=1 ) {printf ("%s@%d: filter \"ip\" doesn't match test data\n" , __func__, __LINE__);return -1 ; } -if (test_bpf_match (pcap,"not ip" , m, cbpf_program_loaders[li]) ==0 ) { +if (test_bpf_match (pcap,"not ip" , cbpf_program_loaders[li]) !=0 ) {printf ("%s@%d: filter \"not ip\" does match test data\n" , __func__, __LINE__);return -1 ;
@@ -4981,7 +5026,6 @@ static const char * const sample_filters[] = {"port 53" ,"host 192.0.2.1 and not (port 80 or port 25)" ,"host 2001:4b98:db0::8 and not port 80 and not port 25" , -"port not 53 and not arp" ,"(tcp[0:2] > 1500 and tcp[0:2] < 1550) or (tcp[2:2] > 1500 and tcp[2:2] < 1550)" ,"ether proto 0x888e" , "ether[0] & 1 = 0 and ip[16] >= 224",
@@ -5008,35 +5052,10 @@ static const char * const sample_filters[] = { "or host 192.0.2.1 or host 192.0.2.100 or host 192.0.2.200"), }; -static int -test_bpf_filter (pcap_t *pcap,const char *s,load_cbpf_program_t load_cbpf_program) -{ -struct bpf_program fcode; -struct rte_bpf *bpf; - -if (pcap_compile (pcap, &fcode, s,1 ,PCAP_NETMASK_UNKNOWN )) { -printf ("%s@%d: pcap_compile(\"%s\") failed: %s;\n" , - __func__, __LINE__, s,pcap_geterr (pcap)); -return -1 ; - } - - bpf =load_cbpf_program (&fcode, s); -if (bpf ==NULL ) { -printf ("%s@%d: failed to load cbpf program for \"%s\", error=%d(%s);\n" , - __func__, __LINE__, s, rte_errno,strerror (rte_errno)); -test_bpf_dump (&fcode,NULL ); - } - -rte_bpf_destroy (bpf); - -pcap_freecode (&fcode); -return (bpf ==NULL ) ?-1 :0 ; -} -static int test_bpf_convert (void ) { -unsigned int i; +unsigned int i, li;pcap_t *pcap;int rc;
@@ -5048,8 +5067,18 @@ test_bpf_convert(void) rc =test_bpf_filter_sanity (pcap);for (i =0 ; i <RTE_DIM (sample_filters); i++) { - rc |=test_bpf_filter (pcap, sample_filters[i], load_cbpf_program_convert); - rc |=test_bpf_filter (pcap, sample_filters[i], load_cbpf_program_direct); +for (li =0 ; li <RTE_DIM (cbpf_program_loaders); li++) { +int m =test_bpf_match (pcap, sample_filters[i], + cbpf_program_loaders[li]); + +/* None of the sample filters match the dummy packet. */ +if (m !=0 ) { +if (m >0 ) +printf ("%s@%d: filter \"%s\" unexpectedly matched\n" , + __func__, __LINE__, sample_filters[i]); + rc =-1 ; + } + } }pcap_close (pcap);
Signature · 1 line
2.53.0
PATCH v8 10/10 bpf: fix uninitialized warning
In reply to Stephen HemmingerCoverity complains uninitialized use of structure.
Coverity issue: 504611
--- lib/bpf/bpf_validate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bpf/bpf_validate.c +1 -1
@@ -659,7 +659,7 @@ eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)static void eval_add (struct bpf_reg_val *rd,const struct bpf_reg_val *rs,uint64_t msk) { -struct bpf_reg_val rs_buf; +struct bpf_reg_val rs_buf = { .v.type =RTE_BPF_ARG_RAW };struct bpf_reg_val rv;if (RTE_BPF_ARG_PTR_TYPE (rs->v.type) !=0 ) {
Signature · 1 line
2.53.0
Thomas Monjalon
AppliedRe: [PATCH v8 00/10] bpf: JIT and interpreter fixes
In reply to Stephen HemmingerQuoted 14 lines
Christophe Fontaine (1):
bpf/arm64: fix offset type to allow a negative jumpStephen Hemminger (9):
bpf/x86: fix JIT encoding of fixed-width immediates test/bpf: add JSET test with small immediate
bpf: mask shift count in interpreter per RFC 9669 bpf/arm64: mask shift count per RFC 9669
test/bpf: add test for large shift
Subject: [PATCH 07/10] bpf/arm64: add BPF_ABS/BPF_IND packet load
support
test/bpf: check that JIT was generated
test/bpf: check thatbpf_convertcan be JIT'd
bpf: fix uninitialized warning
Applied, thanks.