bpf, x86: Add support for signed arena loads
Subsystems (4)
Kumar Kartikeya Dwivedi
maintainerPATCH 1/1 bpf, x86: Add support for signed arena loads
Currently, signed load instructions into arena memory are unsupported. The compiler is free to generate these, and on GCC-14 we see a corresponding error when it happens. The hurdle in supporting them is deciding which unused opcode to use to mark them for the JIT's own consumption. After much thinking, it appears 0xc0 / BPF_NOSPEC can be combined with load instructions to identify signed arena loads. Use this to recognize and JIT them appropriately, and remove the verifier side limitation on the program if the JIT supports them.
--- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++++++++++++++++++- include/linux/filter.h | 4 ++++ kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 10 +++++++--- 4 files changed, 55 insertions(+), 4 deletions(-)
arch/x86/net/bpf_jit_comp.c +39 -1
@@ -1146,11 +1146,38 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i *pprog = prog; } +static void emit_ldsx_index (u8 **pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) +{ +u8 *prog = *pprog; + +switch (size) { +case BPF_B : +/* movsx rax, byte ptr [rax + r12 + off] */ +EMIT3 (add_3mod (0x40 , src_reg, dst_reg, index_reg),0x0F ,0xBE ); +break ; +case BPF_H : +/* movsx rax, word ptr [rax + r12 + off] */ +EMIT3 (add_3mod (0x40 , src_reg, dst_reg, index_reg),0x0F ,0xBF ); +break ; +case BPF_W : +/* movsx rax, dword ptr [rax + r12 + off] */ +EMIT2 (add_3mod (0x40 , src_reg, dst_reg, index_reg),0x63 ); +break ; + } +emit_insn_suffix_SIB (&prog, src_reg, dst_reg, index_reg, off); + *pprog = prog; +} +static void emit_ldx_r12 (u8 **pprog,u32 size,u32 dst_reg,u32 src_reg,int off) {emit_ldx_index (pprog, size, dst_reg, src_reg,X86_REG_R12 , off); } +static void emit_ldsx_r12 (u8 **prog,u32 size,u32 dst_reg,u32 src_reg,int off) +{ +emit_ldsx_index (prog, size, dst_reg, src_reg,X86_REG_R12 , off); +} +/* STX: *(u8*)(dst_reg + off) = src_reg */ static void emit_stx (u8 **pprog,u32 size,u32 dst_reg,u32 src_reg,int off) {
@@ -2010,13 +2037,19 @@ st: if (is_imm8(insn->off))case BPF_LDX |BPF_PROBE_MEM32 |BPF_H :case BPF_LDX |BPF_PROBE_MEM32 |BPF_W :case BPF_LDX |BPF_PROBE_MEM32 |BPF_DW : +case BPF_LDX |BPF_PROBE_MEM32SX |BPF_B : +case BPF_LDX |BPF_PROBE_MEM32SX |BPF_H : +case BPF_LDX |BPF_PROBE_MEM32SX |BPF_W :case BPF_STX |BPF_PROBE_MEM32 |BPF_B :case BPF_STX |BPF_PROBE_MEM32 |BPF_H :case BPF_STX |BPF_PROBE_MEM32 |BPF_W :case BPF_STX |BPF_PROBE_MEM32 |BPF_DW : start_of_ldx = prog;if (BPF_CLASS (insn->code) ==BPF_LDX ) -emit_ldx_r12 (&prog,BPF_SIZE (insn->code), dst_reg, src_reg, insn->off); +if (BPF_MODE (insn->code) ==BPF_PROBE_MEM32SX ) +emit_ldsx_r12 (&prog,BPF_SIZE (insn->code), dst_reg, src_reg, insn->off); +else +emit_ldx_r12 (&prog,BPF_SIZE (insn->code), dst_reg, src_reg, insn->off);else emit_stx_r12 (&prog,BPF_SIZE (insn->code), dst_reg, src_reg, insn->off); populate_extable:
@@ -3875,3 +3908,8 @@ bool bpf_jit_supports_timed_may_goto(void) {return true; } + +bool bpf_jit_supports_signed_arena_load (void ) +{ +return true; +}
include/linux/filter.h +4 -0
@@ -78,6 +78,9 @@ struct ctl_table_header;/* unused opcode to mark special atomic instruction */ #define BPF_PROBE_ATOMIC 0xe0 +/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */ +#define BPF_PROBE_MEM32SX 0xc0 +/* unused opcode to mark call to interpreter with arguments */ #define BPF_CALL_ARGS 0xe0
@@ -1138,6 +1141,7 @@ bool bpf_jit_supports_arena(void);bool bpf_jit_supports_insn (struct bpf_insn *insn,bool in_arena);bool bpf_jit_supports_private_stack (void );bool bpf_jit_supports_timed_may_goto (void ); +bool bpf_jit_supports_signed_arena_load (void );u64 bpf_arch_uaddress_limit (void );void arch_bpf_stack_walk (bool (*consume_fn)(void *cookie,u64 ip,u64 sp,u64 bp),void *cookie);u64 arch_bpf_timed_may_goto (void );
kernel/bpf/core.c +5 -0
@@ -3076,6 +3076,11 @@ bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)return false; } +bool __weak bpf_jit_supports_signed_arena_load (void ) +{ +return false; +} +u64 __weak bpf_arch_uaddress_limit (void ) {#if defined(CONFIG_64BIT ) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE )
kernel/bpf/verifier.c +7 -3
@@ -20957,10 +20957,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)continue ;case PTR_TO_ARENA :if (BPF_MODE (insn->code) ==BPF_MEMSX ) { -verbose (env,"sign extending loads from arena are not supported yet\n" ); -return -EOPNOTSUPP ; +if (!bpf_jit_supports_signed_arena_load ()) { +verbose (env,"sign extending loads from arena are not supported yet\n" ); +return -EOPNOTSUPP ; + } + insn->code =BPF_CLASS (insn->code) |BPF_PROBE_MEM32SX |BPF_SIZE (insn->code); + }else { + insn->code =BPF_CLASS (insn->code) |BPF_PROBE_MEM32 |BPF_SIZE (insn->code); } - insn->code =BPF_CLASS (insn->code) |BPF_PROBE_MEM32 |BPF_SIZE (insn->code); env->prog->aux->num_exentries++;continue ;default :
--
2.47.1
include/linux/filter.h struct ctl_table_header; · line 82
Quoted 4 lines · Kumar Kartikeya Dwivedi
include/linux/filter.h +2
-0+/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */ +#define BPF_PROBE_MEM32SX 0xc0
lgtm. should work.
Quoted 17 lines
include/linux/filter.h +2
-0+/* unused opcode to mark call to interpreter with arguments */ #define BPF_CALL_ARGS 0xe0@@ -1138,6 +1141,7 @@ bool bpf_jit_supports_arena(void);bool bpf_jit_supports_insn (struct bpf_insn *insn,bool in_arena);bool bpf_jit_supports_private_stack (void );bool bpf_jit_supports_timed_may_goto (void ); +bool bpf_jit_supports_signed_arena_load (void );u64 bpf_arch_uaddress_limit (void );void arch_bpf_stack_walk (bool (*consume_fn)(void *cookie,u64 ip,u64 sp,u64 bp),void *cookie);u64 arch_bpf_timed_may_goto (void );kernel/bpf/core.c +0
-0
kernel/bpf/core.c bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) · line 3082
Quoted 7 lines
kernel/bpf/core.c +4
-0@@ -3076,6 +3076,11 @@ bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)return false; } +bool __weak bpf_jit_supports_signed_arena_load (void ) +{ +return false; +}
Instead of introducing a new weak function, let's use bpf_jit_supports_insn() ? We were planning to convert other weak functions to it,
but the work wasn't done.
At least let's not create more tech debt to clean up later.