bpf,x64 Emit IMUL instead of MUL for x86-64
Subsystems (5)
PATCH 1/1 bpf,x64 Emit IMUL instead of MUL for x86-64
IMUL allows for multiple operands and saving and storing rax/rdx is no longer needed. Signedness of the operands doesn't matter here because the we only keep the lower 32/64 bit of the product for 32/64 bit multiplications.
Signed-off-by Jie Meng <[email protected]>
--- arch/x86/net/bpf_jit_comp.c | 53 ++++++++++------------ tools/testing/selftests/bpf/verifier/jit.c | 16 +++++++ 2 files changed, 39 insertions(+), 30 deletions(-)
arch/x86/net/bpf_jit_comp.c +23 -30
@@ -1070,41 +1070,34 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,break ;case BPF_ALU |BPF_MUL |BPF_K : -case BPF_ALU |BPF_MUL |BPF_X :case BPF_ALU64 |BPF_MUL |BPF_K : -case BPF_ALU64 |BPF_MUL |BPF_X : - { -bool is64 =BPF_CLASS (insn->code) ==BPF_ALU64 ; +if (BPF_CLASS (insn->code) ==BPF_ALU64 ) +EMIT1 (add_2mod (0x48 , dst_reg, dst_reg)); +else if (is_ereg (dst_reg)) +EMIT1 (add_2mod (0x40 , dst_reg, dst_reg)); -if (dst_reg !=BPF_REG_0 ) -EMIT1 (0x50 );/* push rax */ -if (dst_reg !=BPF_REG_3 ) -EMIT1 (0x52 );/* push rdx */ +if (is_imm8 (imm32)) +/* imul dst_reg, dst_reg, imm8 */ +EMIT3 (0x6B ,add_2reg (0xC0 , dst_reg, dst_reg), + imm32); +else +/* imul dst_reg, dst_reg, imm32 */ +EMIT2_off32 (0x69 , +add_2reg (0xC0 , dst_reg, dst_reg), + imm32); +break ; -/* mov r11, dst_reg */ -EMIT_mov (AUX_REG , dst_reg); +case BPF_ALU |BPF_MUL |BPF_X : +case BPF_ALU64 |BPF_MUL |BPF_X : +if (BPF_CLASS (insn->code) ==BPF_ALU64 ) +EMIT1 (add_2mod (0x48 , dst_reg, src_reg)); +else if (is_ereg (dst_reg) ||is_ereg (src_reg)) +EMIT1 (add_2mod (0x40 , dst_reg, src_reg)); -if (BPF_SRC (insn->code) ==BPF_X ) -emit_mov_reg (&prog, is64,BPF_REG_0 , src_reg); -else -emit_mov_imm32 (&prog, is64,BPF_REG_0 , imm32); - -if (is64) -EMIT1 (add_1mod (0x48 ,AUX_REG )); -else if (is_ereg (AUX_REG )) -EMIT1 (add_1mod (0x40 ,AUX_REG )); -/* mul(q) r11 */ -EMIT2 (0xF7 ,add_1reg (0xE0 ,AUX_REG )); - -if (dst_reg !=BPF_REG_3 ) -EMIT1 (0x5A );/* pop rdx */ -if (dst_reg !=BPF_REG_0 ) { -/* mov dst_reg, rax */ -EMIT_mov (dst_reg,BPF_REG_0 ); -EMIT1 (0x58 );/* pop rax */ - } +/* imul dst_reg, src_reg */ +EMIT3 (0x0F ,0xAF ,add_2reg (0xC0 , src_reg, dst_reg));break ; - } +/* Shifts */ case BPF_ALU |BPF_LSH |BPF_K :case BPF_ALU |BPF_RSH |BPF_K :
tools/testing/selftests/bpf/verifier/jit.c +16 -0
@@ -62,6 +62,11 @@BPF_JMP_REG (BPF_JEQ ,BPF_REG_3 ,BPF_REG_2 ,2 ),BPF_MOV64_IMM (BPF_REG_0 ,1 ),BPF_EXIT_INSN (), +BPF_LD_IMM64 (BPF_REG_3 ,0xfefefeULL ), +BPF_ALU64_IMM (BPF_MUL ,BPF_REG_3 ,0xefefef ), +BPF_JMP_REG (BPF_JEQ ,BPF_REG_3 ,BPF_REG_2 ,2 ), +BPF_MOV64_IMM (BPF_REG_0 ,1 ), +BPF_EXIT_INSN (),BPF_MOV32_REG (BPF_REG_2 ,BPF_REG_2 ),BPF_LD_IMM64 (BPF_REG_0 ,0xfefefeULL ), BPF_ALU32_REG(BPF_MUL, BPF_REG_0, BPF_REG_1),
@@ -73,6 +78,17 @@BPF_JMP_REG (BPF_JEQ ,BPF_REG_3 ,BPF_REG_2 ,2 ),BPF_MOV64_IMM (BPF_REG_0 ,1 ),BPF_EXIT_INSN (), +BPF_LD_IMM64 (BPF_REG_3 ,0xfefefeULL ), +BPF_ALU32_IMM (BPF_MUL ,BPF_REG_3 ,0xefefef ), +BPF_JMP_REG (BPF_JEQ ,BPF_REG_3 ,BPF_REG_2 ,2 ), +BPF_MOV64_IMM (BPF_REG_0 ,1 ), +BPF_EXIT_INSN (), +BPF_LD_IMM64 (BPF_REG_0 ,0xfefefeULL ), +BPF_LD_IMM64 (BPF_REG_2 ,0x2ad4d4aaULL ), +BPF_ALU32_IMM (BPF_MUL ,BPF_REG_0 ,0x2b ), +BPF_JMP_REG (BPF_JEQ ,BPF_REG_0 ,BPF_REG_2 ,2 ), +BPF_MOV64_IMM (BPF_REG_0 ,1 ), +BPF_EXIT_INSN (),BPF_LD_IMM64 (BPF_REG_0 ,0x952a7bbcULL ),BPF_LD_IMM64 (BPF_REG_1 ,0xfefefeULL ), BPF_LD_IMM64(BPF_REG_2, 0xeeff0d413122ULL),
Signature · 1 line
2.30.2
Quoted 8 lines · Jie Meng
IMUL allows for multiple operands and saving and storing rax/rdx is no longer needed. Signedness of the operands doesn't matter here because the we only keep the lower 32/64 bit of the product for 32/64 bit multiplications.
Signed-off-by Jie Meng <[email protected]>---
This change breaks selftests ([0]), please fix.
[0] https://github.com/kernel-patches/bpf/pull/1746/checks?check_run_id=3550847581
Quoted 4 lines
arch/x86/net/bpf_jit_comp.c | 53 ++++++++++------------ tools/testing/selftests/bpf/verifier/jit.c | 16 +++++++ 2 files changed, 39 insertions(+), 30 deletions(-)
[...]