arm64, bpf: add internal-only MOV instruction to resolve per-CPU addrs
Subsystems (3)
PATCH 1/1 arm64, bpf: add internal-only MOV instruction to resolve per-CPU addrs
Support an instruction for resolving absolute addresses of per-CPU data from their per-CPU offsets. This instruction is internal-only and users are not allowed to use them directly. They will only be used for internal inlining optimizations for now between BPF verifier and BPF JITs.
Since commit 7158627686f0 ("arm64: percpu: implement optimised pcpu access using tpidr_el1"), the per-cpu offset for the CPU is stored in the tpidr_el1/2 register of that CPU.
To support this BPF instruction in the ARM64 JIT, the following ARM64 instructions are emitted:
mov dst, src // Move src to dst, if src != dst
mrs tmp, tpidr_el1/2 // Move per-cpu offset of the current cpu in tmp.
add dst, dst, tmp // Add the per cpu offset to the dst.
If CONFIG_SMP is not defined, then nothing is emitted if src == dst, and mov dst, src is emitted if dst != src.
To measure the performance improvement provided by this change, the benchmark in [1] was used:
Before:
glob-arr-inc : 23.597 ± 0.012M/s arr-inc : 23.173 ± 0.019M/s hash-inc : 12.186 ± 0.028M/s
After:
glob-arr-inc : 23.819 ± 0.034M/s arr-inc : 23.285 ± 0.017M/s hash-inc : 12.419 ± 0.011M/s
[1] https://github.com/anakryiko/linux/commit/8dec900975ef
--- arch/arm64/include/asm/insn.h | 7 +++++++ arch/arm64/lib/insn.c | 11 +++++++++++ arch/arm64/net/bpf_jit.h | 6 ++++++ arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++++++ 4 files changed, 40 insertions(+)
arch/arm64/include/asm/insn.h +7 -0
@@ -135,6 +135,11 @@ enum aarch64_insn_special_register {AARCH64_INSN_SPCLREG_SP_EL2 =0xF210 }; +enum aarch64_insn_system_register { +AARCH64_INSN_SYSREG_TPIDR_EL1 =0xC684 , +AARCH64_INSN_SYSREG_TPIDR_EL2 =0xE682 , +}; +enum aarch64_insn_variant {AARCH64_INSN_VARIANT_32BIT ,AARCH64_INSN_VARIANT_64BIT
@@ -686,6 +691,8 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, }#endif u32 aarch64_insn_gen_dmb (enum aarch64_insn_mb_type type); +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg);s32 aarch64_get_branch_offset (u32 insn);u32 aarch64_set_branch_offset (u32 insn,s32 offset);
arch/arm64/lib/insn.c +11 -0
@@ -1515,3 +1515,14 @@ u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)return insn; } + +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg) +{ +u32 insn =aarch64_insn_get_mrs_value (); + + insn &= ~GENMASK (19 ,0 ); + insn |= sysreg <<5 ; +return aarch64_insn_encode_register (AARCH64_INSN_REGTYPE_RT , + insn, result); +}
arch/arm64/net/bpf_jit.h +6 -0
@@ -297,4 +297,10 @@#define A64_ADR (Rd, offset) \ aarch64_insn_gen_adr(0, offset, Rd, AARCH64_INSN_ADR_TYPE_ADR) +/* MRS */ +#define A64_MRS_TPIDR_EL1 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL1) +#define A64_MRS_TPIDR_EL2 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL2) +#endif /* _BPF_JIT_H */
arch/arm64/net/bpf_jit_comp.c +16 -0
@@ -877,6 +877,17 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,emit (A64_ORR (1 , tmp, dst, tmp), ctx);emit (A64_MOV (1 , dst, tmp), ctx);break ; + }else if (insn_is_mov_percpu_addr (insn)) { +if (dst != src) +emit (A64_MOV (1 , dst, src), ctx); +#ifdef CONFIG_SMP +if (cpus_have_cap (ARM64_HAS_VIRT_HOST_EXTN )) +emit (A64_MRS_TPIDR_EL2 (tmp), ctx); +else +emit (A64_MRS_TPIDR_EL1 (tmp), ctx); +emit (A64_ADD (1 , dst, dst, tmp), ctx); +#endif +break ; }switch (insn->off) {case 0 :
@@ -2527,6 +2538,11 @@ bool bpf_jit_supports_arena(void)return true; } +bool bpf_jit_supports_percpu_insn (void ) +{ +return true; +} +void bpf_jit_free (struct bpf_prog *prog) {if (prog->jited) {
Signature · 1 line
2.40.1
Quoted 29 lines · Puranjay Mohan
Support an instruction for resolving absolute addresses of per-CPU data from their per-CPU offsets. This instruction is internal-only and users are not allowed to use them directly. They will only be used for internal inlining optimizations for now between BPF verifier and BPF JITs.
Since commit
7158627686f0("arm64: percpu: implement optimised pcpu access usingtpidr_el1"), the per-cpu offset for the CPU is stored in the tpidr_el1/2 register of that CPU.To support this BPF instruction in the ARM64 JIT, the following ARM64 instructions are emitted:
mov dst, src // Move src to dst, if src != dst mrs tmp, tpidr_el1/2 // Move per-cpu offset of the current cpu in tmp. add dst, dst, tmp // Add the per cpu offset to the dst.If
CONFIG_SMPis not defined, then nothing is emitted if src == dst, and mov dst, src is emitted if dst != src.To measure the performance improvement provided by this change, the benchmark in [1] was used:
Before:
glob-arr-inc : 23.597 ± 0.012M/s arr-inc : 23.173 ± 0.019M/s hash-inc : 12.186 ± 0.028M/s
Quoted 6 lines
After:
glob-arr-inc : 23.819 ± 0.034M/s arr-inc : 23.285 ± 0.017M/s hash-inc : 12.419 ± 0.011M/s
You don't see as big of a gain, because bpf_get_smp_processor_id() is not inlined yet on arm64.
But even without it I expected bigger gains.
Could you do 'perf report' before/after ?
Just want to see what's on top.
Quoted 105 lines
Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/include/asm/insn.h | 7 +++++++ arch/arm64/lib/insn.c | 11 +++++++++++ arch/arm64/net/bpf_jit.h | 6 ++++++ arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++++++ 4 files changed, 40 insertions(+)arch/arm64/include/asm/insn.h +7
-0@@ -135,6 +135,11 @@ enum aarch64_insn_special_register {AARCH64_INSN_SPCLREG_SP_EL2 =0xF210 }; +enum aarch64_insn_system_register { +AARCH64_INSN_SYSREG_TPIDR_EL1 =0xC684 , +AARCH64_INSN_SYSREG_TPIDR_EL2 =0xE682 , +}; +enum aarch64_insn_variant {AARCH64_INSN_VARIANT_32BIT ,AARCH64_INSN_VARIANT_64BIT @@ -686,6 +691,8 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, }#endif u32 aarch64_insn_gen_dmb (enum aarch64_insn_mb_type type); +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg);s32 aarch64_get_branch_offset (u32 insn);u32 aarch64_set_branch_offset (u32 insn,s32 offset);arch/arm64/lib/insn.c +11
-0@@ -1515,3 +1515,14 @@ u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)return insn; } + +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg) +{ +u32 insn =aarch64_insn_get_mrs_value (); + + insn &= ~GENMASK (19 ,0 ); + insn |= sysreg <<5 ; +return aarch64_insn_encode_register (AARCH64_INSN_REGTYPE_RT , + insn, result); +}arch/arm64/net/bpf_jit.h +6
-0@@ -297,4 +297,10 @@#define A64_ADR (Rd, offset) \ aarch64_insn_gen_adr(0, offset, Rd, AARCH64_INSN_ADR_TYPE_ADR) +/* MRS */ +#define A64_MRS_TPIDR_EL1 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL1) +#define A64_MRS_TPIDR_EL2 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL2) +#endif /* _BPF_JIT_H */ arch/arm64/net/bpf_jit_comp.c +16
-1@@ -877,6 +877,17 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,emit (A64_ORR (1 , tmp, dst, tmp), ctx);emit (A64_MOV (1 , dst, tmp), ctx);break ; + }else if (insn_is_mov_percpu_addr (insn)) { +if (dst != src) +emit (A64_MOV (1 , dst, src), ctx); +#ifdef CONFIG_SMP +if (cpus_have_cap (ARM64_HAS_VIRT_HOST_EXTN )) +emit (A64_MRS_TPIDR_EL2 (tmp), ctx); +else +emit (A64_MRS_TPIDR_EL1 (tmp), ctx); +emit (A64_ADD (1 , dst, dst, tmp), ctx); +#endif +break ; }switch (insn->off) {case 0 :@@ -2527,6 +2538,11 @@ bool bpf_jit_supports_arena(void)return true; } +bool bpf_jit_supports_percpu_insn (void ) +{ +return true; +} +void bpf_jit_free (struct bpf_prog *prog) {if (prog->jited) { --2.40.1
Quoted 36 lines · Alexei Starovoitov
Quoted 34 lines · Puranjay Mohan
Support an instruction for resolving absolute addresses of per-CPU data from their per-CPU offsets. This instruction is internal-only and users are not allowed to use them directly. They will only be used for internal inlining optimizations for now between BPF verifier and BPF JITs.
Since commit
7158627686f0("arm64: percpu: implement optimised pcpu access usingtpidr_el1"), the per-cpu offset for the CPU is stored in the tpidr_el1/2 register of that CPU.To support this BPF instruction in the ARM64 JIT, the following ARM64 instructions are emitted:
mov dst, src // Move src to dst, if src != dst mrs tmp, tpidr_el1/2 // Move per-cpu offset of the current cpu in tmp. add dst, dst, tmp // Add the per cpu offset to the dst.If
CONFIG_SMPis not defined, then nothing is emitted if src == dst, and mov dst, src is emitted if dst != src.To measure the performance improvement provided by this change, the benchmark in [1] was used:
Before:
glob-arr-inc : 23.597 ± 0.012M/s arr-inc : 23.173 ± 0.019M/s hash-inc : 12.186 ± 0.028M/sAfter:
glob-arr-inc : 23.819 ± 0.034M/s arr-inc : 23.285 ± 0.017M/s hash-inc : 12.419 ± 0.011M/s
Quoted 5 lines
Quoted 1 line
You don't see as big of a gain, because
bpf_get_smp_processor_id()is not inlined yet on arm64.
yep, would be nice to add ARM64 and RISC-V support there as well. Though it feels that supporting this in BPF JIT directly might be actually easier for RISC-V/ARM64, not sure?
Quoted 3 lines
But even without it I expected bigger gains.
Could you do 'perf report' before/after ?
Just want to see what's on top.
I also did bpftool p d x id <progid> and bpftool p d j id <progid> to validate expected inlined BPF instructions and jitted code. So it might be a good idea to do that as well.
Either way, thanks for working on this!
Quoted 10 lines
Quoted 10 lines
Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/include/asm/insn.h | 7 +++++++ arch/arm64/lib/insn.c | 11 +++++++++++ arch/arm64/net/bpf_jit.h | 6 ++++++ arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++++++ 4 files changed, 40 insertions(+)
[...]
Will Deacon
maintainerQuoted 48 lines · Puranjay Mohan
Support an instruction for resolving absolute addresses of per-CPU data from their per-CPU offsets. This instruction is internal-only and users are not allowed to use them directly. They will only be used for internal inlining optimizations for now between BPF verifier and BPF JITs.
Since commit
7158627686f0("arm64: percpu: implement optimised pcpu access usingtpidr_el1"), the per-cpu offset for the CPU is stored in the tpidr_el1/2 register of that CPU.To support this BPF instruction in the ARM64 JIT, the following ARM64 instructions are emitted:
mov dst, src // Move src to dst, if src != dst
mrs tmp, tpidr_el1/2 // Move per-cpu offset of the current cpu in tmp.
add dst, dst, tmp // Add the per cpu offset to the dst.If
CONFIG_SMPis not defined, then nothing is emitted if src == dst, and mov dst, src is emitted if dst != src.To measure the performance improvement provided by this change, the benchmark in [1] was used:
Before:
glob-arr-inc : 23.597 ± 0.012M/s arr-inc : 23.173 ± 0.019M/s hash-inc : 12.186 ± 0.028M/sAfter:
glob-arr-inc : 23.819 ± 0.034M/s arr-inc : 23.285 ± 0.017M/s hash-inc : 12.419 ± 0.011M/s[1] https://github.com/anakryiko/linux/commit/8dec900975ef
Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/include/asm/insn.h | 7 +++++++ arch/arm64/lib/insn.c | 11 +++++++++++ arch/arm64/net/bpf_jit.h | 6 ++++++ arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++++++ 4 files changed, 40 insertions(+)arch/arm64/include/asm/insn.h +0
-0
arch/arm64/include/asm/insn.h enum aarch64_insn_special_register { · line 141
Quoted 7 lines
arch/arm64/include/asm/insn.h +4
-0@@ -135,6 +135,11 @@ enum aarch64_insn_special_register { AARCH64_INSN_SPCLREG_SP_EL2 = 0xF210 }; +enum aarch64_insn_system_register { +AARCH64_INSN_SYSREG_TPIDR_EL1 =0xC684 , +AARCH64_INSN_SYSREG_TPIDR_EL2 =0xE682 , +};
I think these constants should have bit 15 as 0...
Quoted 19 lines
arch/arm64/include/asm/insn.h +3
-0+enum aarch64_insn_variant {AARCH64_INSN_VARIANT_32BIT ,AARCH64_INSN_VARIANT_64BIT @@ -686,6 +691,8 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, }#endif u32 aarch64_insn_gen_dmb (enum aarch64_insn_mb_type type); +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg);s32 aarch64_get_branch_offset (u32 insn);u32 aarch64_set_branch_offset (u32 insn,s32 offset);arch/arm64/lib/insn.c +0
-0@@ -1515,3 +1515,14 @@ u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
arch/arm64/lib/insn.c u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) · line 1525
Quoted 10 lines
arch/arm64/lib/insn.c +8
-0return insn; } + +u32 aarch64_insn_gen_mrs (enum aarch64_insn_register result, +enum aarch64_insn_system_register sysreg) +{ +u32 insn =aarch64_insn_get_mrs_value (); + + insn &= ~GENMASK (19 ,0 ); + insn |= sysreg <<5 ;
... otherwise you're shifting into the opcode bits at the top. It works out because bit 20 is 1, but I think it would be better to rework your aarch64_insn_system_register values.
Quoted 23 lines
arch/arm64/lib/insn.c +3
-0+return aarch64_insn_encode_register (AARCH64_INSN_REGTYPE_RT , + insn, result); +}arch/arm64/net/bpf_jit.h +6
-0@@ -297,4 +297,10 @@#define A64_ADR (Rd, offset) \ aarch64_insn_gen_adr(0, offset, Rd, AARCH64_INSN_ADR_TYPE_ADR) +/* MRS */ +#define A64_MRS_TPIDR_EL1 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL1) +#define A64_MRS_TPIDR_EL2 (Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL2) +#endif /* _BPF_JIT_H */ arch/arm64/net/bpf_jit_comp.c +0
-0
arch/arm64/net/bpf_jit_comp.c static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, · line 883
Quoted 7 lines
arch/arm64/net/bpf_jit_comp.c +4
-0@@ -877,6 +877,17 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,emit (A64_ORR (1 , tmp, dst, tmp), ctx);emit (A64_MOV (1 , dst, tmp), ctx);break ; + }else if (insn_is_mov_percpu_addr (insn)) { +if (dst != src) +emit (A64_MOV (1 , dst, src), ctx); +#ifdef CONFIG_SMP
CONFIG_SMP is always 'y' on arm64.
Will