Optimize jump label implementation for ARM64
Subsystems (2)
v8 cover Optimize jump label implementation for ARM64
This patchset tries to optimize arch specfic jump label implementation for ARM64 by dynamic kernel text patching.
To enable this feature, your toolchain must support "asm goto" extension and "%c" constraint extesion.
It has been tested on ARM Fast mode and a real hardware platform.
V7->V8:
1) Minor syntax fix
2) include <asm/insn.h> into <asm/jump_label.h> to avoid duplicated definitions
V6->V7:
1) Minor format clean up
2) Remove smp_mb() pair
V5->V6:
1) optimize stop_machine() related code
2) simplify aarch64_insn_patch_text_nosync() interface
3) refine comments
V4->V5:
1) rework the stop_machine() related code
2) rework aarch64_insn_gen_nop()
V3->V4:
1) resolve a race condition in kernel text patching
2) address other review comments
V2->V3:
1) fix a bug in comparing signed and unsigned values
2) detect big endian by checking __AARCH64EB__
V1->V2: address review comments of V1
1) refine comments
2) add a new interface to always synchronize with stop_machine() when patching code
3) handle endian issue when patching code
Jiang Liu (7):
arm64: introduce basic aarch64 instruction decoding helpers
arm64: introduce interfaces to hotpatch kernel and module code
arm64: move encode_insn_immediate() from module.c to insn.c
arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64, jump label: detect %c support for ARM64
arm64, jump label: optimize jump label implementation
jump_label: use defined macros instead of hard-coding for better
readability
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/insn.h | 108 +++++++++++++ arch/arm64/include/asm/jump_label.h | 52 ++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/insn.c | 304 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/jump_label.c | 58 +++++++ arch/arm64/kernel/module.c | 157 ++++++------------- include/linux/jump_label.h | 19 ++- scripts/gcc-goto.sh | 2 +- 9 files changed, 585 insertions(+), 119 deletions(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/insn.c create mode 100644 arch/arm64/kernel/jump_label.c
Signature · 1 line
1.8.1.2
PATCH v8 1/7 arm64: introduce basic aarch64 instruction decoding helpers
Introduce basic aarch64 instruction decoding helper aarch64_get_insn_class() and aarch64_insn_hotpatch_safe().
--- arch/arm64/include/asm/insn.h | 77 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/insn.c | 91 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/kernel/insn.c
arch/arm64/include/asm/insn.h new file +77 -0
@@ -0,0 +1,77 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef __ASM_INSN_H +#define __ASM_INSN_H +#include <linux/types.h> + +/* +* ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a +* Section C3.1 "A64 instruction index by encoding": +* AArch64 main encoding table +* Bit position +* 28 27 26 25 Encoding Group +* 0 0 - - Unallocated +* 1 0 0 - Data processing, immediate +* 1 0 1 - Branch, exception generation and system instructions +* - 1 - 0 Loads and stores +* - 1 0 1 Data processing - register +* 0 1 1 1 Data processing - SIMD and floating point +* 1 1 1 1 Data processing - SIMD and floating point +* "-" means "don't care" +*/ +enum aarch64_insn_encoding_class { +AARCH64_INSN_CLS_UNKNOWN ,/* UNALLOCATED */ +AARCH64_INSN_CLS_DP_IMM ,/* Data processing - immediate */ +AARCH64_INSN_CLS_DP_REG ,/* Data processing - register */ +AARCH64_INSN_CLS_DP_FPSIMD ,/* Data processing - SIMD and FP */ +AARCH64_INSN_CLS_LDST ,/* Loads and stores */ +AARCH64_INSN_CLS_BR_SYS ,/* Branch, exception generation and +* system instructions */ +}; + +enum aarch64_insn_hint_op { +AARCH64_INSN_HINT_NOP =0x0 <<5 , +AARCH64_INSN_HINT_YIELD =0x1 <<5 , +AARCH64_INSN_HINT_WFE =0x2 <<5 , +AARCH64_INSN_HINT_WFI =0x3 <<5 , +AARCH64_INSN_HINT_SEV =0x4 <<5 , +AARCH64_INSN_HINT_SEVL =0x5 <<5 , +}; + +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ +{ return (code & (mask)) == (val); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ +{ return (val); } + +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F) + +#undef __AARCH64_INSN_FUNCS + +bool aarch64_insn_is_nop (u32 insn); + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); + +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); + +#endif /* __ASM_INSN_H */
arch/arm64/kernel/Makefile +1 -1
@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o + hyp-stub.o psci.o cpu_ops.o insn.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o
arch/arm64/kernel/insn.c new file +91 -0
91 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v8 2/7 arm64: introduce interfaces to hotpatch kernel and module code
In reply to Jiang LiuIntroduce three interfaces to patch kernel and module code:aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed
--- arch/arm64/include/asm/insn.h | 10 +++- arch/arm64/kernel/insn.c | 119 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +9 -1
@@ -18,6 +18,9 @@#define __ASM_INSN_H#include <linux/types.h> +/* A64 instructions are always 32 bits. */ +#define AARCH64_INSN_SIZE 4 +/* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding":
@@ -70,8 +73,13 @@ __AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F)bool aarch64_insn_is_nop (u32 insn); +int aarch64_insn_read (void *addr,u32 *insnp); +int aarch64_insn_write (void *addr,u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); -bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); +int aarch64_insn_patch_text_nosync (void *addr,u32 insn); +int aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt); +#endif /* __ASM_INSN_H */
arch/arm64/kernel/insn.c +119 -0
134 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v8 3/7 arm64: move encode_insn_immediate() from module.c to insn.c
In reply to Jiang LiuFunction encode_insn_immediate() will be used by other instruction manipulate related functions, so move it into insn.c and rename it as aarch64_insn_encode_immediate().
--- arch/arm64/include/asm/insn.h | 13 ++++ arch/arm64/kernel/insn.c | 54 +++++++++++++++ arch/arm64/kernel/module.c | 157 +++++++++++++----------------------------- 3 files changed, 114 insertions(+), 110 deletions(-)
arch/arm64/include/asm/insn.h +13 -0
@@ -55,6 +55,17 @@ enum aarch64_insn_hint_op { AARCH64_INSN_HINT_SEVL = 0x5 << 5, }; +enum aarch64_insn_imm_type { +AARCH64_INSN_IMM_ADR , +AARCH64_INSN_IMM_26 , +AARCH64_INSN_IMM_19 , +AARCH64_INSN_IMM_16 , +AARCH64_INSN_IMM_14 , +AARCH64_INSN_IMM_12 , +AARCH64_INSN_IMM_9 , +AARCH64_INSN_IMM_MAX +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \
@@ -76,6 +87,8 @@ bool aarch64_insn_is_nop(u32 insn);int aarch64_insn_read (void *addr,u32 *insnp);int aarch64_insn_write (void *addr,u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type, +u32 insn,u64 imm);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addr,u32 insn);
arch/arm64/kernel/insn.c +54 -0
57 lines not on this page · raw message
arch/arm64/kernel/module.c +47 -110
303 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v8 4/7 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions
In reply to Jiang LiuIntroduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.
--- arch/arm64/include/asm/insn.h | 10 ++++++++++ arch/arm64/kernel/insn.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+)
arch/arm64/include/asm/insn.h +10 -0
@@ -66,6 +66,11 @@ enum aarch64_insn_imm_type { AARCH64_INSN_IMM_MAX }; +enum aarch64_insn_branch_type { +AARCH64_INSN_BRANCH_NOLINK , +AARCH64_INSN_BRANCH_LINK , +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \
@@ -89,6 +94,11 @@ int aarch64_insn_write(void *addr, u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type,u32 insn,u64 imm); +u32 aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr, +enum aarch64_insn_branch_type type); +u32 aarch64_insn_gen_hint (enum aarch64_insn_hint_op op); +u32 aarch64_insn_gen_nop (void ); +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addr,u32 insn);
arch/arm64/kernel/insn.c +40 -0
@@ -14,6 +14,7 @@* You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/bitops.h> #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/smp.h>
@@ -262,3 +263,42 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,return insn; } + +u32 __kprobes aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr, +enum aarch64_insn_branch_type type) +{ +u32 insn; +long offset; + +/* +* PC: A 64-bit Program Counter holding the address of the current +* instruction. A64 instructions must be word-aligned. +*/ +BUG_ON ((pc &0x3 ) || (addr &0x3 )); + +/* +* B/BL support [-128M, 128M) offset +* ARM64 virtual address arrangement guarantees all kernel and module +* texts are within +/-128M. +*/ + offset = ((long )addr - (long )pc); +BUG_ON (offset < -SZ_128M || offset >=SZ_128M ); + +if (type ==AARCH64_INSN_BRANCH_LINK ) + insn =aarch64_insn_get_bl_value (); +else + insn =aarch64_insn_get_b_value (); + +return aarch64_insn_encode_immediate (AARCH64_INSN_IMM_26 , insn, + offset >>2 ); +} + +u32 __kprobes aarch64_insn_gen_hint (enum aarch64_insn_hint_op op) +{ +return aarch64_insn_get_hint_value () | op; +} + +u32 __kprobes aarch64_insn_gen_nop (void ) +{ +return aarch64_insn_gen_hint (AARCH64_INSN_HINT_NOP ); +}
Signature · 1 line
1.8.1.2
PATCH v8 5/7 arm64, jump label: detect %c support for ARM64
In reply to Jiang LiuAs commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.
Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.
--- scripts/gcc-goto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/gcc-goto.sh +1 -1
@@ -5,7 +5,7 @@ cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" int main(void) { -#ifdef __arm__ +#if defined(__arm__) || defined(__aarch64__) /* * Not related to asm goto, but used by jump label * and broken on some ARM GCC versions (see GCC Bug 48637).
Signature · 1 line
1.8.1.2
PATCH v8 6/7 arm64, jump label: optimize jump label implementation
In reply to Jiang LiuOptimize jump label implementation for ARM64 by dynamically patching kernel text.
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
@@ -19,6 +19,7 @@ config ARM64 select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL select HARDIRQS_SW_RESEND + select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +52 -0
52 lines not on this page · raw message
arch/arm64/kernel/Makefile +1 -0
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +58 -0
58 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v8 6/7] arm64, jump label: optimize jump label implementation
Quoted 4 lines · Jiang Liu
Optimize jump label implementation for ARM64 by dynamically patching kernel text.
Signed-off-by Jiang Liu <[email protected]>
Quoted 8 lines
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.c
Thanks for sticking with this! I think this series is good for merging now.
Will
Re: [PATCH v8 6/7] arm64, jump label: optimize jump label implementation
Hi Will,
Who should I send this patch set for merging?
Thanks!
Gerry
Quoted 20 lines · Will Deacon
Quoted 4 lines · Jiang Liu
Optimize jump label implementation for ARM64 by dynamically patching kernel text.
Signed-off-by Jiang Liu <[email protected]>Quoted 8 lines
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.cReviewed-by Will Deacon <[email protected]>Thanks for sticking with this! I think this series is good for merging now.
Will
Re: [PATCH v8 6/7] arm64, jump label: optimize jump label implementation
Quoted 2 lines · Jiang Liu
Hi Will,
Who should I send this patch set for merging?
Catalin and I will pick up it from here and take it via the arm64 tree, so you don't need to do anything.
Cheers,
Will
Catalin Marinas
maintainerRe: [PATCH v8 6/7] arm64, jump label: optimize jump label implementation
Quoted 5 lines · Will Deacon
Quoted 1 line · Jiang Liu
Who should I send this patch set for merging?
Catalin and I will pick up it from here and take it via the arm64 tree, so you don't need to do anything.
Done. It should appear in linux-next overnight (Jiang, please check in case I missed anything).
Thanks.
Signature · 1 line
Catalin
PATCH v8 7/7 jump_label: use defined macros instead of hard-coding for better readability
In reply to Jiang LiuUse macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.
--- include/linux/jump_label.h | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-)
include/linux/jump_label.h +12 -7
@@ -81,18 +81,21 @@ struct module;#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1ULstatic inline struct jump_entry *jump_label_get_entries (struct static_key *key) {return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TRUE_BRANCH ); + & ~JUMP_LABEL_TYPE_MASK ); }static inline bool jump_label_get_branch_default (struct static_key *key) { -if ((unsigned long )key->entries &JUMP_LABEL_TRUE_BRANCH ) +if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == +JUMP_LABEL_TYPE_TRUE_BRANCH )return true;return false; }
@@ -122,10 +125,12 @@ extern void static_key_slow_inc(struct static_key *key);extern void static_key_slow_dec (struct static_key *key);extern void jump_label_apply_nops (struct module *mod); -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) -#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) +#define STATIC_KEY_INIT_TRUE ((struct static_key) \ + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) +#define STATIC_KEY_INIT_FALSE ((struct static_key) \ + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })#else /* !HAVE_JUMP_LABEL */
Signature · 1 line
1.8.1.2