Optimize jump label implementation for ARM64
Subsystems (2)
v4 cover Optimize jump label implementation for ARM64
From: Jiang Liu <[email protected]>
This patchset tries to optimize arch specfic jump label implementation for ARM64 by dynamic kernel text patching.
To enable this feature, your toolchain must support "asm goto" extension and "%c" constraint extesion. Current GCC for AARCH64 doesn't support "%c", so you need a GCC patch similiar to this:
http://gcc.gnu.org/viewcvs/gcc/trunk/gcc/config/arm/arm.c?view=patch&r1=175293&r2=175565&pathrev=175565
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=48637
It has been tested on ARM Fast mode and a real hardware platform.
Any comments are welcomed!
V3->V4:
1) resolve a race condition in kernel text patching
2) address other review comments
V2->V3:
1) fix a bug in comparing signed and unsigned values
2) detect big endian by checking __AARCH64EB__
V1->V2: address review comments of V1
1) refine comments
2) add a new interface to always synchronize with stop_machine() when patching code
3) handle endian issue when patching code
Jiang Liu (7):
arm64: introduce basic aarch64 instruction decoding helpers
arm64: introduce interfaces to hotpatch kernel and module code
arm64: move encode_insn_immediate() from module.c to insn.c
arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64, jump label: detect %c support for ARM64
arm64, jump label: optimize jump label implementation
jump_label: use defined macros instead of hard-coding for better
readability
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/insn.h | 110 +++++++++++++++++ arch/arm64/include/asm/jump_label.h | 52 ++++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/insn.c | 235 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/jump_label.c | 58 +++++++++ arch/arm64/kernel/module.c | 152 +++++++---------------- include/linux/jump_label.h | 15 ++- scripts/gcc-goto.sh | 2 +- 9 files changed, 514 insertions(+), 114 deletions(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/insn.c create mode 100644 arch/arm64/kernel/jump_label.c
Signature · 1 line
1.8.1.2
PATCH v4 1/7 arm64: introduce basic aarch64 instruction decoding helpers
From: Jiang Liu <[email protected]>
Introduce basic aarch64 instruction decoding helper aarch64_get_insn_class() and aarch64_insn_hotpatch_safe().
--- arch/arm64/include/asm/insn.h | 67 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/insn.c | 72 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/kernel/insn.c
arch/arm64/include/asm/insn.h new file +67 -0
@@ -0,0 +1,67 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef _ASM_ARM64_INSN_H +#define _ASM_ARM64_INSN_H +#include <linux/types.h> + +/* +* ARM Architecture Reference Manual ARMv8, Section C3.1 +* AArch64 main encoding table +* Bit position +* 28 27 26 25 Encoding Group +* 0 0 - - Unallocated +* 1 0 0 - Data processing, immediate +* 1 0 1 - Branch, exception generation and system instructions +* - 1 - 0 Loads and stores +* - 1 0 1 Data processing - register +* 0 1 1 1 Data processing - SIMD and floating point +* 1 1 1 1 Data processing - SIMD and floating point +* "-" means "don't care" +*/ +enum aarch64_insn_encoding_class { +AARCH64_INSN_CLS_UNKNOWN ,/* UNALLOCATED */ +AARCH64_INSN_CLS_DP_IMM ,/* Data processing - immediate */ +AARCH64_INSN_CLS_DP_REG ,/* Data processing - register */ +AARCH64_INSN_CLS_DP_FPSIMD ,/* Data processing - SIMD and FP */ +AARCH64_INSN_CLS_LDST ,/* Loads and stores */ +AARCH64_INSN_CLS_BR_SYS ,/* Branch, exception generation and +* system instructions */ +}; + +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ +{ return (code & (mask)) == (val); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_mask(void) \ +{ return (mask); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ +{ return (val); } + +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) + +#undef __AARCH64_INSN_FUNCS + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); + +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); + +#endif /* _ASM_ARM64_INSN_H */
arch/arm64/kernel/Makefile +1 -1
@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o + hyp-stub.o psci.o insn.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o
arch/arm64/kernel/insn.c new file +72 -0
72 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v4 1/7] arm64: introduce basic aarch64 instruction decoding helpers
Quoted 29 lines · Jiang Liu
From: Jiang Liu <[email protected]>
Introduce basic aarch64 instruction decoding helper
aarch64_get_insn_class()andaarch64_insn_hotpatch_safe().Signed-off-by Jiang Liu <[email protected]>Cc Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 67 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/insn.c | 72 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/kernel/insn.carch/arm64/include/asm/insn.h new file +8
-0@@ -0,0 +1,67 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*
arch/arm64/include/asm/insn.h line 18
Quoted 10 lines
arch/arm64/include/asm/insn.h new file +10
-0+* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef _ASM_ARM64_INSN_H +#define _ASM_ARM64_INSN_H
We've tried to be consistent using __ASM_<name>_H here.
Quoted 101 lines
arch/arm64/include/asm/insn.h new file +49
-0+#include <linux/types.h> + +/* +* ARM Architecture Reference Manual ARMv8, Section C3.1 +* AArch64 main encoding table +* Bit position +* 28 27 26 25 Encoding Group +* 0 0 - - Unallocated +* 1 0 0 - Data processing, immediate +* 1 0 1 - Branch, exception generation and system instructions +* - 1 - 0 Loads and stores +* - 1 0 1 Data processing - register +* 0 1 1 1 Data processing - SIMD and floating point +* 1 1 1 1 Data processing - SIMD and floating point +* "-" means "don't care" +*/ +enum aarch64_insn_encoding_class { +AARCH64_INSN_CLS_UNKNOWN ,/* UNALLOCATED */ +AARCH64_INSN_CLS_DP_IMM ,/* Data processing - immediate */ +AARCH64_INSN_CLS_DP_REG ,/* Data processing - register */ +AARCH64_INSN_CLS_DP_FPSIMD ,/* Data processing - SIMD and FP */ +AARCH64_INSN_CLS_LDST ,/* Loads and stores */ +AARCH64_INSN_CLS_BR_SYS ,/* Branch, exception generation and +* system instructions */ +}; + +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ +{ return (code & (mask)) == (val); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_mask(void) \ +{ return (mask); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ +{ return (val); } + +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) + +#undef __AARCH64_INSN_FUNCS + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); + +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); + +#endif /* _ASM_ARM64_INSN_H */ arch/arm64/kernel/Makefile +1
-1@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o + hyp-stub.o psci.o insn.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.oarch/arm64/kernel/insn.c new file +33
-0@@ -0,0 +1,72 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <asm/insn.h> + +static int aarch64_insn_encoding_cls[] = { +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_REG , +AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_FPSIMD , +AARCH64_INSN_CLS_DP_IMM , +AARCH64_INSN_CLS_DP_IMM , +AARCH64_INSN_CLS_BR_SYS , +AARCH64_INSN_CLS_BR_SYS ,
arch/arm64/kernel/insn.c line 43
Quoted 10 lines
arch/arm64/kernel/insn.c new file +10
-0+AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_REG , +AARCH64_INSN_CLS_LDST , + AARCH64_INSN_CLS_DP_FPSIMD, +}; + +enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class (u32 insn) +{ +return aarch64_insn_encoding_cls[(insn >>25 ) &0xf ]; +}
For some reason, I just read cls' as count leading set', so you could just use `class' here to help lazy readers like me :)
Quoted 18 lines
arch/arm64/kernel/insn.c new file +18
-0+static bool __kprobes __aarch64_insn_hotpatch_safe (u32 insn) +{ +if (aarch64_get_insn_class (insn) !=AARCH64_INSN_CLS_BR_SYS ) +return false; + +return aarch64_insn_is_b (insn) || +aarch64_insn_is_bl (insn) || +aarch64_insn_is_svc (insn) || +aarch64_insn_is_hvc (insn) || +aarch64_insn_is_smc (insn) || +aarch64_insn_is_brk (insn) || +aarch64_insn_is_nop (insn); +} + +/* +* ARMv8-A Section B2.6.5: +* Concurrent modification and execution of instructions can lead to the +* resulting instruction performing any behavior that can be achieved by
arch/arm64/kernel/insn.c line 72
Quoted 10 lines
arch/arm64/kernel/insn.c new file +10
-0+* executing any sequence of instructions that can be executed from the +* same Exception level, except where the instruction before modification +* and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC, +* or SMC instruction. +*/ +bool __kprobes aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn) +{ +return __aarch64_insn_hotpatch_safe (old_insn) && +__aarch64_insn_hotpatch_safe (new_insn); +}
With those cosmetic changes:
Will
Re: [PATCH v4 1/7] arm64: introduce basic aarch64 instruction decoding helpers
Hi Will,
Thanks for review and will make those small changes.
Gerry
Quoted 193 lines · Will Deacon
Quoted 29 lines · Jiang Liu
From: Jiang Liu <[email protected]>
Introduce basic aarch64 instruction decoding helper
aarch64_get_insn_class()andaarch64_insn_hotpatch_safe().Signed-off-by Jiang Liu <[email protected]>Cc Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 67 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/insn.c | 72 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/kernel/insn.carch/arm64/include/asm/insn.h new file +8
-0@@ -0,0 +1,67 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* arch/arm64/include/asm/insn.h line 18
Quoted 10 lines
arch/arm64/include/asm/insn.h new file +10
-0+* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef _ASM_ARM64_INSN_H +#define _ASM_ARM64_INSN_HWe've tried to be consistent using __ASM_<name>_H here.
Quoted 101 lines
arch/arm64/include/asm/insn.h new file +49
-0+#include <linux/types.h> + +/* +* ARM Architecture Reference Manual ARMv8, Section C3.1 +* AArch64 main encoding table +* Bit position +* 28 27 26 25 Encoding Group +* 0 0 - - Unallocated +* 1 0 0 - Data processing, immediate +* 1 0 1 - Branch, exception generation and system instructions +* - 1 - 0 Loads and stores +* - 1 0 1 Data processing - register +* 0 1 1 1 Data processing - SIMD and floating point +* 1 1 1 1 Data processing - SIMD and floating point +* "-" means "don't care" +*/ +enum aarch64_insn_encoding_class { +AARCH64_INSN_CLS_UNKNOWN ,/* UNALLOCATED */ +AARCH64_INSN_CLS_DP_IMM ,/* Data processing - immediate */ +AARCH64_INSN_CLS_DP_REG ,/* Data processing - register */ +AARCH64_INSN_CLS_DP_FPSIMD ,/* Data processing - SIMD and FP */ +AARCH64_INSN_CLS_LDST ,/* Loads and stores */ +AARCH64_INSN_CLS_BR_SYS ,/* Branch, exception generation and +* system instructions */ +}; + +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ +{ return (code & (mask)) == (val); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_mask(void) \ +{ return (mask); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ +{ return (val); } + +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) + +#undef __AARCH64_INSN_FUNCS + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); + +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); + +#endif /* _ASM_ARM64_INSN_H */ arch/arm64/kernel/Makefile +1
-1@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o + hyp-stub.o psci.o insn.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.oarch/arm64/kernel/insn.c new file +33
-0@@ -0,0 +1,72 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <asm/insn.h> + +static int aarch64_insn_encoding_cls[] = { +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_UNKNOWN , +AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_REG , +AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_FPSIMD , +AARCH64_INSN_CLS_DP_IMM , +AARCH64_INSN_CLS_DP_IMM , +AARCH64_INSN_CLS_BR_SYS , +AARCH64_INSN_CLS_BR_SYS ,arch/arm64/kernel/insn.c line 43
Quoted 10 lines
arch/arm64/kernel/insn.c new file +10
-0+AARCH64_INSN_CLS_LDST , +AARCH64_INSN_CLS_DP_REG , +AARCH64_INSN_CLS_LDST , + AARCH64_INSN_CLS_DP_FPSIMD, +}; + +enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class (u32 insn) +{ +return aarch64_insn_encoding_cls[(insn >>25 ) &0xf ]; +}For some reason, I just read
cls' ascount leading set', so you could just use `class' here to help lazy readers like me :)Quoted 18 lines
arch/arm64/kernel/insn.c new file +18
-0+static bool __kprobes __aarch64_insn_hotpatch_safe (u32 insn) +{ +if (aarch64_get_insn_class (insn) !=AARCH64_INSN_CLS_BR_SYS ) +return false; + +return aarch64_insn_is_b (insn) || +aarch64_insn_is_bl (insn) || +aarch64_insn_is_svc (insn) || +aarch64_insn_is_hvc (insn) || +aarch64_insn_is_smc (insn) || +aarch64_insn_is_brk (insn) || +aarch64_insn_is_nop (insn); +} + +/* +* ARMv8-A Section B2.6.5: +* Concurrent modification and execution of instructions can lead to the +* resulting instruction performing any behavior that can be achieved by arch/arm64/kernel/insn.c line 72
Quoted 10 lines
arch/arm64/kernel/insn.c new file +10
-0+* executing any sequence of instructions that can be executed from the +* same Exception level, except where the instruction before modification +* and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC, +* or SMC instruction. +*/ +bool __kprobes aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn) +{ +return __aarch64_insn_hotpatch_safe (old_insn) && +__aarch64_insn_hotpatch_safe (new_insn); +}With those cosmetic changes:
Reviewed-by Will Deacon <[email protected]>Will
PATCH v4 2/7 arm64: introduce interfaces to hotpatch kernel and module code
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
Introduce three interfaces to patch kernel and module code:aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed
--- arch/arm64/include/asm/insn.h | 24 +++++++++++++- arch/arm64/kernel/insn.c | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +23 -1
@@ -60,8 +60,30 @@ __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F)#undef __AARCH64_INSN_FUNCS -enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +/* +* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always +* little-endian. On the other hand, SCTLR_EL1.EE (bit 25, Exception Endianness) +* flag controls endianness for EL1 explicit data accesses and stage 1 +* translation table walks as below: +* 0: little-endian +* 1: big-endian +* So need to handle endianness when patching kernel code. +*/ +static __always_inline u32 aarch64_insn_read(void *addr) +{ +return le32_to_cpu (*(u32 *)addr); +} +static __always_inline void aarch64_insn_write(void *addr, u32 insn) +{ + *(u32 *)addr =cpu_to_le32 (insn); +} + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); +int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt); +#endif /* _ASM_ARM64_INSN_H */
arch/arm64/kernel/insn.c +77 -0
86 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
[adding Tixy for stop_machine() question below]
Quoted 26 lines · Jiang Liu
From: Jiang Liu <[email protected]>
Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize withstop_machine()aarch64_insn_patch_text():
patch code and synchronize withstop_machine()if neededSigned-off-by Jiang Liu <[email protected]>Cc Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 24 +++++++++++++- arch/arm64/kernel/insn.c | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-)arch/arm64/include/asm/insn.h +0
-0@@ -60,8 +60,30 @@ __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F)#undef __AARCH64_INSN_FUNCS
arch/arm64/include/asm/insn.h __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) · line 71
Quoted 10 lines
arch/arm64/include/asm/insn.h +9
-1-enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +/* +* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always +* little-endian. On the other hand, SCTLR_EL1.EE (bit 25, Exception Endianness) +* flag controls endianness for EL1 explicit data accesses and stage 1 +* translation table walks as below: +* 0: little-endian +* 1: big-endian +* So need to handle endianness when patching kernel code. +*/
You can delete this comment now that we're using the helpers...
arch/arm64/include/asm/insn.h __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) · line 80
Quoted 9 lines
arch/arm64/include/asm/insn.h +8
-0+static __always_inline u32 aarch64_insn_read(void *addr) +{ +return le32_to_cpu (*(u32 *)addr); +} +static __always_inline void aarch64_insn_write(void *addr, u32 insn) +{ + *(u32 *)addr =cpu_to_le32 (insn); +}
... then just inline these calls directly.
Quoted 25 lines
arch/arm64/include/asm/insn.h +5
-0+enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); +int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt); +#endif /* _ASM_ARM64_INSN_H */ arch/arm64/kernel/insn.c +3
-0@@ -16,6 +16,9 @@*/ #include <linux/compiler.h> #include <linux/kernel.h> +#include <linux/smp.h> +#include <linux/stop_machine.h> +#include <asm/cacheflush.h> #include <asm/insn.h> static int aarch64_insn_encoding_cls[] = {@@ -70,3 +73,77 @@ bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)return __aarch64_insn_hotpatch_safe (old_insn) &&__aarch64_insn_hotpatch_safe (new_insn);
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 84
Quoted 10 lines
arch/arm64/kernel/insn.c +9
-0} + +int __kprobes aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[], +int cnt) +{ +int i; +u32 *tp; + +if (cnt <=0 ) +return -EINVAL ;
Isn't cnt always 1 for the _nosync patching? Can you just drop the argument and simplify this code? Patching a sequence without syncing is always racy.
Quoted 33 lines
arch/arm64/kernel/insn.c +33
-0+for (i =0 ; i < cnt; i++) { + tp = addrs[i]; +/* A64 instructions must be word aligned */ +if ((uintptr_t )tp &0x3 ) +return -EINVAL ; +aarch64_insn_write (tp, insns[i]); +flush_icache_range ((uintptr_t )tp, (uintptr_t )tp +sizeof (u32)); + } + +return 0 ; +} + +struct aarch64_insn_patch { +void **text_addrs; +u32 *new_insns; +int insn_cnt; +}; + +static int __kprobes aarch64_insn_patch_text_cb (void *arg) +{ +struct aarch64_insn_patch *pp = arg; + +return aarch64_insn_patch_text_nosync (pp->text_addrs, pp->new_insns, + pp->insn_cnt); +} + +int __kprobes aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt) +{ +struct aarch64_insn_patch patch = { + .text_addrs = addrs, + .new_insns = insns, + .insn_cnt = cnt, + };
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 10 lines
arch/arm64/kernel/insn.c +10
-0+ +if (cnt <=0 ) +return -EINVAL ; + +/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}
Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the arch/arm/ call to stop_machine in kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do in patch.c) and the flushing could be separated from the patching too.
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 135
Quoted 6 lines
arch/arm64/kernel/insn.c +6
-0+int __kprobes aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt) +{ +int ret; + +if (cnt ==1 &&aarch64_insn_hotpatch_safe (aarch64_insn_read (addrs[0 ]), + insns[0 ])) {
You could make aarch64_insn_hotpatch_safe take the cnt parameter and return false if cnt != 1.
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 135
Quoted 2 lines
arch/arm64/kernel/insn.c +2
-0+/* +* It doesn't guarantee all CPUs see the new instruction
"It"? You mean the ARMv8 architecture?
Will
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
Quoted 3 lines · Will Deacon
[adding Tixy for
stop_machine()question below]On Thu, Oct 17, 2013 at 07:19:35AM +0100, Jiang Liu wrote:
[...]
Quoted 18 lines
Quoted 7 lines
arch/arm64/kernel/insn.c +7
-0+int __kprobes aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt) +{ +struct aarch64_insn_patch patch = { + .text_addrs = addrs, + .new_insns = insns, + .insn_cnt = cnt, + };arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 10 lines
arch/arm64/kernel/insn.c +10
-0+ +if (cnt <=0 ) +return -EINVAL ; + +/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}
Quoted 7 lines
Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.
[...]
For code modifications done in 32bit ARM kprobes (and ftrace) I'm not sure we ever actually resolved the possible cache flushing issues. If there was specific reasons for flushing on all cores I can't remember them, sorry. I have a suspicion that doing so was a case of sticking with what the code was already doing, and flushing on all cores seemed safest to guard against problems we hadn't thought about.
Some of the issues discussed were that we couldn't have one core potentially executing instructions being modified by another CPU, because that's architecturally unpredictable except for a few instructions [1], and we also have the case where a 32-bit Thumb instruction can straddle two different cache-lines. But these may not be reasons to flush on all cores if stop machine is synchronising all CPU's in a kind of holding pen and the cache operations done on one core are broadcast to others. (Are there correct barriers involved in stop-machine so that when the other cores resume they are guaranteed to only see the new version of the modified code, or do we only get that guarantee because we happen to execute the cache flushing on all cores?)
[1] http://lists.infradead.org/pipermail/linux-arm-kernel/2012-December/136441.html
Another of the issues I hit was big.LITTLE related whereby the cache line size is different on different cores [2].
[2] http://lists.infradead.org/pipermail/linux-arm-kernel/2013-February/149794.html
I don't think anything I've said above actually gives a solid reason why we _must_ execute cache flushing on all cores for kprobes and can't just use the relatively new patch_text function (which checks for the one case we do need to flush on all cores using cache_ops_need_broadcast).
Sorry, I don't think I've added much light on things here have I?
Signature · 1 line
Tixy
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
Quoted 39 lines · Jon Medhurst (Tixy)
Quoted 3 lines · Will Deacon
[adding Tixy for
stop_machine()question below]On Thu, Oct 17, 2013 at 07:19:35AM +0100, Jiang Liu wrote:
[...]
Quoted 18 lines
Quoted 7 lines
arch/arm64/kernel/insn.c +7
-0+int __kprobes aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt) +{ +struct aarch64_insn_patch patch = { + .text_addrs = addrs, + .new_insns = insns, + .insn_cnt = cnt, + };arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 10 lines
arch/arm64/kernel/insn.c +10
-0+ +if (cnt <=0 ) +return -EINVAL ; + +/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}Quoted 7 lines
Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.[...]
For code modifications done in 32bit ARM kprobes (and ftrace) I'm not sure we ever actually resolved the possible cache flushing issues. If there was specific reasons for flushing on all cores I can't remember them, sorry. I have a suspicion that doing so was a case of sticking with what the code was already doing, and flushing on all cores seemed safest to guard against problems we hadn't thought about.
Quoted 11 lines
Some of the issues discussed were that we couldn't have one core potentially executing instructions being modified by another CPU, because that's architecturally unpredictable except for a few instructions [1], and we also have the case where a 32-bit Thumb instruction can straddle two different cache-lines. But these may not be reasons to flush on all cores if stop machine is synchronising all CPU's in a kind of holding pen and the cache operations done on one core are broadcast to others. (Are there correct barriers involved in stop-machine so that when the other cores resume they are guaranteed to only see the new version of the modified code, or do we only get that guarantee because we happen to execute the cache flushing on all cores?)
I think it's cache flushing instead of stop_machine() because cache flusing includes an ISB.
The idea flow should be:
1) master acquire a lock to serialize text patching
2) all CPU barrier
3) master updates memory and flush cache
4) master set a flag to let all other CPU continue
5) all other CPU executes ISB.
Updating memory and flushing cache on every CPU with stop_machine() achieves the same effect with simple implementation, but the implementation really seems a little strange.
If desired, I will implement the standard flow.
Quoted 15 lines
[1] http://lists.infradead.org/pipermail/linux-arm-kernel/2012-December/136441.html
Another of the issues I hit was big.LITTLE related whereby the cache line size is different on different cores [2].
[2] http://lists.infradead.org/pipermail/linux-arm-kernel/2013-February/149794.html
I don't think anything I've said above actually gives a solid reason why we _must_ execute cache flushing on all cores for kprobes and can't just use the relatively new
patch_textfunction (which checks for the one case we do need to flush on all cores usingcache_ops_need_broadcast).Sorry, I don't think I've added much light on things here have I?
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
In reply to Jon Medhurst (Tixy)Hi Tixy,
Quoted 24 lines · Jon Medhurst (Tixy)
Quoted 15 lines · Will Deacon
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 6 lines · Jiang Liu
arch/arm64/kernel/insn.c +6
-0+/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.[...]
For code modifications done in 32bit ARM kprobes (and ftrace) I'm not sure we ever actually resolved the possible cache flushing issues. If there was specific reasons for flushing on all cores I can't remember them, sorry. I have a suspicion that doing so was a case of sticking with what the code was already doing, and flushing on all cores seemed safest to guard against problems we hadn't thought about.
[...]
Quoted 1 line
Sorry, I don't think I've added much light on things here have I?
I think you missed the bit I was confused about :) Flushing the cache on each core is necessary if cache_ops_need_broadcast, so I can understand why you'd have code to do that. The bit I don't understand is that you actually patch the instruction on each core too!
Will
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
Quoted 28 lines · Will Deacon
Hi Tixy,
Quoted 16 lines · Jon Medhurst (Tixy)
Quoted 15 lines · Will Deacon
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 6 lines · Jiang Liu
arch/arm64/kernel/insn.c +6
-0+/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.Quoted 8 lines
[...]
For code modifications done in 32bit ARM kprobes (and ftrace) I'm not sure we ever actually resolved the possible cache flushing issues. If there was specific reasons for flushing on all cores I can't remember them, sorry. I have a suspicion that doing so was a case of sticking with what the code was already doing, and flushing on all cores seemed safest to guard against problems we hadn't thought about.
Quoted 8 lines
[...]
Quoted 1 line
Sorry, I don't think I've added much light on things here have I?
I think you missed the bit I was confused about :) Flushing the cache on each core is necessary if
cache_ops_need_broadcast, so I can understand why you'd have code to do that. The bit I don't understand is that you actually patch the instruction on each core too!
This is only happens when removing a kprobe with __arch_disarm_kprobe(). We can't just use the intelligent patch_text() function there because we want to always force stop machine to be used as this prevents the case where a CPU a hits the probe, starts executing it's handler then another CPU whips away the probe from under it.
That explains why we use stop_machine, but not why all CPU's must modify the instruction. I think it's a case of just that it's simpler to do that unconditionally rather than add extra code for the cache_ops_need_broadcast() case. I mean, stop_machine() is a sledge hammer, which stalls the whole system until the next scheduler tick, and then gets every CPU to busy wait, so there's not much incentive to try and optimise the code to avoid a memory write + cacheline flush on each core.
This reminds me, I'm sure I heard rumours quite some time ago that Paul McKenney was thinking of trying to do away with stop_machine...?
Signature · 1 line
Tixy
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
Quoted 53 lines · Jon Medhurst (Tixy)
Quoted 28 lines · Will Deacon
Hi Tixy,
Quoted 16 lines · Jon Medhurst (Tixy)
Quoted 15 lines · Will Deacon
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 6 lines · Jiang Liu
arch/arm64/kernel/insn.c +6
-0+/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.Quoted 8 lines
[...]
For code modifications done in 32bit ARM kprobes (and ftrace) I'm not sure we ever actually resolved the possible cache flushing issues. If there was specific reasons for flushing on all cores I can't remember them, sorry. I have a suspicion that doing so was a case of sticking with what the code was already doing, and flushing on all cores seemed safest to guard against problems we hadn't thought about.
Quoted 8 lines
[...]
Quoted 1 line
Sorry, I don't think I've added much light on things here have I?
I think you missed the bit I was confused about :) Flushing the cache on each core is necessary if
cache_ops_need_broadcast, so I can understand why you'd have code to do that. The bit I don't understand is that you actually patch the instruction on each core too!This is only happens when removing a kprobe with
__arch_disarm_kprobe(). We can't just use the intelligentpatch_text()function there because we want to always force stop machine to be used as this prevents the case where a CPU a hits the probe, starts executing it's handler then another CPU whips away the probe from under it.That explains why we use
stop_machine, but not why all CPU's must modify the instruction. I think it's a case of just that it's simpler to do that unconditionally rather than add extra code for thecache_ops_need_broadcast()case. I mean,stop_machine()is a sledge hammer, which stalls the whole system until the next scheduler tick, and then gets every CPU to busy wait, so there's not much incentive to try and optimise the code to avoid a memory write + cacheline flush on each core.
Quoted 3 lines
This reminds me, I'm sure I heard rumours quite some time ago that Paul McKenney was thinking of trying to do away with
stop_machine...?
I remember McKenney has tried that, don't know about the progress now.
Re: [PATCH v4 2/7] arm64: introduce interfaces to hotpatch kernel and module code
In reply to Will DeaconQuoted 84 lines · Will Deacon
[adding Tixy for
stop_machine()question below]Quoted 26 lines · Jiang Liu
From: Jiang Liu <[email protected]>
Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize withstop_machine()aarch64_insn_patch_text():
patch code and synchronize withstop_machine()if neededSigned-off-by Jiang Liu <[email protected]>Cc Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 24 +++++++++++++- arch/arm64/kernel/insn.c | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-)arch/arm64/include/asm/insn.h +0
-0@@ -60,8 +60,30 @@ __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F)#undef __AARCH64_INSN_FUNCSarch/arm64/include/asm/insn.h __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) · line 71
Quoted 10 lines
arch/arm64/include/asm/insn.h +9
-1-enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +/* +* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always +* little-endian. On the other hand, SCTLR_EL1.EE (bit 25, Exception Endianness) +* flag controls endianness for EL1 explicit data accesses and stage 1 +* translation table walks as below: +* 0: little-endian +* 1: big-endian +* So need to handle endianness when patching kernel code. +*/ You can delete this comment now that we're using the helpers...
arch/arm64/include/asm/insn.h __AARCH64_INSN_FUNCS(nop, 0xFFFFFFFF, 0xD503201F) · line 80
Quoted 9 lines
arch/arm64/include/asm/insn.h +8
-0+static __always_inline u32 aarch64_insn_read(void *addr) +{ +return le32_to_cpu (*(u32 *)addr); +} +static __always_inline void aarch64_insn_write(void *addr, u32 insn) +{ + *(u32 *)addr =cpu_to_le32 (insn); +}... then just inline these calls directly.
Quoted 30 lines
arch/arm64/include/asm/insn.h +5
-0+enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); +int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt); +#endif /* _ASM_ARM64_INSN_H */ arch/arm64/kernel/insn.c +7
-0@@ -16,6 +16,9 @@*/ #include <linux/compiler.h> #include <linux/kernel.h> +#include <linux/smp.h> +#include <linux/stop_machine.h> +#include <asm/cacheflush.h> #include <asm/insn.h> static int aarch64_insn_encoding_cls[] = {@@ -70,3 +73,77 @@ bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)return __aarch64_insn_hotpatch_safe (old_insn) &&__aarch64_insn_hotpatch_safe (new_insn); } + +int __kprobes aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[], +int cnt) +{
Quoted 8 lines
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 84
Quoted 5 lines
arch/arm64/kernel/insn.c +5
-0+int i; +u32 *tp; + +if (cnt <=0 ) +return -EINVAL ;Isn't cnt always 1 for the
_nosyncpatching? Can you just drop the argument and simplify this code? Patching a sequence without syncing is always racy.
Will drop the third parameter and simplify the code.
Quoted 63 lines
Quoted 34 lines
arch/arm64/kernel/insn.c +33
-0+for (i =0 ; i < cnt; i++) { + tp = addrs[i]; +/* A64 instructions must be word aligned */ +if ((uintptr_t )tp &0x3 ) +return -EINVAL ; +aarch64_insn_write (tp, insns[i]); +flush_icache_range ((uintptr_t )tp, (uintptr_t )tp +sizeof (u32)); + } + +return 0 ; +} + +struct aarch64_insn_patch { +void **text_addrs; +u32 *new_insns; +int insn_cnt; +}; + +static int __kprobes aarch64_insn_patch_text_cb (void *arg) +{ +struct aarch64_insn_patch *pp = arg; + +return aarch64_insn_patch_text_nosync (pp->text_addrs, pp->new_insns, + pp->insn_cnt); +} + +int __kprobes aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt) +{ +struct aarch64_insn_patch patch = { + .text_addrs = addrs, + .new_insns = insns, + .insn_cnt = cnt, + };arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128
Quoted 10 lines
arch/arm64/kernel/insn.c +10
-0+ +if (cnt <=0 ) +return -EINVAL ; + +/* +* Execute __aarch64_insn_patch_text() on every online CPU, +* which ensure serialization among all online CPUs. +*/ +return stop_machine (aarch64_insn_patch_text_cb, &patch,NULL ); +}Whoa, whoa, whoa! The comment here is wrong -- we only run the patching on *one* CPU, which is the right thing to do. However, the
arch/arm/call tostop_machinein kprobes does actually run the patching code on *all* the online cores (including the cache flushing!). I think this is to work around cores without hardware cache maintenance broadcasting, but that could easily be called out specially (like we do inpatch.c) and the flushing could be separated from the patching too.arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 135
Quoted 6 lines
arch/arm64/kernel/insn.c +6
-0+int __kprobes aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt) +{ +int ret; + +if (cnt ==1 &&aarch64_insn_hotpatch_safe (aarch64_insn_read (addrs[0 ]), + insns[0 ])) {You could make
aarch64_insn_hotpatch_safetake the cnt parameter and return false if cnt != 1.
Quoted 4 lines
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 135
Quoted 2 lines
arch/arm64/kernel/insn.c +2
-0+/* +* It doesn't guarantee all CPUs see the new instruction "It"? You mean the ARMv8 architecture?
Yes, I mean ARMv8 architecture.
Quoted 3 lines
Will
PATCH v4 3/7 arm64: move encode_insn_immediate() from module.c to insn.c
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
Function encode_insn_immediate() will be used by other instruction manipulate related functions, so move it into insn.c and rename it as aarch64_insn_encode_immediate().
--- arch/arm64/include/asm/insn.h | 14 ++++ arch/arm64/kernel/insn.c | 58 ++++++++++++++++ arch/arm64/kernel/module.c | 152 +++++++++++++----------------------------- 3 files changed, 117 insertions(+), 107 deletions(-)
arch/arm64/include/asm/insn.h +14 -0
@@ -42,6 +42,18 @@ enum aarch64_insn_encoding_class {* system instructions */ }; +enum aarch64_insn_imm_type { +AARCH64_INSN_IMM_MOVK , +AARCH64_INSN_IMM_ADR , +AARCH64_INSN_IMM_26 , +AARCH64_INSN_IMM_19 , +AARCH64_INSN_IMM_16 , +AARCH64_INSN_IMM_14 , +AARCH64_INSN_IMM_12 , +AARCH64_INSN_IMM_9 , +AARCH64_INSN_IMM_MAX +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \
@@ -80,6 +92,8 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn) }enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type, +u32 insn,u64 imm);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt);
arch/arm64/kernel/insn.c +58 -0
61 lines not on this page · raw message
arch/arm64/kernel/module.c +45 -107
294 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v4 3/7] arm64: move encode_insn_immediate() from module.c to insn.c
Quoted 56 lines · Jiang Liu
From: Jiang Liu <[email protected]>
Function
encode_insn_immediate()will be used by other instruction manipulate related functions, so move it intoinsn.cand rename it asaarch64_insn_encode_immediate().Signed-off-by Jiang Liu <[email protected]>Cc Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 14 ++++ arch/arm64/kernel/insn.c | 58 ++++++++++++++++ arch/arm64/kernel/module.c | 152 +++++++++++++----------------------------- 3 files changed, 117 insertions(+), 107 deletions(-)arch/arm64/include/asm/insn.h +14
-0@@ -42,6 +42,18 @@ enum aarch64_insn_encoding_class {* system instructions */ }; +enum aarch64_insn_imm_type { +AARCH64_INSN_IMM_MOVK , +AARCH64_INSN_IMM_ADR , +AARCH64_INSN_IMM_26 , +AARCH64_INSN_IMM_19 , +AARCH64_INSN_IMM_16 , +AARCH64_INSN_IMM_14 , +AARCH64_INSN_IMM_12 , +AARCH64_INSN_IMM_9 , +AARCH64_INSN_IMM_MAX +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \@@ -80,6 +92,8 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn) }enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type, +u32 insn,u64 imm);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt);arch/arm64/kernel/insn.c +2
-0@@ -147,3 +147,61 @@ int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)return ret; } + +u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
arch/arm64/kernel/insn.c int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) · line 161
Quoted 10 lines
arch/arm64/kernel/insn.c +10
-0+ u32 insn, u64 imm) +{ +u32 immlo, immhi, lomask, himask, mask; +int shift; + +switch (type) { +case AARCH64_INSN_IMM_MOVK : + mask =BIT (16 ) -1 ; + shift =5 ; +break ;
This is just AARCH64_INSN_IMM_16 now that we're not dealing with MOV relocations. You can kill INSN_IMM_MOVK in the enum.
The rest of the patch looks fine.
Will
PATCH v4 4/7 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.
--- arch/arm64/include/asm/insn.h | 7 +++++++ arch/arm64/kernel/insn.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+)
arch/arm64/include/asm/insn.h +7 -0
@@ -94,6 +94,13 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn)enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type,u32 insn,u64 imm); +u32 aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr, +bool link); +static __always_inline u32 aarch64_insn_gen_nop(void) +{ +return aarch64_insn_get_nop_value (); +} +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addrs[],u32 insns[],int cnt);
arch/arm64/kernel/insn.c +28 -0
@@ -14,6 +14,7 @@* You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/bitops.h> #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/smp.h>
@@ -205,3 +206,30 @@ u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,return insn; } + +u32 aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr,bool link) +{ +u32 insn; +long offset; + +/* +* PC: A 64-bit Program Counter holding the address of the current +* instruction. A64 instructions may be word-aligned. +*/ +BUG_ON ((pc &0x3 ) || (addr &0x3 )); + +/* +* B/BL support [-128M, 128M) offset +* ARM64 virtual address arrangement garantees all kernel and module +* texts are within +/-128M. +*/ + offset = ((long )addr - (long )pc) >>2 ; +BUG_ON (abs (offset) >BIT (25 ) || offset ==BIT (25 )); + +if (link) + insn =aarch64_insn_get_bl_value (); +else + insn =aarch64_insn_get_b_value (); + +return aarch64_insn_encode_immediate (AARCH64_INSN_IMM_26 , insn, offset); +}
Signature · 1 line
1.8.1.2
PATCH v4 5/7 arm64, jump label: detect %c support for ARM64
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
As commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.
Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.
--- scripts/gcc-goto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/gcc-goto.sh +1 -1
@@ -5,7 +5,7 @@ cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" int main(void) { -#ifdef __arm__ +#if defined(__arm__) || defined(__aarch64__) /* * Not related to asm goto, but used by jump label * and broken on some ARM GCC versions (see GCC Bug 48637).
Signature · 1 line
1.8.1.2
PATCH v4 6/7 arm64, jump label: optimize jump label implementation
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
Optimize jump label implementation for ARM64 by dynamically patching kernel text.
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
@@ -17,6 +17,7 @@ config ARM64 select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL select HARDIRQS_SW_RESEND + select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +52 -0
52 lines not on this page · raw message
arch/arm64/kernel/Makefile +1 -0
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o smp_psci.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +58 -0
58 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v4 7/7 jump_label: use defined macros instead of hard-coding for better readability
In reply to Jiang LiuFrom: Jiang Liu <[email protected]>
Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.
--- include/linux/jump_label.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
include/linux/jump_label.h +10 -5
@@ -74,18 +74,21 @@ struct module;#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1ULstatic inline struct jump_entry *jump_label_get_entries (struct static_key *key) {return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TRUE_BRANCH ); + & ~JUMP_LABEL_TYPE_MASK ); }static inline bool jump_label_get_branch_default (struct static_key *key) { -if ((unsigned long )key->entries &JUMP_LABEL_TRUE_BRANCH ) +if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == +JUMP_LABEL_TYPE_TRUE_BRANCH )return true;return false; }
@@ -116,9 +119,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })#else /* !HAVE_JUMP_LABEL */
Signature · 1 line
1.8.1.2