Optimize jump label implementation for ARM64
Subsystems (2)
v6 cover Optimize jump label implementation for ARM64
This patchset tries to optimize arch specfic jump label implementation for ARM64 by dynamic kernel text patching.
To enable this feature, your toolchain must support "asm goto" extension and "%c" constraint extesion. Current GCC for AARCH64 doesn't support "%c", and there's a patch for it now:
http://gcc.gnu.org/ml/gcc-patches/2013-10/msg01314.html
It has been tested on ARM Fast mode and a real hardware platform.
Any comments are welcomed!
V5->V6:
1) optimize stop_machine() related code
2) simplify aarch64_insn_patch_text_nosync() interface
3) refine comments
V4->V5:
1) rework the stop_machine() related code
2) rework aarch64_insn_gen_nop()
V3->V4:
1) resolve a race condition in kernel text patching
2) address other review comments
V2->V3:
1) fix a bug in comparing signed and unsigned values
2) detect big endian by checking __AARCH64EB__
V1->V2: address review comments of V1
1) refine comments
2) add a new interface to always synchronize with stop_machine() when patching code
3) handle endian issue when patching code
Jiang Liu (7):
arm64: introduce basic aarch64 instruction decoding helpers
arm64: introduce interfaces to hotpatch kernel and module code
arm64: move encode_insn_immediate() from module.c to insn.c
arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64, jump label: detect %c support for ARM64
arm64, jump label: optimize jump label implementation
jump_label: use defined macros instead of hard-coding for better
readability
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/insn.h | 108 +++++++++++++ arch/arm64/include/asm/jump_label.h | 51 ++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/insn.c | 302 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/jump_label.c | 59 +++++++ arch/arm64/kernel/module.c | 157 ++++++------------- include/linux/jump_label.h | 15 +- scripts/gcc-goto.sh | 2 +- 9 files changed, 581 insertions(+), 117 deletions(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/insn.c create mode 100644 arch/arm64/kernel/jump_label.c
Signature · 1 line
1.8.1.2
PATCH v6 1/7 arm64: introduce basic aarch64 instruction decoding helpers
Introduce basic aarch64 instruction decoding helper aarch64_get_insn_class() and aarch64_insn_hotpatch_safe().
--- arch/arm64/include/asm/insn.h | 77 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/insn.c | 91 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/insn.h create mode 100644 arch/arm64/kernel/insn.c
arch/arm64/include/asm/insn.h new file +77 -0
@@ -0,0 +1,77 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef __ASM_INSN_H +#define __ASM_INSN_H +#include <linux/types.h> + +/* +* ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a +* Section C3.1 "A64 instruction index by encoding": +* AArch64 main encoding table +* Bit position +* 28 27 26 25 Encoding Group +* 0 0 - - Unallocated +* 1 0 0 - Data processing, immediate +* 1 0 1 - Branch, exception generation and system instructions +* - 1 - 0 Loads and stores +* - 1 0 1 Data processing - register +* 0 1 1 1 Data processing - SIMD and floating point +* 1 1 1 1 Data processing - SIMD and floating point +* "-" means "don't care" +*/ +enum aarch64_insn_encoding_class { +AARCH64_INSN_CLS_UNKNOWN ,/* UNALLOCATED */ +AARCH64_INSN_CLS_DP_IMM ,/* Data processing - immediate */ +AARCH64_INSN_CLS_DP_REG ,/* Data processing - register */ +AARCH64_INSN_CLS_DP_FPSIMD ,/* Data processing - SIMD and FP */ +AARCH64_INSN_CLS_LDST ,/* Loads and stores */ +AARCH64_INSN_CLS_BR_SYS ,/* Branch, exception generation and +* system instructions */ +}; + +enum aarch64_insn_hint_op { +AARCH64_INSN_HINT_NOP =0x0 <<5 , +AARCH64_INSN_HINT_YIELD =0x1 <<5 , +AARCH64_INSN_HINT_WFE =0x2 <<5 , +AARCH64_INSN_HINT_WFI =0x3 <<5 , +AARCH64_INSN_HINT_SEV =0x4 <<5 , +AARCH64_INSN_HINT_SEVL =0x5 <<5 , +}; + +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ +{ return (code & (mask)) == (val); } \ +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ +{ return (val); } + +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F) + +#undef __AARCH64_INSN_FUNCS + +bool aarch64_insn_is_nop (u32 insn); + +enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); + +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); + +#endif /* __ASM_INSN_H */
arch/arm64/kernel/Makefile +1 -1
@@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o + hyp-stub.o psci.o cpu_ops.o insn.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o
arch/arm64/kernel/insn.c new file +91 -0
91 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v6 2/7 arm64: introduce interfaces to hotpatch kernel and module code
In reply to Jiang LiuIntroduce three interfaces to patch kernel and module code:aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed
--- arch/arm64/include/asm/insn.h | 10 +++- arch/arm64/kernel/insn.c | 117 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +9 -1
@@ -18,6 +18,9 @@#define __ASM_INSN_H#include <linux/types.h> +/* A64 instructions are always 32 bits. */ +#define AARCH64_INSN_SIZE 4 +/* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding":
@@ -70,8 +73,13 @@ __AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F)bool aarch64_insn_is_nop (u32 insn); +int aarch64_insn_read (void *addr,u32 *insnp); +int aarch64_insn_write (void *addr,u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); -bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn); +int aarch64_insn_patch_text_nosync (void *addr,u32 insn); +int aarch64_insn_patch_text_sync (void *addrs[],u32 insns[],int cnt); +int aarch64_insn_patch_text (void *addrs[],u32 insns[],int cnt); +#endif /* __ASM_INSN_H */
arch/arm64/kernel/insn.c +117 -0
132 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v6 2/7] arm64: introduce interfaces to hotpatch kernel and module code
Quoted 9 lines · Jiang Liu
Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.aarch64_insn_patch_text_sync():
patch code and always synchronize withstop_machine()aarch64_insn_patch_text():
patch code and synchronize withstop_machine()if needed
Quoted 5 lines
Signed-off-by Jiang Liu <[email protected]>--- arch/arm64/include/asm/insn.h | 10 +++- arch/arm64/kernel/insn.c | 117 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-)
Two comments on the patching callback...
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 154
Quoted 12 lines
arch/arm64/kernel/insn.c +12
-0+static int __kprobes aarch64_insn_patch_text_cb (void *arg) +{ +int i, ret =0 ; +struct aarch64_insn_patch *pp = arg; + +/* The first CPU becomes master */ +if (atomic_inc_return (&pp->cpu_count) ==1 ) { +for (i =0 ; ret ==0 && i < pp->insn_cnt; i++) + ret =aarch64_insn_patch_text_nosync (pp->text_addrs[i], + pp->new_insns[i]); +/* Make sure changes are visible to other CPUs and let them continue */ +smp_wmb ();
As I mentioned last time, you can rely on aarch64_insn_patch_text_nosync to enforce ordering (flush_icache_range ends with dsb; isb). Just add a comment saying that.
arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 159
Quoted 5 lines
arch/arm64/kernel/insn.c +5
-0+atomic_set (&pp->cpu_count,-1 ); + }else { +while (atomic_read (&pp->cpu_count) !=-1 ) +cpu_relax (); +smp_rmb ();
You don't need the smp_rmb here.
Will
PATCH v6 3/7 arm64: move encode_insn_immediate() from module.c to insn.c
In reply to Jiang LiuFunction encode_insn_immediate() will be used by other instruction manipulate related functions, so move it into insn.c and rename it as aarch64_insn_encode_immediate().
--- arch/arm64/include/asm/insn.h | 13 ++++ arch/arm64/kernel/insn.c | 54 +++++++++++++++ arch/arm64/kernel/module.c | 157 +++++++++++++----------------------------- 3 files changed, 114 insertions(+), 110 deletions(-)
arch/arm64/include/asm/insn.h +13 -0
@@ -55,6 +55,17 @@ enum aarch64_insn_hint_op { AARCH64_INSN_HINT_SEVL = 0x5 << 5, }; +enum aarch64_insn_imm_type { +AARCH64_INSN_IMM_ADR , +AARCH64_INSN_IMM_26 , +AARCH64_INSN_IMM_19 , +AARCH64_INSN_IMM_16 , +AARCH64_INSN_IMM_14 , +AARCH64_INSN_IMM_12 , +AARCH64_INSN_IMM_9 , +AARCH64_INSN_IMM_MAX +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \
@@ -76,6 +87,8 @@ bool aarch64_insn_is_nop(u32 insn);int aarch64_insn_read (void *addr,u32 *insnp);int aarch64_insn_write (void *addr,u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn); +u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type, +u32 insn,u64 imm);bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addr,u32 insn);
arch/arm64/kernel/insn.c +54 -0
57 lines not on this page · raw message
arch/arm64/kernel/module.c +47 -110
303 lines not on this page · raw message
Signature · 1 line
1.8.1.2
PATCH v6 4/7 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions
In reply to Jiang LiuIntroduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.
--- arch/arm64/include/asm/insn.h | 10 ++++++++++ arch/arm64/kernel/insn.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+)
arch/arm64/include/asm/insn.h +10 -0
@@ -66,6 +66,11 @@ enum aarch64_insn_imm_type { AARCH64_INSN_IMM_MAX }; +enum aarch64_insn_branch_type { +AARCH64_INSN_BRANCH_NOLINK , +AARCH64_INSN_BRANCH_LINK , +}; +#define __AARCH64_INSN_FUNCS (abbr, mask, val) \ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ { return (code & (mask)) == (val); } \
@@ -89,6 +94,11 @@ int aarch64_insn_write(void *addr, u32 insn);enum aarch64_insn_encoding_class aarch64_get_insn_class (u32 insn);u32 aarch64_insn_encode_immediate (enum aarch64_insn_imm_type type,u32 insn,u64 imm); +u32 aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr, +enum aarch64_insn_branch_type type); +u32 aarch64_insn_gen_hint (enum aarch64_insn_hint_op op); +u32 aarch64_insn_gen_nop (void ); +bool aarch64_insn_hotpatch_safe (u32 old_insn,u32 new_insn);int aarch64_insn_patch_text_nosync (void *addr,u32 insn);
arch/arm64/kernel/insn.c +40 -0
@@ -14,6 +14,7 @@* You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/bitops.h> #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/smp.h>
@@ -260,3 +261,42 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,return insn; } + +u32 __kprobes aarch64_insn_gen_branch_imm (unsigned long pc,unsigned long addr, +enum aarch64_insn_branch_type type) +{ +u32 insn; +long offset; + +/* +* PC: A 64-bit Program Counter holding the address of the current +* instruction. A64 instructions must be word-aligned. +*/ +BUG_ON ((pc &0x3 ) || (addr &0x3 )); + +/* +* B/BL support [-128M, 128M) offset +* ARM64 virtual address arrangement guarantees all kernel and module +* texts are within +/-128M. +*/ + offset = ((long )addr - (long )pc); +BUG_ON (offset < -SZ_128M || offset >=SZ_128M ); + +if (type ==AARCH64_INSN_BRANCH_LINK ) + insn =aarch64_insn_get_bl_value (); +else + insn =aarch64_insn_get_b_value (); + +return aarch64_insn_encode_immediate (AARCH64_INSN_IMM_26 , insn, + offset >>2 ); +} + +u32 __kprobes aarch64_insn_gen_hint (enum aarch64_insn_hint_op op) +{ +return aarch64_insn_get_hint_value () | op; +} + +u32 __kprobes aarch64_insn_gen_nop (void ) +{ +return aarch64_insn_gen_hint (AARCH64_INSN_HINT_NOP ); +}
Signature · 1 line
1.8.1.2
Re: [PATCH v6 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions
Quoted 4 lines · Jiang Liu
Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.
Signed-off-by Jiang Liu <[email protected]>
Looks good to me:
Thanks!
Will
PATCH v6 5/7 arm64, jump label: detect %c support for ARM64
In reply to Jiang LiuAs commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.
Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.
--- scripts/gcc-goto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/gcc-goto.sh +1 -1
@@ -5,7 +5,7 @@ cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" int main(void) { -#ifdef __arm__ +#if defined(__arm__) || defined(__aarch64__) /* * Not related to asm goto, but used by jump label * and broken on some ARM GCC versions (see GCC Bug 48637).
Signature · 1 line
1.8.1.2
PATCH v6 6/7 arm64, jump label: optimize jump label implementation
In reply to Jiang LiuOptimize jump label implementation for ARM64 by dynamically patching kernel text.
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 51 ++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 59 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
@@ -19,6 +19,7 @@ config ARM64 select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL select HARDIRQS_SW_RESEND + select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +51 -0
51 lines not on this page · raw message
arch/arm64/kernel/Makefile +1 -0
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +59 -0
59 lines not on this page · raw message
Signature · 1 line
1.8.1.2
Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
Hi Jiang Liu,
Thanks for the updated patches! I still have some small comments, but I don't think they will cause you much work. See inline.
Quoted 41 lines · Jiang Liu
Optimize jump label implementation for ARM64 by dynamically patching kernel text.
Signed-off-by Jiang Liu <[email protected]>--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/jump_label.h | 51 ++++++++++++++++++++++++++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/jump_label.c | 59 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 arch/arm64/include/asm/jump_label.h create mode 100644 arch/arm64/kernel/jump_label.carch/arm64/Kconfig +1
-0@@ -19,6 +19,7 @@ config ARM64 select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL select HARDIRQS_SW_RESEND + select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAKarch/arm64/include/asm/jump_label.h new file +10
-0@@ -0,0 +1,51 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* Based on arch/arm/include/asm/jump_label.h +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +*
arch/arm64/include/asm/jump_label.h line 20
Quoted 10 lines
arch/arm64/include/asm/jump_label.h new file +10
-0+* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#ifndef _ASM_ARM64_JUMP_LABEL_H +#define _ASM_ARM64_JUMP_LABEL_H
Just use __ASM_JUMP_LABEL_H to keep consistent with out other headers.
arch/arm64/include/asm/jump_label.h line 25
Quoted 5 lines
arch/arm64/include/asm/jump_label.h new file +5
-0+#include <linux/types.h> + +#ifdef __KERNEL__ + +#define JUMP_LABEL_NOP_SIZE 4
This is the same as AARCH64_INSN_SIZE; just use the latter.
Quoted 58 lines
arch/arm64/include/asm/jump_label.h new file +26
-0+ +static __always_inline bool arch_static_branch(struct static_key *key) +{ + asm goto("1: nop\n\t" +".pushsection __jump_table, \"aw\"\n\t" +".align 3\n\t" +".quad 1b, %l[l_yes], %c0\n\t" +".popsection\n\t" + : :"i" (key) : : l_yes); + +return false; +l_yes: +return true; +} + +#endif /* __KERNEL__ */ + +typedef u64 jump_label_t ; + +struct jump_entry { +jump_label_t code; +jump_label_t target; +jump_label_t key; +}; + +#endif /* _ASM_ARM64_JUMP_LABEL_H */ arch/arm64/kernel/Makefile +1
-0@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m)arch/arm64/kernel/jump_label.c new file +14
-0@@ -0,0 +1,59 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* Based on arch/arm/kernel/jump_label.c +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details.
arch/arm64/kernel/jump_label.c line 24
Quoted 10 lines
arch/arm64/kernel/jump_label.c new file +10
-0+* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL
Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?
Will
Steven Rostedt
reviewerRe: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:
arch/arm64/kernel/jump_label.c line 24
Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6
-0+#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL Slightly worrying... this should be
CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?
In <linux/jump_label.h> we have:
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
[...]
# include <asm/jump_label.h> # define HAVE_JUMP_LABEL #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
-- Steve
Steven Rostedt
reviewerRe: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
Quoted 13 lines · On Fri, 13 Dec 2013 10:50:23 -0500 Steven Rostedt <[email protected]> wrote:
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:
arch/arm64/kernel/jump_label.c line 24
Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6
-0+#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL Slightly worrying... this should be
CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?
Quoted 7 lines
In <
linux/jump_label.h> we have:#if
defined(CC_HAVE_ASM_GOTO)&&defined(CONFIG_JUMP_LABEL)[...]
# include <asm/jump_label.h>
This also makes the include of <asm/jump_label.h> in the C file redundant.
-- Steve
Quoted 5 lines
# define HAVE_JUMP_LABEL #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */-- Steve
Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
Quoted 20 lines · Steven Rostedt
Quoted 17 lines · On Fri, 13 Dec 2013 10:50:23 -0500 Steven Rostedt <[email protected]> wrote:
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:
arch/arm64/kernel/jump_label.c line 24
Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6
-0+#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL Slightly worrying... this should be
CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?In <
linux/jump_label.h> we have:#if
defined(CC_HAVE_ASM_GOTO)&&defined(CONFIG_JUMP_LABEL)
Quoted 8 lines
Quoted 3 lines
[...]
# include <asm/jump_label.h>This also makes the include of <
asm/jump_label.h> in the C file redundant.-- Steve
Thanks, Steve.
Will remove the redundant include header file.
Quoted 7 lines
Quoted 6 lines
# define HAVE_JUMP_LABEL #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */-- Steve
Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
In reply to Steven RostedtQuoted 15 lines · Steven Rostedt
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:
arch/arm64/kernel/jump_label.c line 24
Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6
-0+#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL Slightly worrying... this should be
CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?In <
linux/jump_label.h> we have:
Quoted 8 lines
#if
defined(CC_HAVE_ASM_GOTO)&&defined(CONFIG_JUMP_LABEL)[...]
# include <asm/jump_label.h> # define HAVE_JUMP_LABEL #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
Phew, that explains things! Cheers, Steve.
Will
Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation
In reply to Will DeaconHi Will,
Thanks for review.
Quoted 10 lines · Will Deacon
Hi Jiang Liu,
Thanks for the updated patches! I still have some small comments, but I don't think they will cause you much work. See inline.
Quoted 4 lines · Jiang Liu
Optimize jump label implementation for ARM64 by dynamically patching kernel text.
Signed-off-by Jiang Liu <[email protected]>
...
Quoted 8 lines
arch/arm64/include/asm/jump_label.h line 20
Quoted 6 lines
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */arch/arm64/include/asm/jump_label.h new file +2
-0+#ifndef _ASM_ARM64_JUMP_LABEL_H +#define _ASM_ARM64_JUMP_LABEL_HJust use
__ASM_JUMP_LABEL_Hto keep consistent with out other headers.
Sure, will make this change.
Quoted 8 lines
arch/arm64/include/asm/jump_label.h line 25
Quoted 6 lines
arch/arm64/include/asm/jump_label.h new file +5
-0+#include <linux/types.h> + +#ifdef __KERNEL__ + +#define JUMP_LABEL_NOP_SIZE 4This is the same as
AARCH64_INSN_SIZE; just use the latter.
Yes, it's the same as AARCH64_INSN_SIZE but JUMP_LABEL_NOP_SIZE is needed by the jump label core. So I directly define JUMP_LABEL_NOP_SIZE as 4 instead of AARCH64_INSN_SIZE to avoid pulling asm/insn.h into asm/jump_label.h.
Thanks!
Gerry
Quoted 75 lines
Quoted 59 lines
arch/arm64/include/asm/jump_label.h new file +26
-0+ +static __always_inline bool arch_static_branch(struct static_key *key) +{ + asm goto("1: nop\n\t" +".pushsection __jump_table, \"aw\"\n\t" +".align 3\n\t" +".quad 1b, %l[l_yes], %c0\n\t" +".popsection\n\t" + : :"i" (key) : : l_yes); + +return false; +l_yes: +return true; +} + +#endif /* __KERNEL__ */ + +typedef u64 jump_label_t ; + +struct jump_entry { +jump_label_t code; +jump_label_t target; +jump_label_t key; +}; + +#endif /* _ASM_ARM64_JUMP_LABEL_H */ arch/arm64/kernel/Makefile +1
-0@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m)arch/arm64/kernel/jump_label.c new file +14
-0@@ -0,0 +1,59 @@ +/* +* Copyright (C) 2013 Huawei Ltd. +* Author: Jiang Liu <[email protected]> +* +* Based on arch/arm/kernel/jump_label.c +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. arch/arm64/kernel/jump_label.c line 24
Quoted 10 lines
arch/arm64/kernel/jump_label.c new file +10
-0+* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +#include <linux/kernel.h> +#include <linux/jump_label.h> +#include <asm/jump_label.h> +#include <asm/insn.h> + +#ifdef HAVE_JUMP_LABEL Slightly worrying... this should be
CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?Will
PATCH v6 7/7 jump_label: use defined macros instead of hard-coding for better readability
In reply to Jiang LiuUse macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.
--- include/linux/jump_label.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
include/linux/jump_label.h +10 -5
@@ -81,18 +81,21 @@ struct module;#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1ULstatic inline struct jump_entry *jump_label_get_entries (struct static_key *key) {return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TRUE_BRANCH ); + & ~JUMP_LABEL_TYPE_MASK ); }static inline bool jump_label_get_branch_default (struct static_key *key) { -if ((unsigned long )key->entries &JUMP_LABEL_TRUE_BRANCH ) +if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == +JUMP_LABEL_TYPE_TRUE_BRANCH )return true;return false; }
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })#else /* !HAVE_JUMP_LABEL */
Signature · 1 line
1.8.1.2
Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability
Quoted 7 lines · Jiang Liu
Use macro
JUMP_LABEL_TRUE_BRANCHinstead of hard-coding for better readability.Signed-off-by Jiang Liu <[email protected]>--- include/linux/jump_label.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
Whilst this looks alright to me, I think we probably need an ack from Steve in order to take changes to the core code.
Will
Quoted 47 lines
include/linux/jump_label.h +10
-5@@ -81,18 +81,21 @@ struct module;#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1ULstatic inline struct jump_entry *jump_label_get_entries (struct static_key *key) {return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TRUE_BRANCH ); + & ~JUMP_LABEL_TYPE_MASK ); }static inline bool jump_label_get_branch_default (struct static_key *key) { -if ((unsigned long )key->entries &JUMP_LABEL_TRUE_BRANCH ) +if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == +JUMP_LABEL_TYPE_TRUE_BRANCH )return true;return false; }@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })#else /* !HAVE_JUMP_LABEL */ Signature · 1 line
1.8.1.2
Steven Rostedt
reviewerRe: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability
[ added Jason Baron too (the original author) ]
Quoted 9 lines · On Fri, 13 Dec 2013 16:00:41 +0000 Will Deacon <[email protected]> wrote:
Quoted 7 lines · Jiang Liu
Use macro
JUMP_LABEL_TRUE_BRANCHinstead of hard-coding for better readability.Signed-off-by Jiang Liu <[email protected]>--- include/linux/jump_label.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
Quoted 5 lines
Whilst this looks alright to me, I think we probably need an ack from Steve in order to take changes to the core code.
Will
Thanks for the heads up. I'll take a look at it.
-- Steve
Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability
In reply to Jiang LiuQuoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:
Use macro
JUMP_LABEL_TRUE_BRANCHinstead of hard-coding for better readability.
Looks good, just a little nit. (see below)
Quoted 36 lines
Signed-off-by Jiang Liu <[email protected]>--- include/linux/jump_label.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)include/linux/jump_label.h +6
-3@@ -81,18 +81,21 @@ struct module;#include <linux/atomic.h> #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1ULstatic inline struct jump_entry *jump_label_get_entries (struct static_key *key) {return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TRUE_BRANCH ); + & ~JUMP_LABEL_TYPE_MASK ); }static inline bool jump_label_get_branch_default (struct static_key *key) { -if ((unsigned long )key->entries &JUMP_LABEL_TRUE_BRANCH ) +if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == +JUMP_LABEL_TYPE_TRUE_BRANCH )return true;return false; }
include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133
Quoted 10 lines
include/linux/jump_label.h +4
-2@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
Can we prettify the backslashes like:
#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), \
.entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), \
.entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })Other than that.
-- Steve
Quoted 3 lines
include/linux/jump_label.h +0
-0#else /* !HAVE_JUMP_LABEL */
Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability
Quoted 12 lines · Steven Rostedt
Quoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:
Use macro
JUMP_LABEL_TRUE_BRANCHinstead of hard-coding for better readability.Looks good, just a little nit. (see below)
Quoted 4 lines
Signed-off-by Jiang Liu <[email protected]>--- include/linux/jump_label.h | 15 ++++++++++-----
...
Quoted 22 lines
include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133
Quoted 11 lines
include/linux/jump_label.h +4
-2@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })Can we prettify the backslashes like:
#define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1), \ .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) #define STATIC_KEY_INIT_FALSE ((struct static_key) \ { .enabled = ATOMIC_INIT(0), \ .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
Quoted 3 lines
Other than that.
Acked-by Steven Rostedt <[email protected]>
Thanks Steve.
Will make this change.
Quoted 7 lines
-- Steve
Quoted 3 lines
include/linux/jump_label.h +0
-0#else /* !HAVE_JUMP_LABEL */
Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability
Quoted 33 lines · Jiang Liu
Quoted 10 lines · Steven Rostedt
Quoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:
Use macro
JUMP_LABEL_TRUE_BRANCHinstead of hard-coding for better readability.Looks good, just a little nit. (see below)
Quoted 3 lines
Signed-off-by Jiang Liu <[email protected]>--- include/linux/jump_label.h | 15 ++++++++++-----...
Quoted 21 lines
include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133
Quoted 11 lines
include/linux/jump_label.h +4
-2@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);extern void jump_label_apply_nops (struct module *mod);#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })Can we prettify the backslashes like:
#define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1), \ .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) #define STATIC_KEY_INIT_FALSE ((struct static_key) \ { .enabled = ATOMIC_INIT(0), \ .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
Quoted 5 lines
Quoted 3 lines
Other than that.
Acked-by Steven Rostedt <[email protected]>Thanks Steve.
Will make this change.
Works for me too, feel free to add:
Thanks,
-Jason