Skip to contentSkip to conversation

Optimize jump label implementation for ARM64

PATCH Superseded Jiang Liu 4 participants 21 messages Dec 10, 2013 – Dec 18, 2013

Subsystems (2)

v6 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Jiang Liu

author
#0 lore raw

v6 cover Optimize jump label implementation for ARM64

This patchset tries to optimize arch specfic jump label implementation for ARM64 by dynamic kernel text patching.

To enable this feature, your toolchain must support "asm goto" extension and "%c" constraint extesion. Current GCC for AARCH64 doesn't support "%c", and there's a patch for it now:
http://gcc.gnu.org/ml/gcc-patches/2013-10/msg01314.html

It has been tested on ARM Fast mode and a real hardware platform.

Any comments are welcomed!

V5->V6:
1) optimize stop_machine() related code
2) simplify aarch64_insn_patch_text_nosync() interface
3) refine comments

V4->V5:
1) rework the stop_machine() related code
2) rework aarch64_insn_gen_nop()

V3->V4:
1) resolve a race condition in kernel text patching
2) address other review comments

V2->V3:
1) fix a bug in comparing signed and unsigned values
2) detect big endian by checking __AARCH64EB__

V1->V2: address review comments of V1
1) refine comments
2) add a new interface to always synchronize with stop_machine() when patching code
3) handle endian issue when patching code

Jiang Liu (7):
arm64: introduce basic aarch64 instruction decoding helpers
arm64: introduce interfaces to hotpatch kernel and module code
arm64: move encode_insn_immediate() from module.c to insn.c
arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64, jump label: detect %c support for ARM64
arm64, jump label: optimize jump label implementation
jump_label: use defined macros instead of hard-coding for better
readability

 arch/arm64/Kconfig                  |   1 +
 arch/arm64/include/asm/insn.h       | 108 +++++++++++++
 arch/arm64/include/asm/jump_label.h |  51 ++++++
 arch/arm64/kernel/Makefile          |   3 +-
 arch/arm64/kernel/insn.c            | 302 ++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/jump_label.c      |  59 +++++++
 arch/arm64/kernel/module.c          | 157 ++++++-------------
 include/linux/jump_label.h          |  15 +-
 scripts/gcc-goto.sh                 |   2 +-
 9 files changed, 581 insertions(+), 117 deletions(-)
 create mode 100644 arch/arm64/include/asm/insn.h
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/insn.c
 create mode 100644 arch/arm64/kernel/jump_label.c
Signature · 1 line
1.8.1.2
author
#1 lore raw

PATCH v6 1/7 arm64: introduce basic aarch64 instruction decoding helpers

Introduce basic aarch64 instruction decoding helper aarch64_get_insn_class() and aarch64_insn_hotpatch_safe().

---
 arch/arm64/include/asm/insn.h | 77 ++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile    |  2 +-
 arch/arm64/kernel/insn.c      | 91 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 169 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/include/asm/insn.h
 create mode 100644 arch/arm64/kernel/insn.c
arch/arm64/include/asm/insn.h new file +77 -0
new file mode 100644
index 0000000..1bdc44c
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef	__ASM_INSN_H
+#define	__ASM_INSN_H
+#include <linux/types.h>
+
+/*
+ * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
+ * Section C3.1 "A64 instruction index by encoding":
+ * AArch64 main encoding table
+ *  Bit position
+ *   28 27 26 25	Encoding Group
+ *   0  0  -  -		Unallocated
+ *   1  0  0  -		Data processing, immediate
+ *   1  0  1  -		Branch, exception generation and system instructions
+ *   -  1  -  0		Loads and stores
+ *   -  1  0  1		Data processing - register
+ *   0  1  1  1		Data processing - SIMD and floating point
+ *   1  1  1  1		Data processing - SIMD and floating point
+ * "-" means "don't care"
+ */
+enum aarch64_insn_encoding_class {
+	AARCH64_INSN_CLS_UNKNOWN,	/* UNALLOCATED */
+	AARCH64_INSN_CLS_DP_IMM,	/* Data processing - immediate */
+	AARCH64_INSN_CLS_DP_REG,	/* Data processing - register */
+	AARCH64_INSN_CLS_DP_FPSIMD,	/* Data processing - SIMD and FP */
+	AARCH64_INSN_CLS_LDST,		/* Loads and stores */
+	AARCH64_INSN_CLS_BR_SYS,	/* Branch, exception generation and
+					 * system instructions */
+};
+
+enum aarch64_insn_hint_op {
+	AARCH64_INSN_HINT_NOP	= 0x0 << 5,
+	AARCH64_INSN_HINT_YIELD	= 0x1 << 5,
+	AARCH64_INSN_HINT_WFE	= 0x2 << 5,
+	AARCH64_INSN_HINT_WFI	= 0x3 << 5,
+	AARCH64_INSN_HINT_SEV	= 0x4 << 5,
+	AARCH64_INSN_HINT_SEVL	= 0x5 << 5,
+};
+
+#define	__AARCH64_INSN_FUNCS(abbr, mask, val)	\
+static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
+{ return (code & (mask)) == (val); } \
+static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \
+{ return (val); }
+
+__AARCH64_INSN_FUNCS(b,		0xFC000000, 0x14000000)
+__AARCH64_INSN_FUNCS(bl,	0xFC000000, 0x94000000)
+__AARCH64_INSN_FUNCS(svc,	0xFFE0001F, 0xD4000001)
+__AARCH64_INSN_FUNCS(hvc,	0xFFE0001F, 0xD4000002)
+__AARCH64_INSN_FUNCS(smc,	0xFFE0001F, 0xD4000003)
+__AARCH64_INSN_FUNCS(brk,	0xFFE0001F, 0xD4200000)
+__AARCH64_INSN_FUNCS(hint,	0xFFFFF01F, 0xD503201F)
+
+#undef	__AARCH64_INSN_FUNCS
+
+bool aarch64_insn_is_nop(u32 insn);
+
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+
+bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
+
+#endif	/* __ASM_INSN_H */
arch/arm64/kernel/Makefile +1 -1
index 5ba2fd4..cd95a25 100644
@@ -9,7 +9,7 @@ AFLAGS_head.o		:= -DTEXT_OFFSET=$(TEXT_OFFSET)
 arm64-obj-y		:= cputable.o debug-monitors.o entry.o irq.o fpsimd.o	\
 			   entry-fpsimd.o process.o ptrace.o setup.o signal.o	\
 			   sys.o stacktrace.o time.o traps.o io.o vdso.o	\
-			   hyp-stub.o psci.o cpu_ops.o
+			   hyp-stub.o psci.o cpu_ops.o insn.o
 
 arm64-obj-$(CONFIG_COMPAT)		+= sys32.o kuser32.o signal32.o 	\
 					   sys_compat.o
arch/arm64/kernel/insn.c new file +91 -0

91 lines not on this page · raw message

Signature · 1 line
1.8.1.2
author
#2 lore raw

PATCH v6 2/7 arm64: introduce interfaces to hotpatch kernel and module code

In reply to Jiang Liu

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

---
 arch/arm64/include/asm/insn.h |  10 +++-
 arch/arm64/kernel/insn.c      | 117 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +9 -1
index 1bdc44c..bf8085f 100644
@@ -18,6 +18,9 @@
 #define	__ASM_INSN_H
 #include <linux/types.h>
 
+/* A64 instructions are always 32 bits. */
+#define	AARCH64_INSN_SIZE		4
+
 /*
  * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
  * Section C3.1 "A64 instruction index by encoding":
@@ -70,8 +73,13 @@ __AARCH64_INSN_FUNCS(hint,	0xFFFFF01F, 0xD503201F)
 
 bool aarch64_insn_is_nop(u32 insn);
 
+int aarch64_insn_read(void *addr, u32 *insnp);
+int aarch64_insn_write(void *addr, u32 insn);
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
-
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
+int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
+
 #endif	/* __ASM_INSN_H */
arch/arm64/kernel/insn.c +117 -0

132 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#3 lore raw

Re: [PATCH v6 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 9 lines · Jiang Liu

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

Quoted 5 lines
---
 arch/arm64/include/asm/insn.h |  10 +++-
 arch/arm64/kernel/insn.c      | 117 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)

Two comments on the patching callback...

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 154

Quoted 12 lines
arch/arm64/kernel/insn.c +12 -0
+static int __kprobes aarch64_insn_patch_text_cb(void *arg)
+{
+	int i, ret = 0;
+	struct aarch64_insn_patch *pp = arg;
+
+	/* The first CPU becomes master */
+	if (atomic_inc_return(&pp->cpu_count) == 1) {
+		for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
+			ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
+							     pp->new_insns[i]);
+		/* Make sure changes are visible to other CPUs and let them continue */
+		smp_wmb();

As I mentioned last time, you can rely on aarch64_insn_patch_text_nosync to enforce ordering (flush_icache_range ends with dsb; isb). Just add a comment saying that.

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 159

Quoted 5 lines
arch/arm64/kernel/insn.c +5 -0
+		atomic_set(&pp->cpu_count, -1);
+	} else {
+		while (atomic_read(&pp->cpu_count) != -1)
+			cpu_relax();
+		smp_rmb();

You don't need the smp_rmb here.

Will

author
#4 lore raw

PATCH v6 3/7 arm64: move encode_insn_immediate() from module.c to insn.c

In reply to Jiang Liu

Function encode_insn_immediate() will be used by other instruction manipulate related functions, so move it into insn.c and rename it as aarch64_insn_encode_immediate().

---
 arch/arm64/include/asm/insn.h |  13 ++++
 arch/arm64/kernel/insn.c      |  54 +++++++++++++++
 arch/arm64/kernel/module.c    | 157 +++++++++++++-----------------------------
 3 files changed, 114 insertions(+), 110 deletions(-)
arch/arm64/include/asm/insn.h +13 -0
index bf8085f..fb44660 100644
@@ -55,6 +55,17 @@ enum aarch64_insn_hint_op {
 	AARCH64_INSN_HINT_SEVL	= 0x5 << 5,
 };
 
+enum aarch64_insn_imm_type {
+	AARCH64_INSN_IMM_ADR,
+	AARCH64_INSN_IMM_26,
+	AARCH64_INSN_IMM_19,
+	AARCH64_INSN_IMM_16,
+	AARCH64_INSN_IMM_14,
+	AARCH64_INSN_IMM_12,
+	AARCH64_INSN_IMM_9,
+	AARCH64_INSN_IMM_MAX
+};
+
 #define	__AARCH64_INSN_FUNCS(abbr, mask, val)	\
 static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
 { return (code & (mask)) == (val); } \
@@ -76,6 +87,8 @@ bool aarch64_insn_is_nop(u32 insn);
 int aarch64_insn_read(void *addr, u32 *insnp);
 int aarch64_insn_write(void *addr, u32 insn);
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
+				  u32 insn, u64 imm);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +54 -0

57 lines not on this page · raw message

arch/arm64/kernel/module.c +47 -110

303 lines not on this page · raw message

Signature · 1 line
1.8.1.2
author
#5 lore raw

PATCH v6 4/7 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

In reply to Jiang Liu

Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.

---
 arch/arm64/include/asm/insn.h | 10 ++++++++++
 arch/arm64/kernel/insn.c      | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)
arch/arm64/include/asm/insn.h +10 -0
index fb44660..c44ad39 100644
@@ -66,6 +66,11 @@ enum aarch64_insn_imm_type {
 	AARCH64_INSN_IMM_MAX
 };
 
+enum aarch64_insn_branch_type {
+	AARCH64_INSN_BRANCH_NOLINK,
+	AARCH64_INSN_BRANCH_LINK,
+};
+
 #define	__AARCH64_INSN_FUNCS(abbr, mask, val)	\
 static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
 { return (code & (mask)) == (val); } \
@@ -89,6 +94,11 @@ int aarch64_insn_write(void *addr, u32 insn);
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 				  u32 insn, u64 imm);
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+				enum aarch64_insn_branch_type type);
+u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op);
+u32 aarch64_insn_gen_nop(void);
+
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +40 -0
index 03e9a5f..ead3239 100644
@@ -14,6 +14,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
@@ -260,3 +261,42 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 
 	return insn;
 }
+
+u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+					  enum aarch64_insn_branch_type type)
+{
+	u32 insn;
+	long offset;
+
+	/*
+	 * PC: A 64-bit Program Counter holding the address of the current
+	 * instruction. A64 instructions must be word-aligned.
+	 */
+	BUG_ON((pc & 0x3) || (addr & 0x3));
+
+	/*
+	 * B/BL support [-128M, 128M) offset
+	 * ARM64 virtual address arrangement guarantees all kernel and module
+	 * texts are within +/-128M.
+	 */
+	offset = ((long)addr - (long)pc);
+	BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
+
+	if (type == AARCH64_INSN_BRANCH_LINK)
+		insn = aarch64_insn_get_bl_value();
+	else
+		insn = aarch64_insn_get_b_value();
+
+	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
+					     offset >> 2);
+}
+
+u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
+{
+	return aarch64_insn_get_hint_value() | op;
+}
+
+u32 __kprobes aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}
Signature · 1 line
1.8.1.2
Reviewed-by on v6 4/7
#6 lore raw

Re: [PATCH v6 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

Quoted 4 lines · Jiang Liu

Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.

Looks good to me:

Thanks!

Will

author
#7 lore raw

PATCH v6 5/7 arm64, jump label: detect %c support for ARM64

In reply to Jiang Liu

As commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.

Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.

---
 scripts/gcc-goto.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/gcc-goto.sh +1 -1
index a2af2e8..c9469d3 100644
@@ -5,7 +5,7 @@
 cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
 int main(void)
 {
-#ifdef __arm__
+#if defined(__arm__) || defined(__aarch64__)
 	/*
 	 * Not related to asm goto, but used by jump label
 	 * and broken on some ARM GCC versions (see GCC Bug 48637).
Signature · 1 line
1.8.1.2
author
#8 lore raw

PATCH v6 6/7 arm64, jump label: optimize jump label implementation

In reply to Jiang Liu

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

---
 arch/arm64/Kconfig                  |  1 +
 arch/arm64/include/asm/jump_label.h | 51 ++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile          |  1 +
 arch/arm64/kernel/jump_label.c      | 59 +++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
index 6d4dd22..5f962e8 100644
@@ -19,6 +19,7 @@ config ARM64
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
+	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +51 -0

51 lines not on this page · raw message

arch/arm64/kernel/Makefile +1 -0
index cd95a25..e155d56 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +59 -0

59 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#9 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

Hi Jiang Liu,

Thanks for the updated patches! I still have some small comments, but I don't think they will cause you much work. See inline.

Quoted 41 lines · Jiang Liu

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

---
 arch/arm64/Kconfig                  |  1 +
 arch/arm64/include/asm/jump_label.h | 51 ++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile          |  1 +
 arch/arm64/kernel/jump_label.c      | 59 +++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
index 6d4dd22..5f962e8 100644
@@ -19,6 +19,7 @@ config ARM64
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
+	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +10 -0
new file mode 100644
index 0000000..ca95a3a
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/include/asm/jump_label.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *

arch/arm64/include/asm/jump_label.h line 20

Quoted 10 lines
arch/arm64/include/asm/jump_label.h new file +10 -0
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _ASM_ARM64_JUMP_LABEL_H
+#define _ASM_ARM64_JUMP_LABEL_H

Just use __ASM_JUMP_LABEL_H to keep consistent with out other headers.

arch/arm64/include/asm/jump_label.h line 25

Quoted 5 lines
arch/arm64/include/asm/jump_label.h new file +5 -0
+#include <linux/types.h>
+
+#ifdef __KERNEL__
+
+#define JUMP_LABEL_NOP_SIZE 4

This is the same as AARCH64_INSN_SIZE; just use the latter.

Quoted 58 lines
arch/arm64/include/asm/jump_label.h new file +26 -0
+
+static __always_inline bool arch_static_branch(struct static_key *key)
+{
+	asm goto("1: nop\n\t"
+		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 3\n\t"
+		 ".quad 1b, %l[l_yes], %c0\n\t"
+		 ".popsection\n\t"
+		 :  :  "i"(key) :  : l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#endif /* __KERNEL__ */
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif	/* _ASM_ARM64_JUMP_LABEL_H */
arch/arm64/kernel/Makefile +1 -0
index cd95a25..e155d56 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +14 -0
new file mode 100644
index 0000000..b13ef62
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/kernel/jump_label.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.

arch/arm64/kernel/jump_label.c line 24

Quoted 10 lines
arch/arm64/kernel/jump_label.c new file +10 -0
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

Will

#10 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/kernel/jump_label.c line 24

Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

In <linux/jump_label.h> we have:

#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)

[...]

# include <asm/jump_label.h>
# define HAVE_JUMP_LABEL
#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */

-- Steve

#11 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

Quoted 13 lines · On Fri, 13 Dec 2013 10:50:23 -0500 Steven Rostedt <[email protected]> wrote:
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/kernel/jump_label.c line 24

Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

Quoted 7 lines

In <linux/jump_label.h> we have:

#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)

[...]

# include <asm/jump_label.h>

This also makes the include of <asm/jump_label.h> in the C file redundant.

-- Steve

Quoted 5 lines
# define HAVE_JUMP_LABEL
#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */

-- Steve

author
#12 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

Quoted 20 lines · Steven Rostedt
Quoted 17 lines · On Fri, 13 Dec 2013 10:50:23 -0500 Steven Rostedt <[email protected]> wrote:
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/kernel/jump_label.c line 24

Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

In <linux/jump_label.h> we have:

#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)

Quoted 8 lines
Quoted 3 lines

[...]

# include <asm/jump_label.h>

This also makes the include of <asm/jump_label.h> in the C file redundant.

-- Steve

Thanks, Steve.
Will remove the redundant include header file.

Quoted 7 lines
Quoted 6 lines
# define HAVE_JUMP_LABEL
#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */

-- Steve

#13 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

In reply to Steven Rostedt
Quoted 15 lines · Steven Rostedt
Quoted 9 lines · On Fri, 13 Dec 2013 15:41:04 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/kernel/jump_label.c line 24

Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

In <linux/jump_label.h> we have:

Quoted 8 lines

#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)

[...]

# include <asm/jump_label.h>
# define HAVE_JUMP_LABEL
#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */

Phew, that explains things! Cheers, Steve.

Will

author
#14 lore raw

Re: [PATCH v6 6/7] arm64, jump label: optimize jump label implementation

In reply to Will Deacon

Hi Will,
Thanks for review.

Quoted 10 lines · Will Deacon

Hi Jiang Liu,

Thanks for the updated patches! I still have some small comments, but I don't think they will cause you much work. See inline.

Quoted 4 lines · Jiang Liu

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

...

Quoted 8 lines

arch/arm64/include/asm/jump_label.h line 20

Quoted 6 lines

+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

arch/arm64/include/asm/jump_label.h new file +2 -0
+#ifndef _ASM_ARM64_JUMP_LABEL_H
+#define _ASM_ARM64_JUMP_LABEL_H

Just use __ASM_JUMP_LABEL_H to keep consistent with out other headers.

Sure, will make this change.

Quoted 8 lines

arch/arm64/include/asm/jump_label.h line 25

Quoted 6 lines
arch/arm64/include/asm/jump_label.h new file +5 -0
+#include <linux/types.h>
+
+#ifdef __KERNEL__
+
+#define JUMP_LABEL_NOP_SIZE 4

This is the same as AARCH64_INSN_SIZE; just use the latter.

Yes, it's the same as AARCH64_INSN_SIZE but JUMP_LABEL_NOP_SIZE is needed by the jump label core. So I directly define JUMP_LABEL_NOP_SIZE as 4 instead of AARCH64_INSN_SIZE to avoid pulling asm/insn.h into asm/jump_label.h.

Thanks!
Gerry

Quoted 75 lines
Quoted 59 lines
arch/arm64/include/asm/jump_label.h new file +26 -0
+
+static __always_inline bool arch_static_branch(struct static_key *key)
+{
+	asm goto("1: nop\n\t"
+		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 3\n\t"
+		 ".quad 1b, %l[l_yes], %c0\n\t"
+		 ".popsection\n\t"
+		 :  :  "i"(key) :  : l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#endif /* __KERNEL__ */
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif	/* _ASM_ARM64_JUMP_LABEL_H */
arch/arm64/kernel/Makefile +1 -0
index cd95a25..e155d56 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +14 -0
new file mode 100644
index 0000000..b13ef62
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/kernel/jump_label.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.

arch/arm64/kernel/jump_label.c line 24

Quoted 10 lines
arch/arm64/kernel/jump_label.c new file +10 -0
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL

Slightly worrying... this should be CONFIG_HAVE_JUMP_LABEL, right? How did you manage to test this code?

Will

author
#15 lore raw

PATCH v6 7/7 jump_label: use defined macros instead of hard-coding for better readability

In reply to Jiang Liu

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

---
 include/linux/jump_label.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
include/linux/jump_label.h +10 -5
index 3999977..192ccba 100644
@@ -81,18 +81,21 @@ struct module;
 #include <linux/atomic.h>
 #ifdef HAVE_JUMP_LABEL
 
-#define JUMP_LABEL_TRUE_BRANCH 1UL
+#define JUMP_LABEL_TYPE_FALSE_BRANCH	0UL
+#define JUMP_LABEL_TYPE_TRUE_BRANCH	1UL
+#define JUMP_LABEL_TYPE_MASK		1UL
 
 static
 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 {
 	return (struct jump_entry *)((unsigned long)key->entries
-						& ~JUMP_LABEL_TRUE_BRANCH);
+						& ~JUMP_LABEL_TYPE_MASK);
 }
 
 static inline bool jump_label_get_branch_default(struct static_key *key)
 {
-	if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+	if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
+	    JUMP_LABEL_TYPE_TRUE_BRANCH)
 		return true;
 	return false;
 }
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
 
 #else  /* !HAVE_JUMP_LABEL */
 
Signature · 1 line
1.8.1.2
#16 lore raw

Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability

Quoted 7 lines · Jiang Liu

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

---
 include/linux/jump_label.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

Whilst this looks alright to me, I think we probably need an ack from Steve in order to take changes to the core code.

Will

Quoted 47 lines
include/linux/jump_label.h +10 -5
index 3999977..192ccba 100644
@@ -81,18 +81,21 @@ struct module;
 #include <linux/atomic.h>
 #ifdef HAVE_JUMP_LABEL
 
-#define JUMP_LABEL_TRUE_BRANCH 1UL
+#define JUMP_LABEL_TYPE_FALSE_BRANCH	0UL
+#define JUMP_LABEL_TYPE_TRUE_BRANCH	1UL
+#define JUMP_LABEL_TYPE_MASK		1UL
 
 static
 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 {
 	return (struct jump_entry *)((unsigned long)key->entries
-						& ~JUMP_LABEL_TRUE_BRANCH);
+						& ~JUMP_LABEL_TYPE_MASK);
 }
 
 static inline bool jump_label_get_branch_default(struct static_key *key)
 {
-	if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+	if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
+	    JUMP_LABEL_TYPE_TRUE_BRANCH)
 		return true;
 	return false;
 }
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
 
 #else  /* !HAVE_JUMP_LABEL */
 
Signature · 1 line
1.8.1.2
#17 lore raw

Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability

[ added Jason Baron too (the original author) ]

Quoted 9 lines · On Fri, 13 Dec 2013 16:00:41 +0000 Will Deacon <[email protected]> wrote:
Quoted 7 lines · Jiang Liu

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

---
 include/linux/jump_label.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
Quoted 5 lines

Whilst this looks alright to me, I think we probably need an ack from Steve in order to take changes to the core code.

Will

Thanks for the heads up. I'll take a look at it.

-- Steve

reviewer Acked-by on v6 7/7
#18 lore raw

Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability

In reply to Jiang Liu
Quoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

Looks good, just a little nit. (see below)

Quoted 36 lines
---
 include/linux/jump_label.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
include/linux/jump_label.h +6 -3
index 3999977..192ccba 100644
@@ -81,18 +81,21 @@ struct module;
 #include <linux/atomic.h>
 #ifdef HAVE_JUMP_LABEL
 
-#define JUMP_LABEL_TRUE_BRANCH 1UL
+#define JUMP_LABEL_TYPE_FALSE_BRANCH	0UL
+#define JUMP_LABEL_TYPE_TRUE_BRANCH	1UL
+#define JUMP_LABEL_TYPE_MASK		1UL
 
 static
 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 {
 	return (struct jump_entry *)((unsigned long)key->entries
-						& ~JUMP_LABEL_TRUE_BRANCH);
+						& ~JUMP_LABEL_TYPE_MASK);
 }
 
 static inline bool jump_label_get_branch_default(struct static_key *key)
 {
-	if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+	if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
+	    JUMP_LABEL_TYPE_TRUE_BRANCH)
 		return true;
 	return false;
 }

include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133

Quoted 10 lines
include/linux/jump_label.h +4 -2
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })

Can we prettify the backslashes like:

#define STATIC_KEY_INIT_TRUE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(1),				\
	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
#define STATIC_KEY_INIT_FALSE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(0),				\
	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })

Other than that.

-- Steve

Quoted 3 lines
include/linux/jump_label.h +0 -0
 #else  /* !HAVE_JUMP_LABEL */
 
author
#19 lore raw

Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability

Quoted 12 lines · Steven Rostedt
Quoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

Looks good, just a little nit. (see below)

Quoted 4 lines
---
 include/linux/jump_label.h | 15 ++++++++++-----

...

Quoted 22 lines

include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133

Quoted 11 lines
include/linux/jump_label.h +4 -2
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })

Can we prettify the backslashes like:

#define STATIC_KEY_INIT_TRUE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(1),				\
	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
#define STATIC_KEY_INIT_FALSE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(0),				\
	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
Quoted 3 lines

Other than that.

Thanks Steve.
Will make this change.

Quoted 7 lines

-- Steve

Quoted 3 lines
include/linux/jump_label.h +0 -0
 #else  /* !HAVE_JUMP_LABEL */
 
maintainer Acked-by
#20 lore raw

Re: [PATCH v6 7/7] jump_label: use defined macros instead of hard-coding for better readability

Quoted 33 lines · Jiang Liu
Quoted 10 lines · Steven Rostedt
Quoted 2 lines · On Wed, 11 Dec 2013 00:03:53 +0800 Jiang Liu <[email protected]> wrote:

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

Looks good, just a little nit. (see below)

Quoted 3 lines
---
 include/linux/jump_label.h | 15 ++++++++++-----

...

Quoted 21 lines

include/linux/jump_label.h extern void static_key_slow_dec(struct static_key *key); · line 133

Quoted 11 lines
include/linux/jump_label.h +4 -2
@@ -123,9 +126,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })

Can we prettify the backslashes like:

#define STATIC_KEY_INIT_TRUE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(1),				\
	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
#define STATIC_KEY_INIT_FALSE ((struct static_key)		\
	{ .enabled = ATOMIC_INIT(0),				\
	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
Quoted 5 lines
Quoted 3 lines

Other than that.

Thanks Steve.
Will make this change.

Works for me too, feel free to add:

Thanks,

-Jason