Skip to contentSkip to conversation

Optimize jump label implementation for ARM64

PATCH Superseded Jiang Liu 6 participants 27 messages Oct 18, 2013 – Nov 28, 2013

Subsystems (2)

v5 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Jiang Liu

author
#0 lore raw

v5 cover Optimize jump label implementation for ARM64

From: Jiang Liu <[email protected]>

This patchset tries to optimize arch specfic jump label implementation for ARM64 by dynamic kernel text patching.

To enable this feature, your toolchain must support "asm goto" extension and "%c" constraint extesion. Current GCC for AARCH64 doesn't support "%c", and there's a patch for it now:
http://gcc.gnu.org/ml/gcc-patches/2013-10/msg01314.html

It has been tested on ARM Fast mode and a real hardware platform.

Any comments are welcomed!

V4->V5:
1) rework the stop_machine() related code
2) rework aarch64_insn_gen_nop()

V3->V4:
1) resolve a race condition in kernel text patching
2) address other review comments

V2->V3:
1) fix a bug in comparing signed and unsigned values
2) detect big endian by checking __AARCH64EB__

V1->V2: address review comments of V1
1) refine comments
2) add a new interface to always synchronize with stop_machine() when patching code
3) handle endian issue when patching code

Jiang Liu (7):
arm64: introduce basic aarch64 instruction decoding helpers
arm64: introduce interfaces to hotpatch kernel and module code
arm64: move encode_insn_immediate() from module.c to insn.c
arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64, jump label: detect %c support for ARM64
arm64, jump label: optimize jump label implementation
jump_label: use defined macros instead of hard-coding for better
readability

 arch/arm64/Kconfig                  |   1 +
 arch/arm64/include/asm/insn.h       | 121 +++++++++++++++++
 arch/arm64/include/asm/jump_label.h |  52 +++++++
 arch/arm64/kernel/Makefile          |   3 +-
 arch/arm64/kernel/insn.c            | 263 ++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/jump_label.c      |  58 ++++++++
 arch/arm64/kernel/module.c          | 153 +++++++--------------
 include/linux/jump_label.h          |  15 +-
 scripts/gcc-goto.sh                 |   2 +-
 9 files changed, 554 insertions(+), 114 deletions(-)
 create mode 100644 arch/arm64/include/asm/insn.h
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/insn.c
 create mode 100644 arch/arm64/kernel/jump_label.c
Signature · 1 line
1.8.1.2
author
#1 lore raw

PATCH v5 1/7 arm64: introduce basic aarch64 instruction decoding helpers

From: Jiang Liu <[email protected]>

Introduce basic aarch64 instruction decoding helper aarch64_get_insn_class() and aarch64_insn_hotpatch_safe().

---
 arch/arm64/include/asm/insn.h | 78 +++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile    |  2 +-
 arch/arm64/kernel/insn.c      | 90 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 169 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/include/asm/insn.h
 create mode 100644 arch/arm64/kernel/insn.c
arch/arm64/include/asm/insn.h new file +78 -0
new file mode 100644
index 0000000..7499490
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef	__ASM_INSN_H
+#define	__ASM_INSN_H
+#include <linux/types.h>
+
+/*
+ * ARM Architecture Reference Manual ARMv8, Section C3.1
+ * AArch64 main encoding table
+ *  Bit position
+ *   28 27 26 25	Encoding Group
+ *   0  0  -  -		Unallocated
+ *   1  0  0  -		Data processing, immediate
+ *   1  0  1  -		Branch, exception generation and system instructions
+ *   -  1  -  0		Loads and stores
+ *   -  1  0  1		Data processing - register
+ *   0  1  1  1		Data processing - SIMD and floating point
+ *   1  1  1  1		Data processing - SIMD and floating point
+ * "-" means "don't care"
+ */
+enum aarch64_insn_encoding_class {
+	AARCH64_INSN_CLS_UNKNOWN,	/* UNALLOCATED */
+	AARCH64_INSN_CLS_DP_IMM,	/* Data processing - immediate */
+	AARCH64_INSN_CLS_DP_REG,	/* Data processing - register */
+	AARCH64_INSN_CLS_DP_FPSIMD,	/* Data processing - SIMD and FP */
+	AARCH64_INSN_CLS_LDST,		/* Loads and stores */
+	AARCH64_INSN_CLS_BR_SYS,	/* Branch, exception generation and
+					 * system instructions */
+};
+
+#define	__AARCH64_INSN_FUNCS(abbr, mask, val)	\
+static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
+{ return (code & (mask)) == (val); }	\
+static __always_inline u32 aarch64_insn_get_##abbr##_mask(void) \
+{ return (mask); } \
+static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \
+{ return (val); }
+
+__AARCH64_INSN_FUNCS(b,		0xFC000000, 0x14000000)
+__AARCH64_INSN_FUNCS(bl,	0xFC000000, 0x94000000)
+__AARCH64_INSN_FUNCS(svc,	0xFFE0001F, 0xD4000001)
+__AARCH64_INSN_FUNCS(hvc,	0xFFE0001F, 0xD4000002)
+__AARCH64_INSN_FUNCS(smc,	0xFFE0001F, 0xD4000003)
+__AARCH64_INSN_FUNCS(brk,	0xFFE0001F, 0xD4200000)
+__AARCH64_INSN_FUNCS(hint,	0xFFFFF01F, 0xD503201F)
+
+#undef	__AARCH64_INSN_FUNCS
+
+enum aarch64_insn_hint_op {
+	AARCH64_INSN_HINT_NOP	= 0x0 << 5,
+	AARCH64_INSN_HINT_YIELD	= 0x1 << 5,
+	AARCH64_INSN_HINT_WFE	= 0x2 << 5,
+	AARCH64_INSN_HINT_WFI	= 0x3 << 5,
+	AARCH64_INSN_HINT_SEV	= 0x4 << 5,
+	AARCH64_INSN_HINT_SEVL	= 0x5 << 5,
+};
+
+bool aarch64_insn_is_nop(u32 insn);
+
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+
+bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
+
+#endif	/* __ASM_INSN_H */
arch/arm64/kernel/Makefile +1 -1
index 7b4b564..9af6cb3 100644
@@ -9,7 +9,7 @@ AFLAGS_head.o		:= -DTEXT_OFFSET=$(TEXT_OFFSET)
 arm64-obj-y		:= cputable.o debug-monitors.o entry.o irq.o fpsimd.o	\
 			   entry-fpsimd.o process.o ptrace.o setup.o signal.o	\
 			   sys.o stacktrace.o time.o traps.o io.o vdso.o	\
-			   hyp-stub.o psci.o
+			   hyp-stub.o psci.o insn.o
 
 arm64-obj-$(CONFIG_COMPAT)		+= sys32.o kuser32.o signal32.o 	\
 					   sys_compat.o
arch/arm64/kernel/insn.c new file +90 -0

90 lines not on this page · raw message

Signature · 1 line
1.8.1.2
maintainer
#2 lore raw

Re: [PATCH v5 1/7] arm64: introduce basic aarch64 instruction decoding helpers

Quoted 2 lines · Jiang Liu

+/*
+ * ARMv8-A Section B2.6.5:

Minor comment - please specify the document revision and the title of the section as they keep changing in subsequent ARMv8 ARM updates and this comment would no longer match.

Signature · 1 line
Catalin
author
#3 lore raw

Re: [PATCH v5 1/7] arm64: introduce basic aarch64 instruction decoding helpers

Quoted 8 lines · Catalin Marinas
Quoted 2 lines · Jiang Liu

+/*
+ * ARMv8-A Section B2.6.5:

Minor comment - please specify the document revision and the title of the section as they keep changing in subsequent ARMv8 ARM updates and this comment would no longer match.

Thanks for review, will improve it in next version.

author
#4 lore raw

PATCH v5 2/7 arm64: introduce interfaces to hotpatch kernel and module code

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

---
 arch/arm64/include/asm/insn.h | 19 ++++++++-
 arch/arm64/kernel/insn.c      | 91 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +18 -1
index 7499490..7a69491 100644
@@ -71,8 +71,25 @@ enum aarch64_insn_hint_op {
 
 bool aarch64_insn_is_nop(u32 insn);
 
-enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+/*
+ * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
+ * little-endian.
+ */
+static __always_inline u32 aarch64_insn_read(void *addr)
+{
+	return le32_to_cpu(*(u32 *)addr);
+}
 
+static __always_inline void aarch64_insn_write(void *addr, u32 insn)
+{
+	*(u32 *)addr = cpu_to_le32(insn);
+}
+
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
+int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
+
 #endif	/* __ASM_INSN_H */
arch/arm64/kernel/insn.c +91 -0

100 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#5 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Hi Jinag Liu,

Sorry for the delayed review, I've been travelling.

Quoted 1 line · Jiang Liu

From: Jiang Liu <[email protected]>

If I try and email you at your Huawei address, I get a bounce from the mail server. Is that expected? If so, it's not very helpful from a commit log perspective if you use that address as the author on your patches.

Quoted 28 lines

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

---
 arch/arm64/include/asm/insn.h | 19 ++++++++-
 arch/arm64/kernel/insn.c      | 91 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +3 -1
index 7499490..7a69491 100644
@@ -71,8 +71,25 @@ enum aarch64_insn_hint_op {
 
 bool aarch64_insn_is_nop(u32 insn);
 
-enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+/*
+ * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
+ * little-endian.

arch/arm64/include/asm/insn.h enum aarch64_insn_hint_op { · line 86

Quoted 10 lines
arch/arm64/include/asm/insn.h +9 -0
+ */
+static __always_inline u32 aarch64_insn_read(void *addr)
+{
+	return le32_to_cpu(*(u32 *)addr);
+}
 
+static __always_inline void aarch64_insn_write(void *addr, u32 insn)
+{
+	*(u32 *)addr = cpu_to_le32(insn);
+}

I wouldn't bother with these helpers. You should probably be using probe_kernel_address or similar, then doing the endianness swabbing on the return value in-line.

Quoted 27 lines
arch/arm64/include/asm/insn.h +5 -0
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
+int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
+
 #endif	/* __ASM_INSN_H */
arch/arm64/kernel/insn.c +4 -0
index f5b779f..3879db4 100644
@@ -16,6 +16,9 @@
  */
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/stop_machine.h>
+#include <asm/cacheflush.h>
 #include <asm/insn.h>
 
 static int aarch64_insn_encoding_class[] = {
@@ -88,3 +91,91 @@ bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
 	return __aarch64_insn_hotpatch_safe(old_insn) &&
 	       __aarch64_insn_hotpatch_safe(new_insn);
 }
+

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 104

Quoted 10 lines
arch/arm64/kernel/insn.c +10 -0
+int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
+{
+	u32 *tp = addr;
+
+	/* A64 instructions must be word aligned */
+	if ((uintptr_t)tp & 0x3)
+		return -EINVAL;
+
+	aarch64_insn_write(tp, insn);
+	flush_icache_range((uintptr_t)tp, (uintptr_t)tp + sizeof(u32));

sizeof(insn) is clearer here.

Quoted 9 lines
arch/arm64/kernel/insn.c +9 -0
+
+	return 0;
+}
+
+struct aarch64_insn_patch {
+	void	**text_addrs;
+	u32	*new_insns;
+	int	insn_cnt;
+};

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 123

Quoted 10 lines
arch/arm64/kernel/insn.c +10 -0
+
+static DEFINE_MUTEX(text_patch_lock);
+static atomic_t text_patch_id;
+
+static int __kprobes aarch64_insn_patch_text_cb(void *arg)
+{
+	int i, ret = 0;
+	struct aarch64_insn_patch *pp = arg;
+
+	if (atomic_read(&text_patch_id) == smp_processor_id()) {

You could actually pass in the test_patch_id as zero-initialised parameter to this function (i.e. it points to something on the stack of the guy issuing the stop_machine). Then you do an inc_return here. If you see zero, you go ahead and do the patching.

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128

Quoted 5 lines
arch/arm64/kernel/insn.c +5 -0
+		for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
+			ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
+							     pp->new_insns[i]);
+		/* Let other CPU continue */
+		atomic_set(&text_patch_id, -1);

You're relying on the barrier in flush_icache_range to order this atomic_set. I think you should add a comment describing that, because it's very subtle.

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 136

Quoted 8 lines
arch/arm64/kernel/insn.c +8 -0
+	} else {
+		while (atomic_read(&text_patch_id) != -1)
+			cpu_relax();
+		isb();
+	}
+
+	return ret;
+}

I don't think you need the isb -- the exception return should do the trick (but again, a comment would be useful).

Quoted 4 lines
arch/arm64/kernel/insn.c +4 -0
+
+int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
+{
+	int ret;

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 150

Quoted 10 lines
arch/arm64/kernel/insn.c +10 -0
+	struct aarch64_insn_patch patch = {
+		.text_addrs = addrs,
+		.new_insns = insns,
+		.insn_cnt = cnt,
+	};
+
+	if (cnt <= 0)
+		return -EINVAL;
+
+	mutex_lock(&text_patch_lock);

Again, if you use a loacl variable, I don't think you need the mutex. What do you think?

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 152

Quoted 2 lines
arch/arm64/kernel/insn.c +2 -0
+	atomic_set(&text_patch_id, smp_processor_id());
+	ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Will

author
#6 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 11 lines · Will Deacon

Hi Jinag Liu,

Sorry for the delayed review, I've been travelling.

Quoted 1 line · Jiang Liu

From: Jiang Liu <[email protected]>

If I try and email you at your Huawei address, I get a bounce from the mail server. Is that expected? If so, it's not very helpful from a commit log perspective if you use that address as the author on your patches.

Hi Will,
Sorry for the inconvenience. I have left Huawei recently and
have had a vacation last two weeks. I will use my gmail account next time.

Quoted 34 lines
Quoted 34 lines

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

---
 arch/arm64/include/asm/insn.h | 19 ++++++++-
 arch/arm64/kernel/insn.c      | 91 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +8 -1
index 7499490..7a69491 100644
@@ -71,8 +71,25 @@ enum aarch64_insn_hint_op {
 
 bool aarch64_insn_is_nop(u32 insn);
 
-enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+/*
+ * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
+ * little-endian.
+ */
+static __always_inline u32 aarch64_insn_read(void *addr)
+{
+	return le32_to_cpu(*(u32 *)addr);
+}
 
Quoted 8 lines

arch/arm64/include/asm/insn.h enum aarch64_insn_hint_op { · line 86

Quoted 4 lines
arch/arm64/include/asm/insn.h +4 -0
+static __always_inline void aarch64_insn_write(void *addr, u32 insn)
+{
+	*(u32 *)addr = cpu_to_le32(insn);
+}

I wouldn't bother with these helpers. You should probably be using probe_kernel_address or similar, then doing the endianness swabbing on the return value in-line.

How about keeping and refining aarch64_insn_read/write interfaces by using probe_kernel_address()? I think they may be used in other places when supporting big endian ARM64 kernel.

Quoted 32 lines
Quoted 32 lines
arch/arm64/include/asm/insn.h +5 -0
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
+int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
+
 #endif	/* __ASM_INSN_H */
arch/arm64/kernel/insn.c +8 -0
index f5b779f..3879db4 100644
@@ -16,6 +16,9 @@
  */
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/stop_machine.h>
+#include <asm/cacheflush.h>
 #include <asm/insn.h>
 
 static int aarch64_insn_encoding_class[] = {
@@ -88,3 +91,91 @@ bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
 	return __aarch64_insn_hotpatch_safe(old_insn) &&
 	       __aarch64_insn_hotpatch_safe(new_insn);
 }
+
+int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
+{
+	u32 *tp = addr;
+
Quoted 9 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 104

Quoted 6 lines
arch/arm64/kernel/insn.c +6 -0
+	/* A64 instructions must be word aligned */
+	if ((uintptr_t)tp & 0x3)
+		return -EINVAL;
+
+	aarch64_insn_write(tp, insn);
+	flush_icache_range((uintptr_t)tp, (uintptr_t)tp + sizeof(u32));

sizeof(insn) is clearer here.

Will make this change in next version.

Quoted 16 lines
Quoted 16 lines
arch/arm64/kernel/insn.c +16 -0
+
+	return 0;
+}
+
+struct aarch64_insn_patch {
+	void	**text_addrs;
+	u32	*new_insns;
+	int	insn_cnt;
+};
+
+static DEFINE_MUTEX(text_patch_lock);
+static atomic_t text_patch_id;
+
+static int __kprobes aarch64_insn_patch_text_cb(void *arg)
+{
+	int i, ret = 0;
Quoted 8 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 123

Quoted 3 lines
arch/arm64/kernel/insn.c +3 -0
+	struct aarch64_insn_patch *pp = arg;
+
+	if (atomic_read(&text_patch_id) == smp_processor_id()) {

You could actually pass in the test_patch_id as zero-initialised parameter to this function (i.e. it points to something on the stack of the guy issuing the stop_machine). Then you do an inc_return here. If you see zero, you go ahead and do the patching.

Good suggestion!
Function stop_machine() already has a mutex to serialize all callers, so we don't need explicitly serialization here. Will simplify the code in next version.

Quoted 10 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128

Quoted 6 lines
arch/arm64/kernel/insn.c +5 -0
+		for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
+			ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
+							     pp->new_insns[i]);
+		/* Let other CPU continue */
+		atomic_set(&text_patch_id, -1);

You're relying on the barrier in flush_icache_range to order this atomic_set. I think you should add a comment describing that, because it's very subtle.

How about an explicitly smp_wmb() here? That would be more maintainable.

Quoted 12 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 136

Quoted 9 lines
arch/arm64/kernel/insn.c +8 -0
+	} else {
+		while (atomic_read(&text_patch_id) != -1)
+			cpu_relax();
+		isb();
+	}
+
+	return ret;
+}

I don't think you need the isb -- the exception return should do the trick (but again, a comment would be useful).

stop_machine() is implemented by thread scheduling instead of cross CPU function call(IPI), so there may be no "eret" after returning from this callback function. So used an "isb" here.

Quoted 10 lines
Quoted 10 lines
arch/arm64/kernel/insn.c +9 -0
+
+int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
+{
+	int ret;
+	struct aarch64_insn_patch patch = {
+		.text_addrs = addrs,
+		.new_insns = insns,
+		.insn_cnt = cnt,
+	};
Quoted 8 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 150

Quoted 5 lines
arch/arm64/kernel/insn.c +5 -0
+
+	if (cnt <= 0)
+		return -EINVAL;
+
+	mutex_lock(&text_patch_lock);

Again, if you use a loacl variable, I don't think you need the mutex. What do you think?

Sure, will make the change.

Quoted 7 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 152

Quoted 3 lines
arch/arm64/kernel/insn.c +2 -0
+	atomic_set(&text_patch_id, smp_processor_id());
+	ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Sandeepa, who is working on kprobe for ARM64, needs the stop_machine() mechanism to synchronize all online CPUs, so it's a preparation for kprobe.

Thanks!
Gerry

Quoted 3 lines

Will

#7 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 188 lines · Jiang Liu
Quoted 11 lines · Will Deacon

Hi Jinag Liu,

Sorry for the delayed review, I've been travelling.

Quoted 1 line · Jiang Liu

From: Jiang Liu <[email protected]>

If I try and email you at your Huawei address, I get a bounce from the mail server. Is that expected? If so, it's not very helpful from a commit log perspective if you use that address as the author on your patches.

Hi Will,
Sorry for the inconvenience. I have left Huawei recently and
have had a vacation last two weeks. I will use my gmail account next time.

Quoted 34 lines
Quoted 34 lines

Introduce three interfaces to patch kernel and module code:
aarch64_insn_patch_text_nosync():
patch code without synchronization, it's caller's responsibility to synchronize all CPUs if needed.
aarch64_insn_patch_text_sync():
patch code and always synchronize with stop_machine()
aarch64_insn_patch_text():
patch code and synchronize with stop_machine() if needed

---
 arch/arm64/include/asm/insn.h | 19 ++++++++-
 arch/arm64/kernel/insn.c      | 91 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 1 deletion(-)
arch/arm64/include/asm/insn.h +8 -1
index 7499490..7a69491 100644
@@ -71,8 +71,25 @@ enum aarch64_insn_hint_op {
 
 bool aarch64_insn_is_nop(u32 insn);
 
-enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+/*
+ * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
+ * little-endian.
+ */
+static __always_inline u32 aarch64_insn_read(void *addr)
+{
+    return le32_to_cpu(*(u32 *)addr);
+}
 
Quoted 8 lines

arch/arm64/include/asm/insn.h enum aarch64_insn_hint_op { · line 86

Quoted 4 lines
arch/arm64/include/asm/insn.h +4 -0
+static __always_inline void aarch64_insn_write(void *addr, u32 insn)
+{
+    *(u32 *)addr = cpu_to_le32(insn);
+}

I wouldn't bother with these helpers. You should probably be using probe_kernel_address or similar, then doing the endianness swabbing on the return value in-line.

How about keeping and refining aarch64_insn_read/write interfaces by using probe_kernel_address()? I think they may be used in other places when supporting big endian ARM64 kernel.

Quoted 32 lines
Quoted 32 lines
arch/arm64/include/asm/insn.h +5 -0
+enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
+int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
+
 #endif      /* __ASM_INSN_H */
arch/arm64/kernel/insn.c +8 -0
index f5b779f..3879db4 100644
@@ -16,6 +16,9 @@
  */
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/stop_machine.h>
+#include <asm/cacheflush.h>
 #include <asm/insn.h>
 
 static int aarch64_insn_encoding_class[] = {
@@ -88,3 +91,91 @@ bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
     return __aarch64_insn_hotpatch_safe(old_insn) &&
            __aarch64_insn_hotpatch_safe(new_insn);
 }
+
+int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
+{
+    u32 *tp = addr;
+
Quoted 9 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 104

Quoted 6 lines
arch/arm64/kernel/insn.c +6 -0
+    /* A64 instructions must be word aligned */
+    if ((uintptr_t)tp & 0x3)
+            return -EINVAL;
+
+    aarch64_insn_write(tp, insn);
+    flush_icache_range((uintptr_t)tp, (uintptr_t)tp + sizeof(u32));

sizeof(insn) is clearer here.

Will make this change in next version.

Quoted 16 lines
Quoted 16 lines
arch/arm64/kernel/insn.c +16 -0
+
+    return 0;
+}
+
+struct aarch64_insn_patch {
+    void    **text_addrs;
+    u32     *new_insns;
+    int     insn_cnt;
+};
+
+static DEFINE_MUTEX(text_patch_lock);
+static atomic_t text_patch_id;
+
+static int __kprobes aarch64_insn_patch_text_cb(void *arg)
+{
+    int i, ret = 0;
Quoted 8 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 123

Quoted 3 lines
arch/arm64/kernel/insn.c +3 -0
+    struct aarch64_insn_patch *pp = arg;
+
+    if (atomic_read(&text_patch_id) == smp_processor_id()) {

You could actually pass in the test_patch_id as zero-initialised parameter to this function (i.e. it points to something on the stack of the guy issuing the stop_machine). Then you do an inc_return here. If you see zero, you go ahead and do the patching.

Good suggestion!
Function stop_machine() already has a mutex to serialize all callers, so we don't need explicitly serialization here. Will simplify the code in next version.

Quoted 10 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 128

Quoted 6 lines
arch/arm64/kernel/insn.c +5 -0
+            for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
+                    ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
+                                                         pp->new_insns[i]);
+            /* Let other CPU continue */
+            atomic_set(&text_patch_id, -1);

You're relying on the barrier in flush_icache_range to order this atomic_set. I think you should add a comment describing that, because it's very subtle.

How about an explicitly smp_wmb() here? That would be more maintainable.

Quoted 12 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 136

Quoted 9 lines
arch/arm64/kernel/insn.c +8 -0
+    } else {
+            while (atomic_read(&text_patch_id) != -1)
+                    cpu_relax();
+            isb();
+    }
+
+    return ret;
+}

I don't think you need the isb -- the exception return should do the trick (but again, a comment would be useful).

stop_machine() is implemented by thread scheduling instead of cross CPU function call(IPI), so there may be no "eret" after returning from this callback function. So used an "isb" here.

Quoted 10 lines
Quoted 10 lines
arch/arm64/kernel/insn.c +9 -0
+
+int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
+{
+    int ret;
+    struct aarch64_insn_patch patch = {
+            .text_addrs = addrs,
+            .new_insns = insns,
+            .insn_cnt = cnt,
+    };
Quoted 8 lines

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 150

Quoted 5 lines
arch/arm64/kernel/insn.c +5 -0
+
+    if (cnt <= 0)
+            return -EINVAL;
+
+    mutex_lock(&text_patch_lock);

Again, if you use a loacl variable, I don't think you need the mutex. What do you think?

Sure, will make the change.

Quoted 4 lines
Quoted 3 lines
arch/arm64/kernel/insn.c +2 -0
+    atomic_set(&text_patch_id, smp_processor_id());
+    ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);
Quoted 6 lines
Quoted 3 lines

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Sandeepa, who is working on kprobe for ARM64, needs the stop_machine() mechanism to synchronize all online CPUs, so it's a preparation for kprobe.

Hi Gerry,

I had published kprobes patches for ARM64:
http://lwn.net/Articles/570648/ and using your patcset (v3) for patching support, it works so far.
I CCed you on my RFC but unfortunately to your huawei email not the gmail.

I can give a try with kick_all_cpus_sync but wanted to understand this a bit detail on hows different from stop_machine and how this work.

Thanks,
Sandeepa

Quoted 7 lines

Thanks!
Gerry

Quoted 3 lines

Will

#8 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 13 lines · Sandeepa Prabhu
Quoted 11 lines · Jiang Liu
Quoted 7 lines · Will Deacon

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 152

Quoted 2 lines · Jiang Liu
arch/arm64/kernel/insn.c +2 -0
+    atomic_set(&text_patch_id, smp_processor_id());
+    ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Sandeepa, who is working on kprobe for ARM64, needs the stop_machine() mechanism to synchronize all online CPUs, so it's a preparation for kprobe.

Quoted 7 lines

I had published kprobes patches for ARM64:
http://lwn.net/Articles/570648/ and using your patcset (v3) for patching support, it works so far.
I CCed you on my RFC but unfortunately to your huawei email not the gmail.

I can give a try with kick_all_cpus_sync but wanted to understand this a bit detail on hows different from stop_machine and how this work.

My point was just that for nosync patching, the update to the instruction stream is atomic with respect to instruction fetch, so stop_machine seems a bit overkill. kick_all_cpus can be used to ensure visibility of the new instruction.

Jiang Liu seemed to imply that this isn't suitable for kprobes, but I would like to know if/why that is the case.

Will

author
#9 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 27 lines · Will Deacon
Quoted 13 lines · Sandeepa Prabhu
Quoted 11 lines · Jiang Liu
Quoted 7 lines · Will Deacon

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 152

Quoted 2 lines · Jiang Liu
arch/arm64/kernel/insn.c +2 -0
+    atomic_set(&text_patch_id, smp_processor_id());
+    ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Sandeepa, who is working on kprobe for ARM64, needs the stop_machine() mechanism to synchronize all online CPUs, so it's a preparation for kprobe.

Quoted 7 lines

I had published kprobes patches for ARM64:
http://lwn.net/Articles/570648/ and using your patcset (v3) for patching support, it works so far.
I CCed you on my RFC but unfortunately to your huawei email not the gmail.

I can give a try with kick_all_cpus_sync but wanted to understand this a bit detail on hows different from stop_machine and how this work.

My point was just that for nosync patching, the update to the instruction stream is atomic with respect to instruction fetch, so stop_machine seems a bit overkill. kick_all_cpus can be used to ensure visibility of the new instruction.

Quoted 5 lines

Jiang Liu seemed to imply that this isn't suitable for kprobes, but I would like to know if/why that is the case.

Will

Hi Will and Sandeepa,
Seems some misunderstanding here. We provide three interfaces
here.
1) aarch64_insn_patch_text_nosync(), which patches kernel/module text without explicitly synchronization. It may be used in cases of 1.a) There's only one CPU running during early boot stage 1.b) All other CPU has been put into safe state in kgdb. 1.c) The instructions before and after patching are both hot-patch safe.

2) aarch64_insn_patch_text_sync(), which patches kernel/module text with explicitly synchronization by stop_machine() mechanism. It may be used to support kprobe because kprobe may patch multiple and/or non-hotpatch safe instructions at runtime, so we need stop_machine() for synchronization.

3) aarch64_insn_patch_text() intelligently choose aarch64_insn_patch_text_nosync() or aarch64_insn_patch_text_sync().

So for kprobe, I think need to use stop_machine() for synchronization.

Thanks!
Gerry

#10 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 48 lines · Jiang Liu
Quoted 27 lines · Will Deacon
Quoted 13 lines · Sandeepa Prabhu
Quoted 11 lines · Jiang Liu
Quoted 7 lines · Will Deacon

arch/arm64/kernel/insn.c bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) · line 152

Quoted 2 lines · Jiang Liu
arch/arm64/kernel/insn.c +2 -0
+    atomic_set(&text_patch_id, smp_processor_id());
+    ret = stop_machine(aarch64_insn_patch_text_cb, &patch, cpu_online_mask);

Instead of doing this, why not instead call aarch64_insn_patch_text_nosync inline, then call kick_all_cpus_sync immediately afterwards, without the need to stop_machine.

Sandeepa, who is working on kprobe for ARM64, needs the stop_machine() mechanism to synchronize all online CPUs, so it's a preparation for kprobe.

Quoted 7 lines

I had published kprobes patches for ARM64:
http://lwn.net/Articles/570648/ and using your patcset (v3) for patching support, it works so far.
I CCed you on my RFC but unfortunately to your huawei email not the gmail.

I can give a try with kick_all_cpus_sync but wanted to understand this a bit detail on hows different from stop_machine and how this work.

My point was just that for nosync patching, the update to the instruction stream is atomic with respect to instruction fetch, so stop_machine seems a bit overkill. kick_all_cpus can be used to ensure visibility of the new instruction.

Quoted 5 lines

Jiang Liu seemed to imply that this isn't suitable for kprobes, but I would like to know if/why that is the case.

Will

Hi Will and Sandeepa,
Seems some misunderstanding here. We provide three interfaces
here.
1) aarch64_insn_patch_text_nosync(), which patches kernel/module text without explicitly synchronization. It may be used in cases of 1.a) There's only one CPU running during early boot stage 1.b) All other CPU has been put into safe state in kgdb. 1.c) The instructions before and after patching are both hot-patch safe.

2) aarch64_insn_patch_text_sync(), which patches kernel/module text with explicitly synchronization by stop_machine() mechanism. It may be used to support kprobe because kprobe may patch multiple and/or non-hotpatch safe instructions at runtime, so we need stop_machine() for synchronization.

Quoted 4 lines

3) aarch64_insn_patch_text() intelligently choose aarch64_insn_patch_text_nosync() or aarch64_insn_patch_text_sync().

So for kprobe, I think need to use stop_machine() for synchronization.

Yup that sounds right. I think I must've got confused as to what exactly was being patched and the interaction with the mutex; if we're patching live code which isn't hot-patch safe, then stop_machine sounds like the thing to do.

Sorry for the confusion,

Will

#11 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

In reply to Jiang Liu

Hi Jiang Liu,

Quoted 9 lines · Jiang Liu
Quoted 4 lines · Will Deacon

If I try and email you at your Huawei address, I get a bounce from the mail server. Is that expected? If so, it's not very helpful from a commit log perspective if you use that address as the author on your patches.

Hi Will,
Sorry for the inconvenience. I have left Huawei recently and
have had a vacation last two weeks. I will use my gmail account next time.

I appreciate that you're probably not being paid to look at this anymore, but have you had a chance to address the comments I made on version 5 of this series? As far as I'm concerned, this is almost ready for merging and I'd love to see it get in for 3.14!

It would be a real shame if this ended up getting dropped...

Cheers,

Will

author
#12 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

Quoted 13 lines · Will Deacon

Hi Jiang Liu,

Quoted 9 lines · Jiang Liu
Quoted 4 lines · Will Deacon

If I try and email you at your Huawei address, I get a bounce from the mail server. Is that expected? If so, it's not very helpful from a commit log perspective if you use that address as the author on your patches.

Hi Will,
Sorry for the inconvenience. I have left Huawei recently and
have had a vacation last two weeks. I will use my gmail account next time.

Quoted 6 lines

I appreciate that you're probably not being paid to look at this anymore, but have you had a chance to address the comments I made on version 5 of this series? As far as I'm concerned, this is almost ready for merging and I'd love to see it get in for 3.14!

It would be a real shame if this ended up getting dropped...

Hi Will,
Thanks for reminder!
I have changed the code according to review comments,
but I'm having troubles to find a real hardware platform to test the new code since I left Huawei. I will try to figure out a way to test the code by contacting ex-teammember at Huawei.
Hope it could catch the of 3.14 merging window.
Cheers,
Gerry

Quoted 5 lines

Cheers,

Will

#13 lore raw

Re: [PATCH v5 2/7] arm64: introduce interfaces to hotpatch kernel and module code

In reply to Jiang Liu
Quoted 16 lines · Jiang Liu
Quoted 7 lines · Will Deacon

Hi Jinag Liu,

Quoted 5 lines
arch/arm64/include/asm/insn.h +4 -0
+static __always_inline u32 aarch64_insn_read(void *addr)
+{
+	return le32_to_cpu(*(u32 *)addr);
+}
Quoted 8 lines

arch/arm64/include/asm/insn.h enum aarch64_insn_hint_op { · line 86

Quoted 4 lines
arch/arm64/include/asm/insn.h +4 -0
+static __always_inline void aarch64_insn_write(void *addr, u32 insn)
+{
+	*(u32 *)addr = cpu_to_le32(insn);
+}

I wouldn't bother with these helpers. You should probably be using probe_kernel_address or similar, then doing the endianness swabbing on the return value in-line.

Quoted 3 lines

How about keeping and refining aarch64_insn_read/write interfaces by using probe_kernel_address()? I think they may be used in other places when supporting big endian ARM64 kernel.

I prefer it (using probe_kernel_read/write) for my ftrace patch.
I would be able to replace some portion of my own function (ftrace_modify_code) to aarch64_insn_patch_text_nosync().

See my comment here:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-October/207001.html
([PATCH 2/6])

Current implementation assumes stop_machine (via arch_ftrace_update_code() in generic ftrace), and, given the discussion btw you and Will, I wonder that it might be relaxed because ftrace on arm64 modifies only a single branch or nop instruction at any time.

-Takahiro AKASHI

author
#14 lore raw

PATCH v5 3/7 arm64: move encode_insn_immediate() from module.c to insn.c

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

Function encode_insn_immediate() will be used by other instruction manipulate related functions, so move it into insn.c and rename it as aarch64_insn_encode_immediate().

---
 arch/arm64/include/asm/insn.h |  13 ++++
 arch/arm64/kernel/insn.c      |  54 +++++++++++++++
 arch/arm64/kernel/module.c    | 153 +++++++++++++-----------------------------
 3 files changed, 113 insertions(+), 107 deletions(-)
arch/arm64/include/asm/insn.h +13 -0
index 7a69491..8f94e48 100644
@@ -42,6 +42,17 @@ enum aarch64_insn_encoding_class {
 					 * system instructions */
 };
 
+enum aarch64_insn_imm_type {
+	AARCH64_INSN_IMM_ADR,
+	AARCH64_INSN_IMM_26,
+	AARCH64_INSN_IMM_19,
+	AARCH64_INSN_IMM_16,
+	AARCH64_INSN_IMM_14,
+	AARCH64_INSN_IMM_12,
+	AARCH64_INSN_IMM_9,
+	AARCH64_INSN_IMM_MAX
+};
+
 #define	__AARCH64_INSN_FUNCS(abbr, mask, val)	\
 static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
 { return (code & (mask)) == (val); }	\
@@ -86,6 +97,8 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn)
 }
 
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
+u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
+				  u32 insn, u64 imm);
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +54 -0

57 lines not on this page · raw message

arch/arm64/kernel/module.c +46 -107

295 lines not on this page · raw message

Signature · 1 line
1.8.1.2
author
#15 lore raw

PATCH v5 4/7 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.

---
 arch/arm64/include/asm/insn.h | 13 +++++++++++++
 arch/arm64/kernel/insn.c      | 28 ++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
arch/arm64/include/asm/insn.h +13 -0
index 8f94e48..ee4a60d 100644
@@ -96,9 +96,22 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn)
 	*(u32 *)addr = cpu_to_le32(insn);
 }
 
+static __always_inline u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
+{
+	return aarch64_insn_get_hint_value() | op;
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}
+
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 				  u32 insn, u64 imm);
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+				bool link);
+
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +28 -0
index 886821f..f7498cc 100644
@@ -14,6 +14,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
@@ -233,3 +234,30 @@ u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 
 	return insn;
 }
+
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, bool link)
+{
+	u32 insn;
+	long offset;
+
+	/*
+	 * PC: A 64-bit Program Counter holding the address of the current
+	 * instruction. A64 instructions may be word-aligned.
+	 */
+	BUG_ON((pc & 0x3) || (addr & 0x3));
+
+	/*
+	 * B/BL support [-128M, 128M) offset
+	 * ARM64 virtual address arrangement garantees all kernel and module
+	 * texts are within +/-128M.
+	 */
+	offset = ((long)addr - (long)pc) >> 2;
+	BUG_ON(abs(offset) > BIT(25) || offset == BIT(25));
+
+	if (link)
+		insn = aarch64_insn_get_bl_value();
+	else
+		insn = aarch64_insn_get_b_value();
+
+	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, offset);
+}
Signature · 1 line
1.8.1.2
#16 lore raw

Re: [PATCH v5 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

Quoted 17 lines · Jiang Liu

From: Jiang Liu <[email protected]>

Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.

---
 arch/arm64/include/asm/insn.h | 13 +++++++++++++
 arch/arm64/kernel/insn.c      | 28 ++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
arch/arm64/include/asm/insn.h +0 -0
index 8f94e48..ee4a60d 100644

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 101

Quoted 6 lines
arch/arm64/include/asm/insn.h +3 -0
@@ -96,9 +96,22 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn)
 	*(u32 *)addr = cpu_to_le32(insn);
 }
 
+static __always_inline u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
+{
+	return aarch64_insn_get_hint_value() | op;

What's the point in that helper function? Does it just return an immediate value (and are there are other users of it?).

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 107

Quoted 6 lines
arch/arm64/include/asm/insn.h +6 -0
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}

Either use plain old `inline' or write these as preprocessor macros.

Quoted 31 lines
arch/arm64/include/asm/insn.h +4 -0
+
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 				  u32 insn, u64 imm);
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+				bool link);
+
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +6 -0
index 886821f..f7498cc 100644
@@ -14,6 +14,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
@@ -233,3 +234,30 @@ u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 
 	return insn;
 }
+
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, bool link)
+{
+	u32 insn;
+	long offset;

arch/arm64/kernel/insn.c u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, · line 251

Quoted 10 lines
arch/arm64/kernel/insn.c +10 -0
+
+	/*
+	 * PC: A 64-bit Program Counter holding the address of the current
+	 * instruction. A64 instructions may be word-aligned.
+	 */
+	BUG_ON((pc & 0x3) || (addr & 0x3));
+
+	/*
+	 * B/BL support [-128M, 128M) offset
+	 * ARM64 virtual address arrangement garantees all kernel and module

Typo: `guarantees'.

arch/arm64/kernel/insn.c u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, · line 255

Quoted 4 lines
arch/arm64/kernel/insn.c +4 -0
+	 * texts are within +/-128M.
+	 */
+	offset = ((long)addr - (long)pc) >> 2;
+	BUG_ON(abs(offset) > BIT(25) || offset == BIT(25));

I really struggle to follow this range checking. Why don't you just compare the absolute difference between addr and pc with SZ_128M?

Will

author
#17 lore raw

Re: [PATCH v5 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

Quoted 21 lines · Will Deacon
Quoted 20 lines · Jiang Liu

From: Jiang Liu <[email protected]>

Introduce aarch64_insn_gen_{nop|branch_imm}() helper functions, which will be used to implement jump label on ARM64.

---
 arch/arm64/include/asm/insn.h | 13 +++++++++++++
 arch/arm64/kernel/insn.c      | 28 ++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
arch/arm64/include/asm/insn.h +0 -0
index 8f94e48..ee4a60d 100644
@@ -96,9 +96,22 @@ static __always_inline void aarch64_insn_write(void *addr, u32 insn)
 	*(u32 *)addr = cpu_to_le32(insn);
 }
 
Quoted 6 lines

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 101

Quoted 3 lines
arch/arm64/include/asm/insn.h +3 -0
+static __always_inline u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
+{
+	return aarch64_insn_get_hint_value() | op;

What's the point in that helper function? Does it just return an immediate value (and are there are other users of it?).

It just returns a const value. The helper function is just for ease maintainence.

Quoted 9 lines

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 107

Quoted 7 lines
arch/arm64/include/asm/insn.h +6 -0
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}

Either use plain old `inline' or write these as preprocessor macros.

Will change to normal (non-inline) and add __kprobe property.

Quoted 36 lines
Quoted 36 lines
arch/arm64/include/asm/insn.h +4 -0
+
 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
 u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 				  u32 insn, u64 imm);
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+				bool link);
+
 bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
 
 int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
arch/arm64/kernel/insn.c +10 -0
index 886821f..f7498cc 100644
@@ -14,6 +14,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/smp.h>
@@ -233,3 +234,30 @@ u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 
 	return insn;
 }
+
+u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, bool link)
+{
+	u32 insn;
+	long offset;
+
+	/*
+	 * PC: A 64-bit Program Counter holding the address of the current
+	 * instruction. A64 instructions may be word-aligned.
Quoted 8 lines

arch/arm64/kernel/insn.c u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, · line 251

Quoted 6 lines
arch/arm64/kernel/insn.c +6 -0
+	 */
+	BUG_ON((pc & 0x3) || (addr & 0x3));
+
+	/*
+	 * B/BL support [-128M, 128M) offset
+	 * ARM64 virtual address arrangement garantees all kernel and module

Typo: `guarantees'.

Thanks, will fix it.

Quoted 8 lines

arch/arm64/kernel/insn.c u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, · line 255

Quoted 5 lines
arch/arm64/kernel/insn.c +4 -0
+	 * texts are within +/-128M.
+	 */
+	offset = ((long)addr - (long)pc) >> 2;
+	BUG_ON(abs(offset) > BIT(25) || offset == BIT(25));

I really struggle to follow this range checking. Why don't you just compare the absolute difference between addr and pc with SZ_128M?

Will change code to follow your suggestion.

Quoted 3 lines

Will

#18 lore raw

Re: [PATCH v5 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

In reply to Will Deacon
Quoted 9 lines · On Wed, 30 Oct 2013 00:48:40 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 107

Quoted 6 lines
arch/arm64/include/asm/insn.h +6 -0
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}

Either use plain old `inline' or write these as preprocessor macros.

I'm curious to why you say that? Preprocessor macros are rather ugly, and in x86 we use "__always_inline" quite liberally.

-- Steve

#19 lore raw

Re: [PATCH v5 4/7] arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions

Quoted 14 lines · Steven Rostedt
Quoted 9 lines · On Wed, 30 Oct 2013 00:48:40 +0000 Will Deacon <[email protected]> wrote:

arch/arm64/include/asm/insn.h static __always_inline void aarch64_insn_write(void *addr, u32 insn) · line 107

Quoted 6 lines
arch/arm64/include/asm/insn.h +6 -0
+}
+
+static __always_inline u32 aarch64_insn_gen_nop(void)
+{
+	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
+}

Either use plain old `inline' or write these as preprocessor macros.

Quoted 2 lines

I'm curious to why you say that? Preprocessor macros are rather ugly, and in x86 we use "__always_inline" quite liberally.

I can understand why you might use __always_inline over the preprocessor when you *really* need something inlined, but in this case I don't see why `inline' isn't sufficient. It's just a cosmetic issue, really.

Will

author
#20 lore raw

PATCH v5 5/7 arm64, jump label: detect %c support for ARM64

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

As commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.

Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.

---
 scripts/gcc-goto.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/gcc-goto.sh +1 -1
index a2af2e8..c9469d3 100644
@@ -5,7 +5,7 @@
 cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
 int main(void)
 {
-#ifdef __arm__
+#if defined(__arm__) || defined(__aarch64__)
 	/*
 	 * Not related to asm goto, but used by jump label
 	 * and broken on some ARM GCC versions (see GCC Bug 48637).
Signature · 1 line
1.8.1.2
Acked-by on v5 5/7
#21 lore raw

Re: [PATCH v5 5/7] arm64, jump label: detect %c support for ARM64

Quoted 10 lines · Jiang Liu

From: Jiang Liu <[email protected]>

As commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.

Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.

The toolchain is being fixed upstream for this, but we should have the check for older versions.

Will

author
#22 lore raw

Re: [PATCH v5 5/7] arm64, jump label: detect %c support for ARM64

Quoted 12 lines · Will Deacon
Quoted 10 lines · Jiang Liu

From: Jiang Liu <[email protected]>

As commit a9468f30b5eac6 "ARM: 7333/2: jump label: detect %c support for ARM", this patch detects the same thing for ARM64 because some ARM64 GCC versions have the same issue.

Some versions of ARM64 GCC which do support asm goto, do not support the %c specifier. Since we need the %c to support jump labels on ARM64, detect that too in the asm goto detection script to avoid build errors with these versions.

Quoted 7 lines

The toolchain is being fixed upstream for this, but we should have the check for older versions.

Will

Thanks for review!

author
#23 lore raw

PATCH v5 6/7 arm64, jump label: optimize jump label implementation

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

---
 arch/arm64/Kconfig                  |  1 +
 arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile          |  1 +
 arch/arm64/kernel/jump_label.c      | 58 +++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
index c044548..da388e4 100644
@@ -17,6 +17,7 @@ config ARM64
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
+	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +52 -0

52 lines not on this page · raw message

arch/arm64/kernel/Makefile +1 -0
index 9af6cb3..b7db65e 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o smp_psci.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +58 -0

58 lines not on this page · raw message

Signature · 1 line
1.8.1.2
#24 lore raw

Re: [PATCH v5 6/7] arm64, jump label: optimize jump label implementation

Quoted 54 lines · Jiang Liu

From: Jiang Liu <[email protected]>

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

---
 arch/arm64/Kconfig                  |  1 +
 arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile          |  1 +
 arch/arm64/kernel/jump_label.c      | 58 +++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
index c044548..da388e4 100644
@@ -17,6 +17,7 @@ config ARM64
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
+	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +20 -0
new file mode 100644
index 0000000..d268fab
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/include/asm/jump_label.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _ASM_ARM64_JUMP_LABEL_H
+#define _ASM_ARM64_JUMP_LABEL_H

arch/arm64/include/asm/jump_label.h line 30

Quoted 10 lines
arch/arm64/include/asm/jump_label.h new file +10 -0
+#include <linux/types.h>
+
+#ifdef __KERNEL__
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key)
+{
+	asm goto("1:\n\t"
+		 "nop\n\t"

Minor nit: can you put the label on the same line as the nop please?

Quoted 65 lines
arch/arm64/include/asm/jump_label.h new file +22 -0
+		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 3\n\t"
+		 ".quad 1b, %l[l_yes], %c0\n\t"
+		 ".popsection\n\t"
+		 :  :  "i"(key) :  : l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#endif /* __KERNEL__ */
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif	/* _ASM_ARM64_JUMP_LABEL_H */
arch/arm64/kernel/Makefile +1 -0
index 9af6cb3..b7db65e 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o smp_psci.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +25 -0
new file mode 100644
index 0000000..871786a
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/kernel/jump_label.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL
+

arch/arm64/kernel/jump_label.c line 35

Quoted 10 lines
arch/arm64/kernel/jump_label.c new file +10 -0
+static void __arch_jump_label_transform(struct jump_entry *entry,
+					enum jump_label_type type,
+					bool is_static)
+{
+	void *addr = (void *)entry->code;
+	u32 insn;
+
+	if (type == JUMP_LABEL_ENABLE) {
+		insn = aarch64_insn_gen_branch_imm(entry->code,
+						   entry->target, 0);

Maybe have an enum or #define for the last parameter -- it's not at all obvious that '0' means `no link'.

arch/arm64/kernel/jump_label.c line 41

Quoted 6 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+	} else {
+		insn = aarch64_insn_gen_nop();
+	}
+
+	if (is_static)
+		aarch64_insn_patch_text_nosync(&addr, &insn, 1);

I think I mentioned this before, but is it ever safe to patch more than one instruction in the _nosync case? If not, I wouldn't even bother with that last argument.

Will

author
#25 lore raw

Re: [PATCH v5 6/7] arm64, jump label: optimize jump label implementation

Quoted 59 lines · Will Deacon
Quoted 58 lines · Jiang Liu

From: Jiang Liu <[email protected]>

Optimize jump label implementation for ARM64 by dynamically patching kernel text.

---
 arch/arm64/Kconfig                  |  1 +
 arch/arm64/include/asm/jump_label.h | 52 +++++++++++++++++++++++++++++++++
 arch/arm64/kernel/Makefile          |  1 +
 arch/arm64/kernel/jump_label.c      | 58 +++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 arch/arm64/include/asm/jump_label.h
 create mode 100644 arch/arm64/kernel/jump_label.c
arch/arm64/Kconfig +1 -0
index c044548..da388e4 100644
@@ -17,6 +17,7 @@ config ARM64
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
+	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
arch/arm64/include/asm/jump_label.h new file +24 -0
new file mode 100644
index 0000000..d268fab
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/include/asm/jump_label.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _ASM_ARM64_JUMP_LABEL_H
+#define _ASM_ARM64_JUMP_LABEL_H
+#include <linux/types.h>
+
+#ifdef __KERNEL__
+
Quoted 8 lines

arch/arm64/include/asm/jump_label.h line 30

Quoted 6 lines
arch/arm64/include/asm/jump_label.h new file +6 -0
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key)
+{
+	asm goto("1:\n\t"
+		 "nop\n\t"

Minor nit: can you put the label on the same line as the nop please?

Sure, will refine it.

Quoted 71 lines
Quoted 71 lines
arch/arm64/include/asm/jump_label.h new file +22 -0
+		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 3\n\t"
+		 ".quad 1b, %l[l_yes], %c0\n\t"
+		 ".popsection\n\t"
+		 :  :  "i"(key) :  : l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#endif /* __KERNEL__ */
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif	/* _ASM_ARM64_JUMP_LABEL_H */
arch/arm64/kernel/Makefile +1 -0
index 9af6cb3..b7db65e 100644
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o smp_psci.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
arch/arm64/kernel/jump_label.c new file +30 -0
new file mode 100644
index 0000000..871786a
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <[email protected]>
+ *
+ * Based on arch/arm/kernel/jump_label.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static void __arch_jump_label_transform(struct jump_entry *entry,
+					enum jump_label_type type,
+					bool is_static)
+{
+	void *addr = (void *)entry->code;
Quoted 8 lines

arch/arm64/kernel/jump_label.c line 35

Quoted 5 lines
arch/arm64/kernel/jump_label.c new file +5 -0
+	u32 insn;
+
+	if (type == JUMP_LABEL_ENABLE) {
+		insn = aarch64_insn_gen_branch_imm(entry->code,
+						   entry->target, 0);

Maybe have an enum or #define for the last parameter -- it's not at all obvious that '0' means `no link'.

Thanks, will improve it.

Quoted 11 lines

arch/arm64/kernel/jump_label.c line 41

Quoted 7 lines
arch/arm64/kernel/jump_label.c new file +6 -0
+	} else {
+		insn = aarch64_insn_gen_nop();
+	}
+
+	if (is_static)
+		aarch64_insn_patch_text_nosync(&addr, &insn, 1);

I think I mentioned this before, but is it ever safe to patch more than one instruction in the _nosync case? If not, I wouldn't even bother with that last argument.

I originally plan to optimize jump_label implementation with the third parameter. But for now, I will remove the third parameter to make code simple and add them back in future when needed.
Thanks!

Quoted 3 lines

Will

author
#26 lore raw

PATCH v5 7/7 jump_label: use defined macros instead of hard-coding for better readability

In reply to Jiang Liu

From: Jiang Liu <[email protected]>

Use macro JUMP_LABEL_TRUE_BRANCH instead of hard-coding for better readability.

---
 include/linux/jump_label.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
include/linux/jump_label.h +10 -5
index a507907..6e54029 100644
@@ -74,18 +74,21 @@ struct module;
 #include <linux/atomic.h>
 #ifdef HAVE_JUMP_LABEL
 
-#define JUMP_LABEL_TRUE_BRANCH 1UL
+#define JUMP_LABEL_TYPE_FALSE_BRANCH	0UL
+#define JUMP_LABEL_TYPE_TRUE_BRANCH	1UL
+#define JUMP_LABEL_TYPE_MASK		1UL
 
 static
 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 {
 	return (struct jump_entry *)((unsigned long)key->entries
-						& ~JUMP_LABEL_TRUE_BRANCH);
+						& ~JUMP_LABEL_TYPE_MASK);
 }
 
 static inline bool jump_label_get_branch_default(struct static_key *key)
 {
-	if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
+	if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
+	    JUMP_LABEL_TYPE_TRUE_BRANCH)
 		return true;
 	return false;
 }
@@ -116,9 +119,11 @@ extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 
 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
+	{ .enabled = ATOMIC_INIT(1), \
+	  .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
-	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
+	{ .enabled = ATOMIC_INIT(0), \
+	  .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
 
 #else  /* !HAVE_JUMP_LABEL */
 
Signature · 1 line
1.8.1.2