RISC-V: kprobes/kretprobe support

Needed for kprobes support. Copied and adapted from arm64 code.

Signed-off-by Patrick Stählin <[email protected]>

First implementation, adapted from arm64. The C.ADDISP16 instruction gets simulated and the kprobes-handler called by inserting a C.EBREAK instruction.

C.ADDISP16 was chosen as it sets-up the stack frame for functions. Some work has been done to support probes on non-compressed instructions but there is no support yet for decoding those.

The way forward should be to uncompress the instructions for simulation to reduce the number of instructions used to decode the immediate values on probe hit.

Signed-off-by Patrick St?hlin <[email protected]>

---
 arch/riscv/Kconfig                            |   5 +-
 arch/riscv/include/asm/kprobes.h              |  30 ++
 arch/riscv/include/asm/probes.h               |  26 ++
 arch/riscv/kernel/Makefile                    |   1 +
 arch/riscv/kernel/probes/Makefile             |   3 +
 arch/riscv/kernel/probes/decode-insn.c        |  38 ++
 arch/riscv/kernel/probes/decode-insn.h        |  23 +
 arch/riscv/kernel/probes/kprobes.c            | 401 ++++++++++++++++++
 arch/riscv/kernel/probes/kprobes_trampoline.S |  91 ++++
 arch/riscv/kernel/probes/simulate-insn.c      |  33 ++
 arch/riscv/kernel/probes/simulate-insn.h      |   8 +
 arch/riscv/kernel/traps.c                     |  13 +-
 arch/riscv/mm/fault.c                         |  28 +-
 13 files changed, 694 insertions(+), 6 deletions(-)
 create mode 100644 arch/riscv/include/asm/probes.h
 create mode 100644 arch/riscv/kernel/probes/Makefile
 create mode 100644 arch/riscv/kernel/probes/decode-insn.c
 create mode 100644 arch/riscv/kernel/probes/decode-insn.h
 create mode 100644 arch/riscv/kernel/probes/kprobes.c
 create mode 100644 arch/riscv/kernel/probes/kprobes_trampoline.S
 create mode 100644 arch/riscv/kernel/probes/simulate-insn.c
 create mode 100644 arch/riscv/kernel/probes/simulate-insn.h

arch/riscv/Kconfig +2 -3

index b157ac82d486..11ef4030e8f2 100644
@@ -44,6 +44,8 @@ config RISCV
 	select GENERIC_IRQ_MULTI_HANDLER
 	select ARCH_HAS_PTE_SPECIAL
 	select HAVE_REGS_AND_STACK_ACCESS_API
+	select HAVE_KPROBES
+	select HAVE_KRETPROBES
 
 config MMU
 	def_bool y

@@ -89,9 +91,6 @@ config PGTABLE_LEVELS
 	default 3 if 64BIT
 	default 2
 
-config HAVE_KPROBES
-	def_bool n
-
 menu "Platform type"
 
 choice

arch/riscv/include/asm/kprobes.h +30 -0

index c7eb010d1528..657adcd35a3d 100644
@@ -19,4 +19,34 @@
 
 #include <asm-generic/kprobes.h>
 
+#ifdef CONFIG_KPROBES
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/percpu.h>
+
+#define flush_insn_slot(p)		do { } while (0)
+#define kretprobe_blacklist_size	0
+
+#include <asm/probes.h>
+
+struct prev_kprobe {
+	struct kprobe *kp;
+	unsigned int status;
+};
+
+/* per-cpu kprobe control block */
+struct kprobe_ctlblk {
+	unsigned int kprobe_status;
+	struct prev_kprobe prev_kprobe;
+};
+
+void arch_remove_kprobe(struct kprobe *p);
+int kprobe_fault_handler(struct pt_regs *regs, unsigned int cause);
+int kprobe_exceptions_notify(struct notifier_block *self,
+			     unsigned long val, void *data);
+int kprobe_breakpoint_handler(struct pt_regs *regs);
+void kretprobe_trampoline(void);
+void __kprobes *trampoline_probe_handler(struct pt_regs *regs);
+
+#endif /* CONFIG_KPROBES */
 #endif /* _RISCV_KPROBES_H */

arch/riscv/include/asm/probes.h new file +14 -0

new file mode 100644
index 000000000000..64cf12567539
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Based on arch/arm64/include/asm/probes.h
+ *
+ * Copyright (C) 2013 Linaro Limited
+ */
+#ifndef _RISCV_PROBES_H
+#define _RISCV_PROBES_H
+
+typedef u32 probe_opcode_t;
+typedef void (probes_handler_t) (u32 opcode, long addr, struct pt_regs *);
+
+/* architecture specific copy of original instruction */
+struct arch_probe_insn {

arch/riscv/include/asm/probes.h new file +2 -0

+
+#endif /* _RISCV_PROBES_H */

arch/riscv/kernel/Makefile +1 -0

index f13f7f276639..5360a445b9d3 100644
@@ -28,6 +28,7 @@ obj-y	+= stacktrace.o
 obj-y	+= vdso.o
 obj-y	+= cacheinfo.o
 obj-y	+= vdso/
+obj-y	+= probes/
 
 CFLAGS_setup.o := -mcmodel=medany
 

arch/riscv/kernel/probes/Makefile new file +3 -0

new file mode 100644
index 000000000000..144d1c1743fb
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_KPROBES)		+= kprobes.o kprobes_trampoline.o \
+				   decode-insn.o simulate-insn.o

arch/riscv/kernel/probes/decode-insn.c new file +10 -0

new file mode 100644
index 000000000000..2d8f46f4c2e7
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <asm/sections.h>
+
+#include "decode-insn.h"
+#include "simulate-insn.h"

arch/riscv/kernel/probes/decode-insn.c new file +18 -0

+{
+	probe_opcode_t insn = le32_to_cpu(*addr);
+
+	if (!is_compressed_insn(insn)) {
+		pr_warn("Can't handle non-compressed instruction %x\n", insn);
+		return INSN_REJECTED;
+	}
+
+	/* c.addisp16 imm */
+	if ((insn & C_ADDISP16_MASK) == C_ADDISP16_VAL) {
+		api->handler = simulate_caddisp16;
+	} else {
+		pr_warn("Rejected unknown instruction %x\n", insn);
+		return INSN_REJECTED;
+	}
+
+	return INSN_GOOD_NO_SLOT;
+}

arch/riscv/kernel/probes/decode-insn.h new file +23 -0

new file mode 100644
index 000000000000..0053ed6a308a
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef _RISCV_KERNEL_KPROBES_DECODE_INSN_H
+#define _RISCV_KERNEL_KPROBES_DECODE_INSN_H
+
+#include <asm/sections.h>
+#include <asm/kprobes.h>
+
+enum probe_insn {
+	INSN_REJECTED,
+	INSN_GOOD_NO_SLOT,
+};
+
+/*
+ * Compressed instruction format:
+ * xxxxxxxxxxxxxxaa where aa != 11
+ */
+#define is_compressed_insn(insn) ((insn & 0x3) != 0x3)
+
+#ifdef CONFIG_KPROBES
+enum probe_insn __kprobes
+riscv_probe_decode_insn(kprobe_opcode_t *addr, struct arch_probe_insn *asi);
+#endif
+#endif /* _RISCV_KERNEL_KPROBES_DECODE_INSN_H */

arch/riscv/kernel/probes/kprobes.c new file +35 -0

new file mode 100644
index 000000000000..3c7b5cf72ee1
@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * Kprobes support for RISC-V
+ *
+ * Author: Patrick St?hlin <[email protected]>
+ */
+#include <linux/kprobes.h>
+#include <linux/extable.h>
+#include <linux/slab.h>
+#include <asm/ptrace.h>
+#include <linux/uaccess.h>
+#include <asm/sections.h>
+
+#include "decode-insn.h"
+
+DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
+DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
+
+static void __kprobes
+post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
+
+static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
+{
+	if (is_compressed_insn(opcode))
+		*(u16 *)addr = cpu_to_le16(opcode);
+	else
+		*addr = cpu_to_le32(opcode);
+
+	return 0;
+}
+
+static void __kprobes arch_prepare_simulate(struct kprobe *p)
+{
+	unsigned long offset = is_compressed_insn(p->opcode) ? 2 : 4;

arch/riscv/kernel/probes/kprobes.c new file +100 -0

+
+	/* single instruction simulated, now go for post processing */
+	post_kprobe_handler(kcb, regs);
+}
+
+int __kprobes arch_prepare_kprobe(struct kprobe *p)
+{
+	unsigned long probe_addr = (unsigned long)p->addr;
+	extern char __start_rodata[];
+	extern char __end_rodata[];
+
+	if (probe_addr & 0x1) {
+		pr_warn("Address not aligned.\n");
+		return -EINVAL;
+	}
+
+	/* copy instruction */
+	p->opcode = le32_to_cpu(*p->addr);
+
+	if (probe_addr >= (unsigned long) __start_rodata &&
+	    probe_addr <= (unsigned long) __end_rodata) {
+		return -EINVAL;
+	}
+
+	/* decode instruction */
+	switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) {
+	case INSN_REJECTED:	/* insn not supported */
+		return -EINVAL;
+
+	case INSN_GOOD_NO_SLOT:	/* insn needs simulation */
+		break;
+	}
+
+	arch_prepare_simulate(p);
+
+	return 0;
+}
+
+#define C_EBREAK_OPCODE 0x9002
+
+/* arm kprobe: install breakpoint in text */
+void __kprobes arch_arm_kprobe(struct kprobe *p)
+{
+	patch_text(p->addr, C_EBREAK_OPCODE);
+}
+
+/* disarm kprobe: remove breakpoint from text */
+void __kprobes arch_disarm_kprobe(struct kprobe *p)
+{
+	patch_text(p->addr, p->opcode);
+}
+
+void __kprobes arch_remove_kprobe(struct kprobe *p)
+{
+}
+
+static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+	kcb->prev_kprobe.kp = kprobe_running();
+	kcb->prev_kprobe.status = kcb->kprobe_status;
+}
+
+static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
+	kcb->kprobe_status = kcb->prev_kprobe.status;
+}
+
+static void __kprobes set_current_kprobe(struct kprobe *p)
+{
+	__this_cpu_write(current_kprobe, p);
+}
+
+static void __kprobes simulate(struct kprobe *p,
+			       struct pt_regs *regs,
+			       struct kprobe_ctlblk *kcb, int reenter)
+{
+	if (reenter) {
+		save_previous_kprobe(kcb);
+		set_current_kprobe(p);
+		kcb->kprobe_status = KPROBE_REENTER;
+	} else {
+		kcb->kprobe_status = KPROBE_HIT_SS;
+	}
+
+	arch_simulate_insn(p, regs);
+}
+
+static int __kprobes reenter_kprobe(struct kprobe *p,
+				    struct pt_regs *regs,
+				    struct kprobe_ctlblk *kcb)
+{
+	switch (kcb->kprobe_status) {
+	case KPROBE_HIT_SSDONE:
+	case KPROBE_HIT_ACTIVE:
+		kprobes_inc_nmissed_count(p);
+		simulate(p, regs, kcb, 1);
+		break;
+	case KPROBE_HIT_SS:
+	case KPROBE_REENTER:

arch/riscv/kernel/probes/kprobes.c new file +39 -0

+}
+
+static void __kprobes
+post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
+{
+	struct kprobe *cur = kprobe_running();
+
+	if (!cur)
+		return;
+
+	/* return addr restore if non-branching insn */
+	if (cur->ainsn.api.restore != 0)
+		instruction_pointer_set(regs, cur->ainsn.api.restore);
+
+	/* restore back original saved kprobe variables and continue */
+	if (kcb->kprobe_status == KPROBE_REENTER) {
+		restore_previous_kprobe(kcb);
+		return;
+	}
+	/* call post handler */
+	kcb->kprobe_status = KPROBE_HIT_SSDONE;
+	if (cur->post_handler)	{
+		/* post_handler can hit breakpoint and single step
+		 * again, so we enable D-flag for recursive exception.
+		 */
+		cur->post_handler(cur, regs, 0);
+	}
+
+	reset_current_kprobe();
+}
+
+int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int cause)
+{
+	struct kprobe *cur = kprobe_running();
+	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+
+	switch (kcb->kprobe_status) {
+	case KPROBE_HIT_SS:
+	case KPROBE_REENTER:

arch/riscv/kernel/probes/kprobes.c new file +28 -0

+
+		if (kcb->kprobe_status == KPROBE_REENTER)
+			restore_previous_kprobe(kcb);
+		else
+			reset_current_kprobe();
+
+		break;
+	case KPROBE_HIT_ACTIVE:
+	case KPROBE_HIT_SSDONE:
+		/*
+		 * We increment the nmissed count for accounting,
+		 * we can also use npre/npostfault count for accounting
+		 * these specific fault cases.
+		 */
+		kprobes_inc_nmissed_count(cur);
+
+		/*
+		 * We come here because instructions in the pre/post
+		 * handler caused the page_fault, this could happen
+		 * if handler tries to access user space by
+		 * copy_from_user(), get_user() etc. Let the
+		 * user-specified handler try to fix it first.
+		 */
+		if (cur->fault_handler && cur->fault_handler(cur, regs, cause))
+			return 1;
+
+		/*
+		 * In case the user-specified fault handler returned

arch/riscv/kernel/probes/kprobes.c new file +18 -0

+	struct kprobe *p, *cur_kprobe;
+	struct kprobe_ctlblk *kcb;
+	unsigned long addr = instruction_pointer(regs);
+
+	kcb = get_kprobe_ctlblk();
+	cur_kprobe = kprobe_running();
+
+	p = get_kprobe((kprobe_opcode_t *) addr);
+
+	if (p) {
+		if (cur_kprobe) {
+			if (reenter_kprobe(p, regs, kcb))
+				return;
+		} else {
+			/* Probe hit */
+			set_current_kprobe(p);
+			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+

arch/riscv/kernel/probes/kprobes.c new file +5 -0

+			 */
+			if (!p->pre_handler || !p->pre_handler(p, regs))
+				simulate(p, regs, kcb, 0);
+			else
+				reset_current_kprobe();

arch/riscv/kernel/probes/kprobes.c new file +14 -0

+}
+
+bool arch_within_kprobe_blacklist(unsigned long addr)
+{
+	if ((addr >= (unsigned long)__kprobes_text_start &&
+	    addr < (unsigned long)__kprobes_text_end) ||
+	    (addr >= (unsigned long)__entry_text_start &&
+	    addr < (unsigned long)__entry_text_end) ||
+	    !!search_exception_tables(addr))
+		return true;
+
+	return false;
+}
+

arch/riscv/kernel/probes/kprobes.c new file +85 -0

+	INIT_HLIST_HEAD(&empty_rp);
+	kretprobe_hash_lock(current, &head, &flags);
+
+	/*
+	 * It is possible to have multiple instances associated with a given
+	 * task either because multiple functions in the call path have
+	 * return probes installed on them, and/or more than one
+	 * return probe was registered for a target function.
+	 *
+	 * We can handle this because:
+	 *     - instances are always pushed into the head of the list
+	 *     - when multiple return probes are registered for the same
+	 *	 function, the (chronologically) first instance's ret_addr
+	 *	 will be the real return address, and all the rest will
+	 *	 point to kretprobe_trampoline.
+	 */
+	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
+		if (ri->task != current)
+			/* another task is sharing our hash bucket */
+			continue;
+
+		orig_ret_address = (unsigned long)ri->ret_addr;
+
+		if (orig_ret_address != trampoline_address)
+			/*
+			 * This is the real return address. Any other
+			 * instances associated with this task are for
+			 * other calls deeper on the call stack
+			 */
+			break;
+	}
+
+	kretprobe_assert(ri, orig_ret_address, trampoline_address);
+
+	correct_ret_addr = ri->ret_addr;
+	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
+		if (ri->task != current)
+			/* another task is sharing our hash bucket */
+			continue;
+
+		orig_ret_address = (unsigned long)ri->ret_addr;
+		if (ri->rp && ri->rp->handler) {
+			__this_cpu_write(current_kprobe, &ri->rp->kp);
+			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
+			ri->ret_addr = correct_ret_addr;
+			ri->rp->handler(ri, regs);
+			__this_cpu_write(current_kprobe, NULL);
+		}
+
+		recycle_rp_inst(ri, &empty_rp);
+
+		if (orig_ret_address != trampoline_address)
+			/*
+			 * This is the real return address. Any other
+			 * instances associated with this task are for
+			 * other calls deeper on the call stack
+			 */
+			break;
+	}
+
+	kretprobe_hash_unlock(current, &flags);
+
+	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
+		hlist_del(&ri->hlist);
+		kfree(ri);
+	}
+	return (void *)orig_ret_address;
+}
+
+void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
+				      struct pt_regs *regs)
+{
+	ri->ret_addr = (kprobe_opcode_t *)regs->ra;
+	regs->ra = (long)&kretprobe_trampoline;
+}
+
+int __kprobes arch_trampoline_kprobe(struct kprobe *p)
+{
+	return 0;
+}
+
+int __init arch_init_kprobes(void)
+{
+	return 0;
+}

arch/riscv/kernel/probes/kprobes_trampoline.S new file +91 -0

new file mode 100644
index 000000000000..c7ceda9556a3
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.altmacro
+
+	.macro save_all_base_regs
+	REG_S x1,  PT_RA(sp)
+	REG_S x3,  PT_GP(sp)
+	REG_S x4,  PT_TP(sp)
+	REG_S x5,  PT_T0(sp)
+	REG_S x6,  PT_T1(sp)
+	REG_S x7,  PT_T2(sp)
+	REG_S x8,  PT_S0(sp)
+	REG_S x9,  PT_S1(sp)
+	REG_S x10, PT_A0(sp)
+	REG_S x11, PT_A1(sp)
+	REG_S x12, PT_A2(sp)
+	REG_S x13, PT_A3(sp)
+	REG_S x14, PT_A4(sp)
+	REG_S x15, PT_A5(sp)
+	REG_S x16, PT_A6(sp)
+	REG_S x17, PT_A7(sp)
+	REG_S x18, PT_S2(sp)
+	REG_S x19, PT_S3(sp)
+	REG_S x20, PT_S4(sp)
+	REG_S x21, PT_S5(sp)
+	REG_S x22, PT_S6(sp)
+	REG_S x23, PT_S7(sp)
+	REG_S x24, PT_S8(sp)
+	REG_S x25, PT_S9(sp)
+	REG_S x26, PT_S10(sp)
+	REG_S x27, PT_S11(sp)
+	REG_S x28, PT_T3(sp)
+	REG_S x29, PT_T4(sp)
+	REG_S x30, PT_T5(sp)
+	REG_S x31, PT_T6(sp)
+	.endm
+
+	.macro restore_all_base_regs
+	REG_L x3,  PT_GP(sp)
+	REG_L x4,  PT_TP(sp)
+	REG_L x5,  PT_T0(sp)
+	REG_L x6,  PT_T1(sp)
+	REG_L x7,  PT_T2(sp)
+	REG_L x8,  PT_S0(sp)
+	REG_L x9,  PT_S1(sp)
+	REG_L x10, PT_A0(sp)
+	REG_L x11, PT_A1(sp)
+	REG_L x12, PT_A2(sp)
+	REG_L x13, PT_A3(sp)
+	REG_L x14, PT_A4(sp)
+	REG_L x15, PT_A5(sp)
+	REG_L x16, PT_A6(sp)
+	REG_L x17, PT_A7(sp)
+	REG_L x18, PT_S2(sp)
+	REG_L x19, PT_S3(sp)
+	REG_L x20, PT_S4(sp)
+	REG_L x21, PT_S5(sp)
+	REG_L x22, PT_S6(sp)
+	REG_L x23, PT_S7(sp)
+	REG_L x24, PT_S8(sp)
+	REG_L x25, PT_S9(sp)
+	REG_L x26, PT_S10(sp)
+	REG_L x27, PT_S11(sp)
+	REG_L x28, PT_T3(sp)
+	REG_L x29, PT_T4(sp)
+	REG_L x30, PT_T5(sp)
+	REG_L x31, PT_T6(sp)
+	.endm
+
+ENTRY(kretprobe_trampoline)
+	addi sp, sp, -(PT_SIZE_ON_STACK)
+	save_all_base_regs
+
+	move a0, sp /* pt_regs */
+
+	call trampoline_probe_handler
+
+	/* use the result as the return-address */
+	move ra, a0
+
+	restore_all_base_regs
+	addi sp, sp, PT_SIZE_ON_STACK
+
+	ret
+ENDPROC(kretprobe_trampoline)

arch/riscv/kernel/probes/simulate-insn.c new file +22 -0

new file mode 100644
index 000000000000..5734d9bae22f
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+
+#include "simulate-insn.h"
+
+#define bit_at(value, bit)		((value) & (1 << (bit)))
+#define move_bit_at(value, bit, to)	((bit_at(value, bit) >> bit) << to)
+
+void __kprobes
+simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs)
+{
+	s16 imm;
+
+	/*
+	 * Immediate value layout in c.addisp16:
+	 * xxx9 xxxx x468 75xx
+	 * 1  1    8    4    0
+	 * 5  2
+	 */

arch/riscv/kernel/probes/simulate-insn.c new file +1 -0

+}

arch/riscv/kernel/probes/simulate-insn.h new file +8 -0

new file mode 100644
index 000000000000..dc2c06c30167
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#ifndef _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+#define _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+
+void simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs);
+
+#endif /* _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H */

arch/riscv/kernel/traps.c +1 -0

index 24a9333dda2c..d7113178d401 100644
@@ -18,6 +18,7 @@
 #include <linux/sched/signal.h>
 #include <linux/signal.h>
 #include <linux/kdebug.h>
+#include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/mm.h>
 #include <linux/module.h>

arch/riscv/kernel/traps.c +5 -2

+#endif
 #ifdef CONFIG_GENERIC_BUG
-	if (!user_mode(regs)) {
+	if (!handler_found && !user_mode(regs)) {
 		enum bug_trap_type type;
 
 		type = report_bug(regs->sepc, regs);

@@ -137,7 +144,9 @@ asmlinkage void do_trap_break(struct pt_regs *regs)
 	}
 #endif /* CONFIG_GENERIC_BUG */
 
-	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)(regs->sepc), current);
+	if (!handler_found)
+		force_sig_fault(SIGTRAP, TRAP_BRKPT,
+			(void __user *)(regs->sepc), current);
 }
 
 #ifdef CONFIG_GENERIC_BUG

arch/riscv/mm/fault.c +1 -0

index 88401d5125bc..eff816e33479 100644
@@ -22,6 +22,7 @@
 
 #include <linux/mm.h>
 #include <linux/kernel.h>
+#include <linux/kprobes.h>
 #include <linux/interrupt.h>
 #include <linux/perf_event.h>
 #include <linux/signal.h>

arch/riscv/mm/fault.c +24 -1

+{
+	int ret = 0;
+
+	/* kprobe_running() needs smp_processor_id() */
+	if (!user_mode(regs)) {
+		preempt_disable();
+		if (kprobe_running() && kprobe_fault_handler(regs, cause))
+			ret = 1;
+		preempt_enable();
+	}
+
+	return ret;
+}
+#else
+static inline int notify_page_fault(struct pt_regs *regs, unsigned int cause)
+{
+	return 0;
+}
+#endif
+
 /*
  * This routine handles page faults.  It determines the address and the
  * problem, and then passes it off to one of the appropriate routines.
  */
-asmlinkage void do_page_fault(struct pt_regs *regs)
+asmlinkage void __kprobes do_page_fault(struct pt_regs *regs)
 {
 	struct task_struct *tsk;
 	struct vm_area_struct *vma;

@@ -47,6 +70,9 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
 	cause = regs->scause;
 	addr = regs->sbadaddr;
 
+	if (notify_page_fault(regs, cause))
+		return;
+
 	tsk = current;
 	mm = tsk->mm;
 

Signature · 1 line

2.17.1

arch/riscv/kernel/probes/kprobes.c line 29

Quoted 8 lines

arch/riscv/kernel/probes/kprobes.c new file +8 -0

+
+static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
+{
+	if (is_compressed_insn(opcode))
+		*(u16 *)addr = cpu_to_le16(opcode);
+	else
+		*addr = cpu_to_le32(opcode);
+

Quoted 72 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +66 -0

new file mode 100644
index 000000000000..c7ceda9556a3
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.altmacro
+
+	.macro save_all_base_regs
+	REG_S x1,  PT_RA(sp)
+	REG_S x3,  PT_GP(sp)
+	REG_S x4,  PT_TP(sp)
+	REG_S x5,  PT_T0(sp)
+	REG_S x6,  PT_T1(sp)
+	REG_S x7,  PT_T2(sp)
+	REG_S x8,  PT_S0(sp)
+	REG_S x9,  PT_S1(sp)
+	REG_S x10, PT_A0(sp)
+	REG_S x11, PT_A1(sp)
+	REG_S x12, PT_A2(sp)
+	REG_S x13, PT_A3(sp)
+	REG_S x14, PT_A4(sp)
+	REG_S x15, PT_A5(sp)
+	REG_S x16, PT_A6(sp)
+	REG_S x17, PT_A7(sp)
+	REG_S x18, PT_S2(sp)
+	REG_S x19, PT_S3(sp)
+	REG_S x20, PT_S4(sp)
+	REG_S x21, PT_S5(sp)
+	REG_S x22, PT_S6(sp)
+	REG_S x23, PT_S7(sp)
+	REG_S x24, PT_S8(sp)
+	REG_S x25, PT_S9(sp)
+	REG_S x26, PT_S10(sp)
+	REG_S x27, PT_S11(sp)
+	REG_S x28, PT_T3(sp)
+	REG_S x29, PT_T4(sp)
+	REG_S x30, PT_T5(sp)
+	REG_S x31, PT_T6(sp)
+	.endm
+
+	.macro restore_all_base_regs
+	REG_L x3,  PT_GP(sp)
+	REG_L x4,  PT_TP(sp)
+	REG_L x5,  PT_T0(sp)
+	REG_L x6,  PT_T1(sp)
+	REG_L x7,  PT_T2(sp)
+	REG_L x8,  PT_S0(sp)
+	REG_L x9,  PT_S1(sp)
+	REG_L x10, PT_A0(sp)
+	REG_L x11, PT_A1(sp)
+	REG_L x12, PT_A2(sp)
+	REG_L x13, PT_A3(sp)
+	REG_L x14, PT_A4(sp)
+	REG_L x15, PT_A5(sp)
+	REG_L x16, PT_A6(sp)
+	REG_L x17, PT_A7(sp)
+	REG_L x18, PT_S2(sp)
+	REG_L x19, PT_S3(sp)
+	REG_L x20, PT_S4(sp)
+	REG_L x21, PT_S5(sp)
+	REG_L x22, PT_S6(sp)
+	REG_L x23, PT_S7(sp)
+	REG_L x24, PT_S8(sp)

arch/riscv/kernel/probes/kprobes_trampoline.S line 74

Quoted 8 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +8 -0

+	REG_L x25, PT_S9(sp)
+	REG_L x26, PT_S10(sp)
+	REG_L x27, PT_S11(sp)
+	REG_L x28, PT_T3(sp)
+	REG_L x29, PT_T4(sp)
+	REG_L x30, PT_T5(sp)
+	REG_L x31, PT_T6(sp)
+	.endm

Quoted 8 lines · On Wed, 14 Nov 2018 00:37:30 -0800 Masami Hiramatsu <[email protected]> wrote:

arch/riscv/kernel/probes/kprobes.c line 29

Quoted 8 lines

arch/riscv/kernel/probes/kprobes.c new file +8 -0

+
+static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
+{
+	if (is_compressed_insn(opcode))
+		*(u16 *)addr = cpu_to_le16(opcode);
+	else
+		*addr = cpu_to_le32(opcode);
+

BTW, don't RISC-V need any i-cache flush and per-core serialization for patching the text area? (and no text_mutex protection?)

Quoted 76 lines

Quoted 76 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +69 -0

new file mode 100644
index 000000000000..c7ceda9556a3
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.altmacro
+
+	.macro save_all_base_regs
+	REG_S x1,  PT_RA(sp)
+	REG_S x3,  PT_GP(sp)
+	REG_S x4,  PT_TP(sp)
+	REG_S x5,  PT_T0(sp)
+	REG_S x6,  PT_T1(sp)
+	REG_S x7,  PT_T2(sp)
+	REG_S x8,  PT_S0(sp)
+	REG_S x9,  PT_S1(sp)
+	REG_S x10, PT_A0(sp)
+	REG_S x11, PT_A1(sp)
+	REG_S x12, PT_A2(sp)
+	REG_S x13, PT_A3(sp)
+	REG_S x14, PT_A4(sp)
+	REG_S x15, PT_A5(sp)
+	REG_S x16, PT_A6(sp)
+	REG_S x17, PT_A7(sp)
+	REG_S x18, PT_S2(sp)
+	REG_S x19, PT_S3(sp)
+	REG_S x20, PT_S4(sp)
+	REG_S x21, PT_S5(sp)
+	REG_S x22, PT_S6(sp)
+	REG_S x23, PT_S7(sp)
+	REG_S x24, PT_S8(sp)
+	REG_S x25, PT_S9(sp)
+	REG_S x26, PT_S10(sp)
+	REG_S x27, PT_S11(sp)
+	REG_S x28, PT_T3(sp)
+	REG_S x29, PT_T4(sp)
+	REG_S x30, PT_T5(sp)
+	REG_S x31, PT_T6(sp)
+	.endm
+
+	.macro restore_all_base_regs
+	REG_L x3,  PT_GP(sp)
+	REG_L x4,  PT_TP(sp)
+	REG_L x5,  PT_T0(sp)
+	REG_L x6,  PT_T1(sp)
+	REG_L x7,  PT_T2(sp)
+	REG_L x8,  PT_S0(sp)
+	REG_L x9,  PT_S1(sp)
+	REG_L x10, PT_A0(sp)
+	REG_L x11, PT_A1(sp)
+	REG_L x12, PT_A2(sp)
+	REG_L x13, PT_A3(sp)
+	REG_L x14, PT_A4(sp)
+	REG_L x15, PT_A5(sp)
+	REG_L x16, PT_A6(sp)
+	REG_L x17, PT_A7(sp)
+	REG_L x18, PT_S2(sp)
+	REG_L x19, PT_S3(sp)
+	REG_L x20, PT_S4(sp)
+	REG_L x21, PT_S5(sp)
+	REG_L x22, PT_S6(sp)
+	REG_L x23, PT_S7(sp)
+	REG_L x24, PT_S8(sp)
+	REG_L x25, PT_S9(sp)
+	REG_L x26, PT_S10(sp)
+	REG_L x27, PT_S11(sp)

Quoted 5 lines

arch/riscv/kernel/probes/kprobes_trampoline.S line 74

Quoted 5 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +5 -0

+	REG_L x28, PT_T3(sp)
+	REG_L x29, PT_T4(sp)
+	REG_L x30, PT_T5(sp)
+	REG_L x31, PT_T6(sp)
+	.endm

It seems thses macros can be (partially?) shared with entry.S

Quoted 14 lines · Masami Hiramatsu

Quoted 8 lines · On Wed, 14 Nov 2018 00:37:30 -0800 Masami Hiramatsu <[email protected]> wrote:

arch/riscv/kernel/probes/kprobes.c line 29

Quoted 8 lines

arch/riscv/kernel/probes/kprobes.c new file +8 -0

+
+static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
+{
+	if (is_compressed_insn(opcode))
+		*(u16 *)addr = cpu_to_le16(opcode);
+	else
+		*addr = cpu_to_le32(opcode);
+

BTW, don't RISC-V need any i-cache flush and per-core serialization for patching the text area? (and no text_mutex protection?)

Yes, we should probably call flush_icache_all. This code works on QEMU/virt but I guess on real hardware you may run into problems, especially when disarming the kprobe. I'll have a look at the arm64 code again to see what's missing.

Quoted 75 lines

Quoted 75 lines

Quoted 75 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +69 -0

new file mode 100644
index 000000000000..c7ceda9556a3
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.altmacro
+
+	.macro save_all_base_regs
+	REG_S x1,  PT_RA(sp)
+	REG_S x3,  PT_GP(sp)
+	REG_S x4,  PT_TP(sp)
+	REG_S x5,  PT_T0(sp)
+	REG_S x6,  PT_T1(sp)
+	REG_S x7,  PT_T2(sp)
+	REG_S x8,  PT_S0(sp)
+	REG_S x9,  PT_S1(sp)
+	REG_S x10, PT_A0(sp)
+	REG_S x11, PT_A1(sp)
+	REG_S x12, PT_A2(sp)
+	REG_S x13, PT_A3(sp)
+	REG_S x14, PT_A4(sp)
+	REG_S x15, PT_A5(sp)
+	REG_S x16, PT_A6(sp)
+	REG_S x17, PT_A7(sp)
+	REG_S x18, PT_S2(sp)
+	REG_S x19, PT_S3(sp)
+	REG_S x20, PT_S4(sp)
+	REG_S x21, PT_S5(sp)
+	REG_S x22, PT_S6(sp)
+	REG_S x23, PT_S7(sp)
+	REG_S x24, PT_S8(sp)
+	REG_S x25, PT_S9(sp)
+	REG_S x26, PT_S10(sp)
+	REG_S x27, PT_S11(sp)
+	REG_S x28, PT_T3(sp)
+	REG_S x29, PT_T4(sp)
+	REG_S x30, PT_T5(sp)
+	REG_S x31, PT_T6(sp)
+	.endm
+
+	.macro restore_all_base_regs
+	REG_L x3,  PT_GP(sp)
+	REG_L x4,  PT_TP(sp)
+	REG_L x5,  PT_T0(sp)
+	REG_L x6,  PT_T1(sp)
+	REG_L x7,  PT_T2(sp)
+	REG_L x8,  PT_S0(sp)
+	REG_L x9,  PT_S1(sp)
+	REG_L x10, PT_A0(sp)
+	REG_L x11, PT_A1(sp)
+	REG_L x12, PT_A2(sp)
+	REG_L x13, PT_A3(sp)
+	REG_L x14, PT_A4(sp)
+	REG_L x15, PT_A5(sp)
+	REG_L x16, PT_A6(sp)
+	REG_L x17, PT_A7(sp)
+	REG_L x18, PT_S2(sp)
+	REG_L x19, PT_S3(sp)
+	REG_L x20, PT_S4(sp)
+	REG_L x21, PT_S5(sp)
+	REG_L x22, PT_S6(sp)
+	REG_L x23, PT_S7(sp)
+	REG_L x24, PT_S8(sp)
+	REG_L x25, PT_S9(sp)
+	REG_L x26, PT_S10(sp)
+	REG_L x27, PT_S11(sp)

Quoted 8 lines

Quoted 5 lines

arch/riscv/kernel/probes/kprobes_trampoline.S line 74

Quoted 5 lines

arch/riscv/kernel/probes/kprobes_trampoline.S new file +5 -0

+	REG_L x28, PT_T3(sp)
+	REG_L x29, PT_T4(sp)
+	REG_L x30, PT_T5(sp)
+	REG_L x31, PT_T6(sp)
+	.endm

It seems thses macros can be (partially?) shared with entry.S

Yes, I wanted to avoid somebody changing the shared code and breaking random things. But that's what reviews are for. I'll think of something for v2.

Are you still working on these kprobes/kretprobe patches for RISC-V?

Thank you very much for implementing kprobes on RISC-V :)

Quoted 4 lines · On Tue, 13 Nov 2018 20:58:04 +0100 Patrick St?hlin <[email protected]> wrote:

First implementation, adapted from arm64. The C.ADDISP16 instruction gets simulated and the kprobes-handler called by inserting a C.EBREAK instruction.

Quoted 3 lines

C.ADDISP16 was chosen as it sets-up the stack frame for functions. Some work has been done to support probes on non-compressed instructions but there is no support yet for decoding those.

Does this only support C.ADDISP16? No other insns are supported? Supporting 1 insn is too few I think.

Can RISC-V do single stepping? If not, we need to prepare emulator as match as possible, or for ALU instructions, we can run it on buffer and hook it.

Quoted 3 lines

The way forward should be to uncompress the instructions for simulation to reduce the number of instructions used to decode the immediate values on probe hit.

I have some comments on the patch, please review.

Quoted 112 lines

Signed-off-by Patrick St?hlin <[email protected]>

---
 arch/riscv/Kconfig                            |   5 +-
 arch/riscv/include/asm/kprobes.h              |  30 ++
 arch/riscv/include/asm/probes.h               |  26 ++
 arch/riscv/kernel/Makefile                    |   1 +
 arch/riscv/kernel/probes/Makefile             |   3 +
 arch/riscv/kernel/probes/decode-insn.c        |  38 ++
 arch/riscv/kernel/probes/decode-insn.h        |  23 +
 arch/riscv/kernel/probes/kprobes.c            | 401 ++++++++++++++++++
 arch/riscv/kernel/probes/kprobes_trampoline.S |  91 ++++
 arch/riscv/kernel/probes/simulate-insn.c      |  33 ++
 arch/riscv/kernel/probes/simulate-insn.h      |   8 +
 arch/riscv/kernel/traps.c                     |  13 +-
 arch/riscv/mm/fault.c                         |  28 +-
 13 files changed, 694 insertions(+), 6 deletions(-)
 create mode 100644 arch/riscv/include/asm/probes.h
 create mode 100644 arch/riscv/kernel/probes/Makefile
 create mode 100644 arch/riscv/kernel/probes/decode-insn.c
 create mode 100644 arch/riscv/kernel/probes/decode-insn.h
 create mode 100644 arch/riscv/kernel/probes/kprobes.c
 create mode 100644 arch/riscv/kernel/probes/kprobes_trampoline.S
 create mode 100644 arch/riscv/kernel/probes/simulate-insn.c
 create mode 100644 arch/riscv/kernel/probes/simulate-insn.h

arch/riscv/Kconfig +2 -3

index b157ac82d486..11ef4030e8f2 100644
@@ -44,6 +44,8 @@ config RISCV
 	select GENERIC_IRQ_MULTI_HANDLER
 	select ARCH_HAS_PTE_SPECIAL
 	select HAVE_REGS_AND_STACK_ACCESS_API
+	select HAVE_KPROBES
+	select HAVE_KRETPROBES
 
 config MMU
 	def_bool y

@@ -89,9 +91,6 @@ config PGTABLE_LEVELS
 	default 3 if 64BIT
 	default 2
 
-config HAVE_KPROBES
-	def_bool n
-
 menu "Platform type"
 
 choice

arch/riscv/include/asm/kprobes.h +30 -0

index c7eb010d1528..657adcd35a3d 100644
@@ -19,4 +19,34 @@
 
 #include <asm-generic/kprobes.h>
 
+#ifdef CONFIG_KPROBES
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/percpu.h>
+
+#define flush_insn_slot(p)		do { } while (0)
+#define kretprobe_blacklist_size	0
+
+#include <asm/probes.h>
+
+struct prev_kprobe {
+	struct kprobe *kp;
+	unsigned int status;
+};
+
+/* per-cpu kprobe control block */
+struct kprobe_ctlblk {
+	unsigned int kprobe_status;
+	struct prev_kprobe prev_kprobe;
+};
+
+void arch_remove_kprobe(struct kprobe *p);
+int kprobe_fault_handler(struct pt_regs *regs, unsigned int cause);
+int kprobe_exceptions_notify(struct notifier_block *self,
+			     unsigned long val, void *data);
+int kprobe_breakpoint_handler(struct pt_regs *regs);
+void kretprobe_trampoline(void);
+void __kprobes *trampoline_probe_handler(struct pt_regs *regs);
+
+#endif /* CONFIG_KPROBES */
 #endif /* _RISCV_KPROBES_H */

arch/riscv/include/asm/probes.h new file +18 -0

new file mode 100644
index 000000000000..64cf12567539
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Based on arch/arm64/include/asm/probes.h
+ *
+ * Copyright (C) 2013 Linaro Limited
+ */
+#ifndef _RISCV_PROBES_H
+#define _RISCV_PROBES_H
+
+typedef u32 probe_opcode_t;
+typedef void (probes_handler_t) (u32 opcode, long addr, struct pt_regs *);
+
+/* architecture specific copy of original instruction */
+struct arch_probe_insn {
+	probes_handler_t *handler;
+	/* restore address after simulation */
+	unsigned long restore;
+};

arch/riscv/include/asm/probes.h line 24

Quoted 6 lines

arch/riscv/include/asm/probes.h new file +6 -0

+#ifdef CONFIG_KPROBES
+typedef u32 kprobe_opcode_t;
+struct arch_specific_insn {
+	struct arch_probe_insn api;
+};
+#endif

Are there any reason of putting this kprobes dedicated data structure here?

Quoted 46 lines

arch/riscv/include/asm/probes.h new file +2 -0

+
+#endif /* _RISCV_PROBES_H */

arch/riscv/kernel/Makefile +1 -0

index f13f7f276639..5360a445b9d3 100644
@@ -28,6 +28,7 @@ obj-y	+= stacktrace.o
 obj-y	+= vdso.o
 obj-y	+= cacheinfo.o
 obj-y	+= vdso/
+obj-y	+= probes/
 
 CFLAGS_setup.o := -mcmodel=medany
 

arch/riscv/kernel/probes/Makefile new file +3 -0

new file mode 100644
index 000000000000..144d1c1743fb
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_KPROBES)		+= kprobes.o kprobes_trampoline.o \
+				   decode-insn.o simulate-insn.o

arch/riscv/kernel/probes/decode-insn.c new file +16 -0

new file mode 100644
index 000000000000..2d8f46f4c2e7
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <asm/sections.h>
+
+#include "decode-insn.h"
+#include "simulate-insn.h"
+
+#define C_ADDISP16_MASK 0x6F83
+#define C_ADDISP16_VAL  0x6101
+
+/* Return:
+ *   INSN_REJECTED     If instruction is one not allowed to kprobe,

arch/riscv/kernel/probes/decode-insn.c line 20

Quoted 4 lines

arch/riscv/kernel/probes/decode-insn.c new file +4 -0

+ *   INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot.
+ */
+enum probe_insn __kprobes
+riscv_probe_decode_insn(kprobe_opcode_t *addr, struct arch_probe_insn *api)

Please don't use __kprobes anymore. That is old stlye, instead, please use NOKPROBE_SYMBOL() or nokprobe_inline for static-inline function. (NOKPROBE_SYMBOL() will make the symbol non-inline always)

Quoted 93 lines

arch/riscv/kernel/probes/decode-insn.c new file +18 -0

+{
+	probe_opcode_t insn = le32_to_cpu(*addr);
+
+	if (!is_compressed_insn(insn)) {
+		pr_warn("Can't handle non-compressed instruction %x\n", insn);
+		return INSN_REJECTED;
+	}
+
+	/* c.addisp16 imm */
+	if ((insn & C_ADDISP16_MASK) == C_ADDISP16_VAL) {
+		api->handler = simulate_caddisp16;
+	} else {
+		pr_warn("Rejected unknown instruction %x\n", insn);
+		return INSN_REJECTED;
+	}
+
+	return INSN_GOOD_NO_SLOT;
+}

arch/riscv/kernel/probes/decode-insn.h new file +23 -0

new file mode 100644
index 000000000000..0053ed6a308a
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef _RISCV_KERNEL_KPROBES_DECODE_INSN_H
+#define _RISCV_KERNEL_KPROBES_DECODE_INSN_H
+
+#include <asm/sections.h>
+#include <asm/kprobes.h>
+
+enum probe_insn {
+	INSN_REJECTED,
+	INSN_GOOD_NO_SLOT,
+};
+
+/*
+ * Compressed instruction format:
+ * xxxxxxxxxxxxxxaa where aa != 11
+ */
+#define is_compressed_insn(insn) ((insn & 0x3) != 0x3)
+
+#ifdef CONFIG_KPROBES
+enum probe_insn __kprobes
+riscv_probe_decode_insn(kprobe_opcode_t *addr, struct arch_probe_insn *asi);
+#endif
+#endif /* _RISCV_KERNEL_KPROBES_DECODE_INSN_H */

arch/riscv/kernel/probes/kprobes.c new file +39 -0

new file mode 100644
index 000000000000..3c7b5cf72ee1
@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * Kprobes support for RISC-V
+ *
+ * Author: Patrick St?hlin <[email protected]>
+ */
+#include <linux/kprobes.h>
+#include <linux/extable.h>
+#include <linux/slab.h>
+#include <asm/ptrace.h>
+#include <linux/uaccess.h>
+#include <asm/sections.h>
+
+#include "decode-insn.h"
+
+DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
+DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
+
+static void __kprobes
+post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
+
+static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
+{
+	if (is_compressed_insn(opcode))
+		*(u16 *)addr = cpu_to_le16(opcode);
+	else
+		*addr = cpu_to_le32(opcode);
+
+	return 0;
+}
+
+static void __kprobes arch_prepare_simulate(struct kprobe *p)
+{
+	unsigned long offset = is_compressed_insn(p->opcode) ? 2 : 4;
+
+	p->ainsn.api.restore = (unsigned long)p->addr + offset;
+}
+

arch/riscv/kernel/probes/kprobes.c line 45

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
+{
+	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+
+	if (p->ainsn.api.handler)
+		p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs);

it seems api.handler must be here,

Quoted 105 lines

arch/riscv/kernel/probes/kprobes.c new file +104 -0

+
+	/* single instruction simulated, now go for post processing */
+	post_kprobe_handler(kcb, regs);
+}
+
+int __kprobes arch_prepare_kprobe(struct kprobe *p)
+{
+	unsigned long probe_addr = (unsigned long)p->addr;
+	extern char __start_rodata[];
+	extern char __end_rodata[];
+
+	if (probe_addr & 0x1) {
+		pr_warn("Address not aligned.\n");
+		return -EINVAL;
+	}
+
+	/* copy instruction */
+	p->opcode = le32_to_cpu(*p->addr);
+
+	if (probe_addr >= (unsigned long) __start_rodata &&
+	    probe_addr <= (unsigned long) __end_rodata) {
+		return -EINVAL;
+	}
+
+	/* decode instruction */
+	switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) {
+	case INSN_REJECTED:	/* insn not supported */
+		return -EINVAL;
+
+	case INSN_GOOD_NO_SLOT:	/* insn needs simulation */
+		break;
+	}
+
+	arch_prepare_simulate(p);
+
+	return 0;
+}
+
+#define C_EBREAK_OPCODE 0x9002
+
+/* arm kprobe: install breakpoint in text */
+void __kprobes arch_arm_kprobe(struct kprobe *p)
+{
+	patch_text(p->addr, C_EBREAK_OPCODE);
+}
+
+/* disarm kprobe: remove breakpoint from text */
+void __kprobes arch_disarm_kprobe(struct kprobe *p)
+{
+	patch_text(p->addr, p->opcode);
+}
+
+void __kprobes arch_remove_kprobe(struct kprobe *p)
+{
+}
+
+static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+	kcb->prev_kprobe.kp = kprobe_running();
+	kcb->prev_kprobe.status = kcb->kprobe_status;
+}
+
+static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
+	kcb->kprobe_status = kcb->prev_kprobe.status;
+}
+
+static void __kprobes set_current_kprobe(struct kprobe *p)
+{
+	__this_cpu_write(current_kprobe, p);
+}
+
+static void __kprobes simulate(struct kprobe *p,
+			       struct pt_regs *regs,
+			       struct kprobe_ctlblk *kcb, int reenter)
+{
+	if (reenter) {
+		save_previous_kprobe(kcb);
+		set_current_kprobe(p);
+		kcb->kprobe_status = KPROBE_REENTER;
+	} else {
+		kcb->kprobe_status = KPROBE_HIT_SS;
+	}
+
+	arch_simulate_insn(p, regs);
+}
+
+static int __kprobes reenter_kprobe(struct kprobe *p,
+				    struct pt_regs *regs,
+				    struct kprobe_ctlblk *kcb)
+{
+	switch (kcb->kprobe_status) {
+	case KPROBE_HIT_SSDONE:
+	case KPROBE_HIT_ACTIVE:
+		kprobes_inc_nmissed_count(p);
+		simulate(p, regs, kcb, 1);
+		break;
+	case KPROBE_HIT_SS:
+	case KPROBE_REENTER:
+		pr_warn("Unrecoverable kprobe detected.\n");
+		dump_kprobe(p);
+		BUG();
+		break;

arch/riscv/kernel/probes/kprobes.c line 155

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+	default:
+		WARN_ON(1);
+		return 0;
+	}
+
+	return 1;

If this always return 1, we can make it void return.

Quoted 44 lines

arch/riscv/kernel/probes/kprobes.c new file +43 -0

+}
+
+static void __kprobes
+post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
+{
+	struct kprobe *cur = kprobe_running();
+
+	if (!cur)
+		return;
+
+	/* return addr restore if non-branching insn */
+	if (cur->ainsn.api.restore != 0)
+		instruction_pointer_set(regs, cur->ainsn.api.restore);
+
+	/* restore back original saved kprobe variables and continue */
+	if (kcb->kprobe_status == KPROBE_REENTER) {
+		restore_previous_kprobe(kcb);
+		return;
+	}
+	/* call post handler */
+	kcb->kprobe_status = KPROBE_HIT_SSDONE;
+	if (cur->post_handler)	{
+		/* post_handler can hit breakpoint and single step
+		 * again, so we enable D-flag for recursive exception.
+		 */
+		cur->post_handler(cur, regs, 0);
+	}
+
+	reset_current_kprobe();
+}
+
+int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int cause)
+{
+	struct kprobe *cur = kprobe_running();
+	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+
+	switch (kcb->kprobe_status) {
+	case KPROBE_HIT_SS:
+	case KPROBE_REENTER:
+		/*
+		 * We are here because the instruction being single
+		 * stepped caused a page fault. We reset the current
+		 * kprobe and the ip points back to the probe address

arch/riscv/kernel/probes/kprobes.c line 204

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+		 * and allow the page fault handler to continue as a
+		 * normal page fault.
+		 */
+		instruction_pointer_set(regs, (unsigned long) cur->addr);
+		if (!instruction_pointer(regs))
+			BUG();

Use BUG_ON().

Quoted 33 lines

arch/riscv/kernel/probes/kprobes.c new file +32 -0

+
+		if (kcb->kprobe_status == KPROBE_REENTER)
+			restore_previous_kprobe(kcb);
+		else
+			reset_current_kprobe();
+
+		break;
+	case KPROBE_HIT_ACTIVE:
+	case KPROBE_HIT_SSDONE:
+		/*
+		 * We increment the nmissed count for accounting,
+		 * we can also use npre/npostfault count for accounting
+		 * these specific fault cases.
+		 */
+		kprobes_inc_nmissed_count(cur);
+
+		/*
+		 * We come here because instructions in the pre/post
+		 * handler caused the page_fault, this could happen
+		 * if handler tries to access user space by
+		 * copy_from_user(), get_user() etc. Let the
+		 * user-specified handler try to fix it first.
+		 */
+		if (cur->fault_handler && cur->fault_handler(cur, regs, cause))
+			return 1;
+
+		/*
+		 * In case the user-specified fault handler returned
+		 * zero, try to fix up.
+		 */
+		if (fixup_exception(regs))
+			return 1;

arch/riscv/kernel/probes/kprobes.c line 242

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+	}
+	return 0;
+}
+
+static void __kprobes kprobe_handler(struct pt_regs *regs)
+{

Does this handler run under local IRQ disabled? (for making sure)

Quoted 23 lines

arch/riscv/kernel/probes/kprobes.c new file +22 -0

+	struct kprobe *p, *cur_kprobe;
+	struct kprobe_ctlblk *kcb;
+	unsigned long addr = instruction_pointer(regs);
+
+	kcb = get_kprobe_ctlblk();
+	cur_kprobe = kprobe_running();
+
+	p = get_kprobe((kprobe_opcode_t *) addr);
+
+	if (p) {
+		if (cur_kprobe) {
+			if (reenter_kprobe(p, regs, kcb))
+				return;
+		} else {
+			/* Probe hit */
+			set_current_kprobe(p);
+			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+
+			/*
+			 * If we have no pre-handler or it returned 0, we
+			 * continue with normal processing.  If we have a
+			 * pre-handler and it returned non-zero, it will

arch/riscv/kernel/probes/kprobes.c line 270

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+			 * modify the execution path and no need to single
+			 * stepping. Let's just reset current kprobe and exit.
+			 *
+			 * pre_handler can hit a breakpoint and can step thru
+			 * before return, keep PSTATE D-flag enabled until
+			 * pre_handler return back.

Is this true on RISC-V too?

Quoted 11 lines

arch/riscv/kernel/probes/kprobes.c new file +10 -0

+			 */
+			if (!p->pre_handler || !p->pre_handler(p, regs))
+				simulate(p, regs, kcb, 0);
+			else
+				reset_current_kprobe();
+		}
+	}
+	/*
+	 * The breakpoint instruction was removed right
+	 * after we hit it.  Another cpu has removed

arch/riscv/kernel/probes/kprobes.c line 285

Quoted 5 lines

arch/riscv/kernel/probes/kprobes.c new file +5 -0

+	 * either a probepoint or a debugger breakpoint
+	 * at this address.  In either case, no further
+	 * handling of this interrupt is appropriate.
+	 * Return back to original instruction, and continue.
+	 */

This should return 0 if it doesn't handle anything, but if it handles c.break should return 1.

arch/riscv/kernel/probes/kprobes.c line 292

Quoted 7 lines

arch/riscv/kernel/probes/kprobes.c new file +7 -0

+}
+
+int __kprobes
+kprobe_breakpoint_handler(struct pt_regs *regs)
+{
+	kprobe_handler(regs);
+	return 1;

Why don't you call kprobe_handler directly?

Quoted 19 lines

arch/riscv/kernel/probes/kprobes.c new file +18 -0

+}
+
+bool arch_within_kprobe_blacklist(unsigned long addr)
+{
+	if ((addr >= (unsigned long)__kprobes_text_start &&
+	    addr < (unsigned long)__kprobes_text_end) ||
+	    (addr >= (unsigned long)__entry_text_start &&
+	    addr < (unsigned long)__entry_text_end) ||
+	    !!search_exception_tables(addr))
+		return true;
+
+	return false;
+}
+
+void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
+{
+	struct kretprobe_instance *ri = NULL;
+	struct hlist_head *head, empty_rp;

arch/riscv/kernel/probes/kprobes.c line 316

Quoted 6 lines

arch/riscv/kernel/probes/kprobes.c new file +6 -0

+	struct hlist_node *tmp;
+	unsigned long flags, orig_ret_address = 0;
+	unsigned long trampoline_address =
+		(unsigned long)&kretprobe_trampoline;
+	kprobe_opcode_t *correct_ret_addr = NULL;
+

It seems you don't setup instruction_pointer.

Quoted 215 lines

arch/riscv/kernel/probes/kprobes.c new file +85 -0

+	INIT_HLIST_HEAD(&empty_rp);
+	kretprobe_hash_lock(current, &head, &flags);
+
+	/*
+	 * It is possible to have multiple instances associated with a given
+	 * task either because multiple functions in the call path have
+	 * return probes installed on them, and/or more than one
+	 * return probe was registered for a target function.
+	 *
+	 * We can handle this because:
+	 *     - instances are always pushed into the head of the list
+	 *     - when multiple return probes are registered for the same
+	 *	 function, the (chronologically) first instance's ret_addr
+	 *	 will be the real return address, and all the rest will
+	 *	 point to kretprobe_trampoline.
+	 */
+	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
+		if (ri->task != current)
+			/* another task is sharing our hash bucket */
+			continue;
+
+		orig_ret_address = (unsigned long)ri->ret_addr;
+
+		if (orig_ret_address != trampoline_address)
+			/*
+			 * This is the real return address. Any other
+			 * instances associated with this task are for
+			 * other calls deeper on the call stack
+			 */
+			break;
+	}
+
+	kretprobe_assert(ri, orig_ret_address, trampoline_address);
+
+	correct_ret_addr = ri->ret_addr;
+	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
+		if (ri->task != current)
+			/* another task is sharing our hash bucket */
+			continue;
+
+		orig_ret_address = (unsigned long)ri->ret_addr;
+		if (ri->rp && ri->rp->handler) {
+			__this_cpu_write(current_kprobe, &ri->rp->kp);
+			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
+			ri->ret_addr = correct_ret_addr;
+			ri->rp->handler(ri, regs);
+			__this_cpu_write(current_kprobe, NULL);
+		}
+
+		recycle_rp_inst(ri, &empty_rp);
+
+		if (orig_ret_address != trampoline_address)
+			/*
+			 * This is the real return address. Any other
+			 * instances associated with this task are for
+			 * other calls deeper on the call stack
+			 */
+			break;
+	}
+
+	kretprobe_hash_unlock(current, &flags);
+
+	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
+		hlist_del(&ri->hlist);
+		kfree(ri);
+	}
+	return (void *)orig_ret_address;
+}
+
+void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
+				      struct pt_regs *regs)
+{
+	ri->ret_addr = (kprobe_opcode_t *)regs->ra;
+	regs->ra = (long)&kretprobe_trampoline;
+}
+
+int __kprobes arch_trampoline_kprobe(struct kprobe *p)
+{
+	return 0;
+}
+
+int __init arch_init_kprobes(void)
+{
+	return 0;
+}

arch/riscv/kernel/probes/kprobes_trampoline.S new file +91 -0

new file mode 100644
index 000000000000..c7ceda9556a3
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.altmacro
+
+	.macro save_all_base_regs
+	REG_S x1,  PT_RA(sp)
+	REG_S x3,  PT_GP(sp)
+	REG_S x4,  PT_TP(sp)
+	REG_S x5,  PT_T0(sp)
+	REG_S x6,  PT_T1(sp)
+	REG_S x7,  PT_T2(sp)
+	REG_S x8,  PT_S0(sp)
+	REG_S x9,  PT_S1(sp)
+	REG_S x10, PT_A0(sp)
+	REG_S x11, PT_A1(sp)
+	REG_S x12, PT_A2(sp)
+	REG_S x13, PT_A3(sp)
+	REG_S x14, PT_A4(sp)
+	REG_S x15, PT_A5(sp)
+	REG_S x16, PT_A6(sp)
+	REG_S x17, PT_A7(sp)
+	REG_S x18, PT_S2(sp)
+	REG_S x19, PT_S3(sp)
+	REG_S x20, PT_S4(sp)
+	REG_S x21, PT_S5(sp)
+	REG_S x22, PT_S6(sp)
+	REG_S x23, PT_S7(sp)
+	REG_S x24, PT_S8(sp)
+	REG_S x25, PT_S9(sp)
+	REG_S x26, PT_S10(sp)
+	REG_S x27, PT_S11(sp)
+	REG_S x28, PT_T3(sp)
+	REG_S x29, PT_T4(sp)
+	REG_S x30, PT_T5(sp)
+	REG_S x31, PT_T6(sp)
+	.endm
+
+	.macro restore_all_base_regs
+	REG_L x3,  PT_GP(sp)
+	REG_L x4,  PT_TP(sp)
+	REG_L x5,  PT_T0(sp)
+	REG_L x6,  PT_T1(sp)
+	REG_L x7,  PT_T2(sp)
+	REG_L x8,  PT_S0(sp)
+	REG_L x9,  PT_S1(sp)
+	REG_L x10, PT_A0(sp)
+	REG_L x11, PT_A1(sp)
+	REG_L x12, PT_A2(sp)
+	REG_L x13, PT_A3(sp)
+	REG_L x14, PT_A4(sp)
+	REG_L x15, PT_A5(sp)
+	REG_L x16, PT_A6(sp)
+	REG_L x17, PT_A7(sp)
+	REG_L x18, PT_S2(sp)
+	REG_L x19, PT_S3(sp)
+	REG_L x20, PT_S4(sp)
+	REG_L x21, PT_S5(sp)
+	REG_L x22, PT_S6(sp)
+	REG_L x23, PT_S7(sp)
+	REG_L x24, PT_S8(sp)
+	REG_L x25, PT_S9(sp)
+	REG_L x26, PT_S10(sp)
+	REG_L x27, PT_S11(sp)
+	REG_L x28, PT_T3(sp)
+	REG_L x29, PT_T4(sp)
+	REG_L x30, PT_T5(sp)
+	REG_L x31, PT_T6(sp)
+	.endm
+
+ENTRY(kretprobe_trampoline)
+	addi sp, sp, -(PT_SIZE_ON_STACK)
+	save_all_base_regs
+
+	move a0, sp /* pt_regs */
+
+	call trampoline_probe_handler
+
+	/* use the result as the return-address */
+	move ra, a0
+
+	restore_all_base_regs
+	addi sp, sp, PT_SIZE_ON_STACK
+
+	ret
+ENDPROC(kretprobe_trampoline)

arch/riscv/kernel/probes/simulate-insn.c new file +26 -0

new file mode 100644
index 000000000000..5734d9bae22f
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+
+#include "simulate-insn.h"
+
+#define bit_at(value, bit)		((value) & (1 << (bit)))
+#define move_bit_at(value, bit, to)	((bit_at(value, bit) >> bit) << to)
+
+void __kprobes
+simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs)
+{
+	s16 imm;
+
+	/*
+	 * Immediate value layout in c.addisp16:
+	 * xxx9 xxxx x468 75xx
+	 * 1  1    8    4    0
+	 * 5  2
+	 */
+	imm = sign_extend32(
+		move_bit_at(opcode, 12, 9) |
+		move_bit_at(opcode, 6, 4) |
+		move_bit_at(opcode, 5, 6) |

arch/riscv/kernel/probes/simulate-insn.c line 32

Quoted 6 lines

arch/riscv/kernel/probes/simulate-insn.c new file +6 -0

+		move_bit_at(opcode, 4, 8) |
+		move_bit_at(opcode, 3, 7) |
+		move_bit_at(opcode, 2, 5),
+		9);
+
+	regs->sp += imm;

What about updating regs->sepc?

Quoted 30 lines

arch/riscv/kernel/probes/simulate-insn.c new file +1 -0

+}

arch/riscv/kernel/probes/simulate-insn.h new file +8 -0

new file mode 100644
index 000000000000..dc2c06c30167
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#ifndef _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+#define _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+
+void simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs);
+
+#endif /* _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H */

arch/riscv/kernel/traps.c +1 -0

index 24a9333dda2c..d7113178d401 100644
@@ -18,6 +18,7 @@
 #include <linux/sched/signal.h>
 #include <linux/signal.h>
 #include <linux/kdebug.h>
+#include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/mm.h>
 #include <linux/module.h>

@@ -120,8 +121,14 @@ DO_ERROR_INFO(do_trap_ecall_m,
 

arch/riscv/kernel/traps.c DO_ERROR_INFO(do_trap_ecall_m, · line 128

Quoted 7 lines

arch/riscv/kernel/traps.c +5 -0

 asmlinkage void do_trap_break(struct pt_regs *regs)
 {
+	bool handler_found = false;
+
+#ifdef CONFIG_KPROBES
+	if (kprobe_breakpoint_handler(regs))
+		handler_found = 1;

Why don't you just return from here?

Quoted 35 lines

arch/riscv/kernel/traps.c +5 -2

+#endif
 #ifdef CONFIG_GENERIC_BUG
-	if (!user_mode(regs)) {
+	if (!handler_found && !user_mode(regs)) {
 		enum bug_trap_type type;
 
 		type = report_bug(regs->sepc, regs);

@@ -137,7 +144,9 @@ asmlinkage void do_trap_break(struct pt_regs *regs)
 	}
 #endif /* CONFIG_GENERIC_BUG */
 
-	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)(regs->sepc), current);
+	if (!handler_found)
+		force_sig_fault(SIGTRAP, TRAP_BRKPT,
+			(void __user *)(regs->sepc), current);
 }
 
 #ifdef CONFIG_GENERIC_BUG

arch/riscv/mm/fault.c +1 -0

index 88401d5125bc..eff816e33479 100644
@@ -22,6 +22,7 @@
 
 #include <linux/mm.h>
 #include <linux/kernel.h>
+#include <linux/kprobes.h>
 #include <linux/interrupt.h>
 #include <linux/perf_event.h>
 #include <linux/signal.h>

@@ -30,11 +31,33 @@
 #include <asm/pgalloc.h>
 #include <asm/ptrace.h>
 

arch/riscv/mm/fault.c line 35

Quoted 2 lines

arch/riscv/mm/fault.c +2 -0

+#ifdef CONFIG_KPROBES
+static inline int notify_page_fault(struct pt_regs *regs, unsigned int cause)

please use nokprobe_inline.

Quoted 43 lines

arch/riscv/mm/fault.c +24 -1

+{
+	int ret = 0;
+
+	/* kprobe_running() needs smp_processor_id() */
+	if (!user_mode(regs)) {
+		preempt_disable();
+		if (kprobe_running() && kprobe_fault_handler(regs, cause))
+			ret = 1;
+		preempt_enable();
+	}
+
+	return ret;
+}
+#else
+static inline int notify_page_fault(struct pt_regs *regs, unsigned int cause)
+{
+	return 0;
+}
+#endif
+
 /*
  * This routine handles page faults.  It determines the address and the
  * problem, and then passes it off to one of the appropriate routines.
  */
-asmlinkage void do_page_fault(struct pt_regs *regs)
+asmlinkage void __kprobes do_page_fault(struct pt_regs *regs)
 {
 	struct task_struct *tsk;
 	struct vm_area_struct *vma;

@@ -47,6 +70,9 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
 	cause = regs->scause;
 	addr = regs->sbadaddr;
 
+	if (notify_page_fault(regs, cause))
+		return;
+
 	tsk = current;
 	mm = tsk->mm;
 

Signature · 1 line

2.17.1

Thank you,

Hi Masami,

thank you for your remarks.

On 14.11.18 09:37, Masami Hiramatsu wrote>

Quoted 13 lines

Thank you very much for implementing kprobes on RISC-V :)

Quoted 7 lines · On Tue, 13 Nov 2018 20:58:04 +0100 Patrick St?hlin <[email protected]> wrote:

First implementation, adapted from arm64. The C.ADDISP16 instruction gets simulated and the kprobes-handler called by inserting a C.EBREAK instruction.

C.ADDISP16 was chosen as it sets-up the stack frame for functions. Some work has been done to support probes on non-compressed instructions but there is no support yet for decoding those.

Quoted 2 lines

Does this only support C.ADDISP16? No other insns are supported? Supporting 1 insn is too few I think.

At the moment, yes. I'm waiting for some input from somebody with deeper insights into the RISC-V architecture than me before implementing more instructions, should solution I've chosen be woefully inadequate.

Quoted 3 lines

Can RISC-V do single stepping? If not, we need to prepare emulator as match as possible, or for ALU instructions, we can run it on buffer and hook it.

The debug-specification is still a draft but there are some softcores that implement it. But even if it was finalized I don't think this will be made a mandatory extension so we need to simulate/emulate a good part of the instruction set anyway.

Quoted 22 lines

Quoted 22 lines

arch/riscv/kernel/probes/decode-insn.c new file +16 -0

new file mode 100644
index 000000000000..2d8f46f4c2e7
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <asm/sections.h>
+
+#include "decode-insn.h"
+#include "simulate-insn.h"
+
+#define C_ADDISP16_MASK 0x6F83
+#define C_ADDISP16_VAL  0x6101
+
+/* Return:
+ *   INSN_REJECTED     If instruction is one not allowed to kprobe,

Quoted 8 lines

arch/riscv/kernel/probes/decode-insn.c line 20

Quoted 4 lines

arch/riscv/kernel/probes/decode-insn.c new file +4 -0

+ *   INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot.
+ */
+enum probe_insn __kprobes
+riscv_probe_decode_insn(kprobe_opcode_t *addr, struct arch_probe_insn *api)

Please don't use __kprobes anymore. That is old stlye, instead, please use NOKPROBE_SYMBOL() or nokprobe_inline for static-inline function. (NOKPROBE_SYMBOL() will make the symbol non-inline always)

OK, should I make a note to change that in the arm64 port as well in a separate patch?

Quoted 5 lines

arch/riscv/kernel/probes/decode-insn.c line 23

Quoted 3 lines

arch/riscv/kernel/probes/decode-insn.c new file +3 -0

+
+static void __kprobes kprobe_handler(struct pt_regs *regs)
+{

Does this handler run under local IRQ disabled? (for making sure)

Exceptions are being handled with locals IRQs _enabled_.

As we've not
implemented any simulated instructions to modify the instruction pointer, I think this is safe? Then again, I'm new to this, so please bear with me.

Quoted 22 lines

Quoted 22 lines

arch/riscv/kernel/probes/decode-insn.c new file +22 -0

+	struct kprobe *p, *cur_kprobe;
+	struct kprobe_ctlblk *kcb;
+	unsigned long addr = instruction_pointer(regs);
+
+	kcb = get_kprobe_ctlblk();
+	cur_kprobe = kprobe_running();
+
+	p = get_kprobe((kprobe_opcode_t *) addr);
+
+	if (p) {
+		if (cur_kprobe) {
+			if (reenter_kprobe(p, regs, kcb))
+				return;
+		} else {
+			/* Probe hit */
+			set_current_kprobe(p);
+			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+
+			/*
+			 * If we have no pre-handler or it returned 0, we
+			 * continue with normal processing.  If we have a
+			 * pre-handler and it returned non-zero, it will

Quoted 8 lines

arch/riscv/kernel/probes/decode-insn.c

Quoted 6 lines

arch/riscv/kernel/probes/decode-insn.c new file +6 -0

+			 * modify the execution path and no need to single
+			 * stepping. Let's just reset current kprobe and exit.
+			 *
+			 * pre_handler can hit a breakpoint and can step thru
+			 * before return, keep PSTATE D-flag enabled until
+			 * pre_handler return back.

Is this true on RISC-V too?

It's not as we don't have a debug-unit at the moment. I'll remove the second part of the comment block.

Quoted 10 lines

Quoted 10 lines

arch/riscv/kernel/probes/decode-insn.c new file +10 -0

+			 */
+			if (!p->pre_handler || !p->pre_handler(p, regs))
+				simulate(p, regs, kcb, 0);
+			else
+				reset_current_kprobe();
+		}
+	}
+	/*
+	 * The breakpoint instruction was removed right
+	 * after we hit it.  Another cpu has removed

Quoted 8 lines

arch/riscv/kernel/probes/decode-insn.c

Quoted 5 lines

arch/riscv/kernel/probes/decode-insn.c new file +5 -0

+	 * either a probepoint or a debugger breakpoint
+	 * at this address.  In either case, no further
+	 * handling of this interrupt is appropriate.
+	 * Return back to original instruction, and continue.
+	 */

This should return 0 if it doesn't handle anything, but if it handles c.break should return 1.

I thought so too, but didn't insert one because of the comment (again, copied from arm64). If get_kprobe doesn't return a result, it could have been removed between the time the exception got raised or is the comment just wrong? On the other hand, solving it this way effectively means that we'll silently drop any other exceptions.

Quoted 9 lines

arch/riscv/kernel/probes/decode-insn.c

Quoted 7 lines

arch/riscv/kernel/probes/decode-insn.c new file +7 -0

+}
+
+int __kprobes
+kprobe_breakpoint_handler(struct pt_regs *regs)
+{
+	kprobe_handler(regs);
+	return 1;

Why don't you call kprobe_handler directly?

I should.

Quoted 19 lines

Quoted 19 lines

arch/riscv/kernel/probes/decode-insn.c new file +18 -0

+}
+
+bool arch_within_kprobe_blacklist(unsigned long addr)
+{
+	if ((addr >= (unsigned long)__kprobes_text_start &&
+	    addr < (unsigned long)__kprobes_text_end) ||
+	    (addr >= (unsigned long)__entry_text_start &&
+	    addr < (unsigned long)__entry_text_end) ||
+	    !!search_exception_tables(addr))
+		return true;
+
+	return false;
+}
+
+void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
+{
+	struct kretprobe_instance *ri = NULL;
+	struct hlist_head *head, empty_rp;

Quoted 8 lines

arch/riscv/kernel/probes/decode-insn.c

Quoted 6 lines

arch/riscv/kernel/probes/decode-insn.c new file +6 -0

+	struct hlist_node *tmp;
+	unsigned long flags, orig_ret_address = 0;
+	unsigned long trampoline_address =
+		(unsigned long)&kretprobe_trampoline;
+	kprobe_opcode_t *correct_ret_addr = NULL;
+

It seems you don't setup instruction_pointer.

I don't think I get what you mean by that. The instruction pointer (or rather the return address register) gets set by kretprobe_trampoline.S to the address returned from this function.

Quoted 32 lines

Quoted 32 lines

arch/riscv/kernel/probes/simulate-insn.c new file +26 -0

new file mode 100644
index 000000000000..5734d9bae22f
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+
+#include "simulate-insn.h"
+
+#define bit_at(value, bit)		((value) & (1 << (bit)))
+#define move_bit_at(value, bit, to)	((bit_at(value, bit) >> bit) << to)
+
+void __kprobes
+simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs)
+{
+	s16 imm;
+
+	/*
+	 * Immediate value layout in c.addisp16:
+	 * xxx9 xxxx x468 75xx
+	 * 1  1    8    4    0
+	 * 5  2
+	 */
+	imm = sign_extend32(
+		move_bit_at(opcode, 12, 9) |
+		move_bit_at(opcode, 6, 4) |
+		move_bit_at(opcode, 5, 6) |

Quoted 8 lines

arch/riscv/kernel/probes/simulate-insn.c line 32

Quoted 6 lines

arch/riscv/kernel/probes/simulate-insn.c new file +6 -0

+		move_bit_at(opcode, 4, 8) |
+		move_bit_at(opcode, 3, 7) |
+		move_bit_at(opcode, 2, 5),
+		9);
+
+	regs->sp += imm;

What about updating regs->sepc?

sepc gets updated by instruction_pointer_set in kprobe_handler

Quoted 29 lines

Quoted 29 lines

arch/riscv/kernel/probes/simulate-insn.c new file +1 -0

+}

arch/riscv/kernel/probes/simulate-insn.h new file +8 -0

new file mode 100644
index 000000000000..dc2c06c30167
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#ifndef _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+#define _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H
+
+void simulate_caddisp16(u32 opcode, long addr, struct pt_regs *regs);
+
+#endif /* _RISCV_KERNEL_KPROBES_SIMULATE_INSN_H */

arch/riscv/kernel/traps.c +1 -0

index 24a9333dda2c..d7113178d401 100644
@@ -18,6 +18,7 @@
 #include <linux/sched/signal.h>
 #include <linux/signal.h>
 #include <linux/kdebug.h>
+#include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/mm.h>
 #include <linux/module.h>

@@ -120,8 +121,14 @@ DO_ERROR_INFO(do_trap_ecall_m,
 

Quoted 9 lines

arch/riscv/kernel/traps.c DO_ERROR_INFO(do_trap_ecall_m, · line 128

Quoted 7 lines

arch/riscv/kernel/traps.c +5 -0

 asmlinkage void do_trap_break(struct pt_regs *regs)
 {
+	bool handler_found = false;
+
+#ifdef CONFIG_KPROBES
+	if (kprobe_breakpoint_handler(regs))
+		handler_found = 1;

Why don't you just return from here?

Following the pattern I've seen in other places, I can change that.

Hi all

Quoted 15 lines · Zong Li

Quoted 14 lines · Zong Li

From: [email protected] (Masami Hiramatsu)

Quoted 7 lines · On Wed, 14 Nov 2018 21:52:57 +0100 Patrick Staehlin <[email protected]> wrote:

Yeah, I think it's simpler.

And I found that the kprobe_breakpoint_handler() was called without checking !user_mode(regs). In that case, you should add the check in front of kprobe_breakpoint_handler() call.

Thank you,

Quoted 4 lines

Quoted 4 lines

Hi all,

Is there any update? I was wondering if this patch are keep going? If not, I think I could pick it up to go head

I am still working on it, albeit slowly, holiday season coming up here in Switzerland may accelerate that a bit. All the feedback I got from Masami has been implemented.

Additionally I added instruction simulation for everything except memory accesses. I am currently working on getting compressed instructions decoded properly into regular instructions but that is very tedious work. I guess I am two or three full days of work away from getting a RFC/RFT v2 series ready I guess that will happen in early January.

What I currently have is at [1], that is mostly untested as of now (beware I will rebase/squash that branch regularly). What I could use help with in the future (as in for v2), is somebody testing it on real hardware, as I've never gotten any testing feedback on my original patch-series (that would not have worked properly because of the lacking cache-flush).

[1] https://github.com/packi/linux/tree/kprobes-riscv

Patrick

Quoted 24 lines · Patrick Stählin

Hi all

Quoted 19 lines · Zong Li

Quoted 18 lines · Zong Li

From: [email protected] (Masami Hiramatsu)

Quoted 7 lines · On Wed, 14 Nov 2018 21:52:57 +0100 Patrick Staehlin <[email protected]> wrote:

Yeah, I think it's simpler.

And I found that the kprobe_breakpoint_handler() was called without checking !user_mode(regs). In that case, you should add the check in front of kprobe_breakpoint_handler() call.

Thank you,

Hi all,

Is there any update? I was wondering if this patch are keep going? If not, I think I could pick it up to go head

Quoted 3 lines

I am still working on it, albeit slowly, holiday season coming up here in Switzerland may accelerate that a bit. All the feedback I got from Masami has been implemented.

As Masami's suggestion, I don't see stop_machine in your implementation [1], are there some concerns to use it on SMP?

Quoted 6 lines

Additionally I added instruction simulation for everything except memory accesses. I am currently working on getting compressed instructions decoded properly into regular instructions but that is very tedious work. I guess I am two or three full days of work away from getting a RFC/RFT v2 series ready I guess that will happen in early January.

Thanks for the efforts.

Quoted 7 lines

What I currently have is at [1], that is mostly untested as of now (beware I will rebase/squash that branch regularly). What I could use help with in the future (as in for v2), is somebody testing it on real hardware, as I've never gotten any testing feedback on my original patch-series (that would not have worked properly because of the lacking cache-flush).

I give some quick tests by using kprobe sanity test and simple LKM on Hifive Unleashed board,
it seems to work normally. I could help to test your next version patch as well.

Quoted 4 lines

[1] https://github.com/packi/linux/tree/kprobes-riscv

Patrick

Quoted 26 lines · Zong Li

Quoted 24 lines · Patrick Stählin

Hi all

Quoted 19 lines · Zong Li

Quoted 18 lines · Zong Li

From: [email protected] (Masami Hiramatsu)

Quoted 7 lines · On Wed, 14 Nov 2018 21:52:57 +0100 Patrick Staehlin <[email protected]> wrote:

Yeah, I think it's simpler.

And I found that the kprobe_breakpoint_handler() was called without checking !user_mode(regs). In that case, you should add the check in front of kprobe_breakpoint_handler() call.

Thank you,

Hi all,

Is there any update? I was wondering if this patch are keep going? If not, I think I could pick it up to go head

Quoted 6 lines

Quoted 3 lines

I am still working on it, albeit slowly, holiday season coming up here in Switzerland may accelerate that a bit. All the feedback I got from Masami has been implemented.

As Masami's suggestion, I don't see stop_machine in your implementation [1], are there some concerns to use it on SMP?

Hi all,
I introduced the patching code framework. Maybe it is helpful for self-modify code here. (Please see:
https://lkml.org/lkml/2020/3/9/679)
This patch series is reviewing, maybe we could apply the implementation after it was merged.

Quoted 20 lines

Quoted 6 lines

Additionally I added instruction simulation for everything except memory accesses. I am currently working on getting compressed instructions decoded properly into regular instructions but that is very tedious work. I guess I am two or three full days of work away from getting a RFC/RFT v2 series ready I guess that will happen in early January.

Thanks for the efforts.

Quoted 7 lines

What I currently have is at [1], that is mostly untested as of now (beware I will rebase/squash that branch regularly). What I could use help with in the future (as in for v2), is somebody testing it on real hardware, as I've never gotten any testing feedback on my original patch-series (that would not have worked properly because of the lacking cache-flush).

I give some quick tests by using kprobe sanity test and simple LKM on Hifive Unleashed board,
it seems to work normally. I could help to test your next version patch as well.

Quoted 5 lines

Quoted 5 lines

[1] https://github.com/packi/linux/tree/kprobes-riscv

Patrick

Quoted 35 lines · Zong Li

Quoted 26 lines · Zong Li

Quoted 24 lines · Patrick Stählin

Hi all

Quoted 19 lines · Zong Li

Quoted 18 lines · Zong Li

From: [email protected] (Masami Hiramatsu)

Quoted 7 lines · On Wed, 14 Nov 2018 21:52:57 +0100 Patrick Staehlin <[email protected]> wrote:

Yeah, I think it's simpler.

And I found that the kprobe_breakpoint_handler() was called without checking !user_mode(regs). In that case, you should add the check in front of kprobe_breakpoint_handler() call.

Thank you,

Hi all,

Is there any update? I was wondering if this patch are keep going? If not, I think I could pick it up to go head

Quoted 6 lines

Quoted 3 lines

I am still working on it, albeit slowly, holiday season coming up here in Switzerland may accelerate that a bit. All the feedback I got from Masami has been implemented.

As Masami's suggestion, I don't see stop_machine in your implementation [1], are there some concerns to use it on SMP?

Quoted 28 lines

Hi all,
I introduced the patching code framework. Maybe it is helpful for self-modify code here. (Please see:
https://lkml.org/lkml/2020/3/9/679)
This patch series is reviewing, maybe we could apply the implementation after it was merged.

Quoted 20 lines

Quoted 6 lines

Additionally I added instruction simulation for everything except memory accesses. I am currently working on getting compressed instructions decoded properly into regular instructions but that is very tedious work. I guess I am two or three full days of work away from getting a RFC/RFT v2 series ready I guess that will happen in early January.

Thanks for the efforts.

Quoted 7 lines

What I currently have is at [1], that is mostly untested as of now (beware I will rebase/squash that branch regularly). What I could use help with in the future (as in for v2), is somebody testing it on real hardware, as I've never gotten any testing feedback on my original patch-series (that would not have worked properly because of the lacking cache-flush).

I give some quick tests by using kprobe sanity test and simple LKM on Hifive Unleashed board,
it seems to work normally. I could help to test your next version patch as well.

Hi Patrick,

I have noticed that "riscv: introduce interfaces to patch kernel code" (mentioned by Zong above) has been merged in 5.7 kernel.

I was wondering if you plan to submit a new revision of the patchset?

Thanks,
david

Quoted 6 lines

Quoted 5 lines

Quoted 5 lines

[1] https://github.com/packi/linux/tree/kprobes-riscv

Patrick

Hi Patrick,

I'm the author of csky's kprobe & uprobe, please ref to kprobe [1], uprobe [2]

1: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=33e53ae1ce413a081254e686d9b27cc1b3585e2f
2: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=8f6bb793b2be82f1f73bfb416486f156f70b4314

May I continue your patches to finish the work? I'll keep the primary author's name of you with the patches which you've done.

Most of the instructions' emulation codes will be deprecated, but the rest is ok. Maybe you should finish executing out of the slot first and then implement pc-related instructions' emulation.

Quoted 83 lines · David Abdurachmanov

Quoted 35 lines · Zong Li

Quoted 26 lines · Zong Li

Quoted 24 lines · Patrick Stählin

Hi all

Quoted 19 lines · Zong Li

Quoted 18 lines · Zong Li

From: [email protected] (Masami Hiramatsu)

Quoted 7 lines · On Wed, 14 Nov 2018 21:52:57 +0100 Patrick Staehlin <[email protected]> wrote:

Yeah, I think it's simpler.

And I found that the kprobe_breakpoint_handler() was called without checking !user_mode(regs). In that case, you should add the check in front of kprobe_breakpoint_handler() call.

Thank you,

Hi all,

Is there any update? I was wondering if this patch are keep going? If not, I think I could pick it up to go head

Quoted 6 lines

Quoted 3 lines

I am still working on it, albeit slowly, holiday season coming up here in Switzerland may accelerate that a bit. All the feedback I got from Masami has been implemented.

As Masami's suggestion, I don't see stop_machine in your implementation [1], are there some concerns to use it on SMP?

Quoted 28 lines

Hi all,
I introduced the patching code framework. Maybe it is helpful for self-modify code here. (Please see:
https://lkml.org/lkml/2020/3/9/679)
This patch series is reviewing, maybe we could apply the implementation after it was merged.

Quoted 20 lines

Quoted 6 lines

Additionally I added instruction simulation for everything except memory accesses. I am currently working on getting compressed instructions decoded properly into regular instructions but that is very tedious work. I guess I am two or three full days of work away from getting a RFC/RFT v2 series ready I guess that will happen in early January.

Thanks for the efforts.

Quoted 7 lines

What I currently have is at [1], that is mostly untested as of now (beware I will rebase/squash that branch regularly). What I could use help with in the future (as in for v2), is somebody testing it on real hardware, as I've never gotten any testing feedback on my original patch-series (that would not have worked properly because of the lacking cache-flush).

I give some quick tests by using kprobe sanity test and simple LKM on Hifive Unleashed board,
it seems to work normally. I could help to test your next version patch as well.

Hi Patrick,

I have noticed that "riscv: introduce interfaces to patch kernel code" (mentioned by Zong above) has been merged in 5.7 kernel.

I was wondering if you plan to submit a new revision of the patchset?

Thanks,
david

Quoted 6 lines

Quoted 5 lines

Quoted 5 lines

[1] https://github.com/packi/linux/tree/kprobes-riscv

Patrick

Signature · 4 lines

Best Regards
 Guo Ren

ML: https://lore.kernel.org/linux-csky/

Hi Guo and Patrick,

Quoted 6 lines · On Wed, 17 Jun 2020 20:58:32 +0800 Guo Ren <[email protected]> wrote:

Hi Patrick,

I'm the author of csky's kprobe & uprobe, please ref to kprobe [1], uprobe [2]

1: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=33e53ae1ce413a081254e686d9b27cc1b3585e2f
2: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=8f6bb793b2be82f1f73bfb416486f156f70b4314

Thanks for the porting!

Quoted 4 lines

May I continue your patches to finish the work? I'll keep the primary author's name of you with the patches which you've done.

Quoted 3 lines

Most of the instructions' emulation codes will be deprecated, but the rest is ok. Maybe you should finish executing out of the slot first and then implement pc-related instructions' emulation.

I can review the patch if you update it. Anyway, my major concern was the self-modifying code, others are minor points.

Hi Guo and Patrick,

Quoted 6 lines · On Wed, 17 Jun 2020 20:58:32 +0800 Guo Ren <[email protected]> wrote:

Hi Patrick,

I'm the author of csky's kprobe & uprobe, please ref to kprobe [1], uprobe [2]

1: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=33e53ae1ce413a081254e686d9b27cc1b3585e2f
2: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/arch/csky?h=v5.8-rc1&id=8f6bb793b2be82f1f73bfb416486f156f70b4314

Thanks for the porting!

Quoted 4 lines

May I continue your patches to finish the work? I'll keep the primary author's name of you with the patches which you've done.

Quoted 3 lines

Most of the instructions' emulation codes will be deprecated, but the rest is ok. Maybe you should finish executing out of the slot first and then implement pc-related instructions' emulation.

I can review the patch if you update it. Anyway, my major concern was the self-modifying code, others are minor points.