tracing: wprobe: x86: Add wprobe for watchpoint

index cfb74be19994..61ade8b5c9d7 100644
@@ -2108,6 +2108,7 @@ bool filter_mce(struct mce *m)
 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 {
 	irqentry_state_t irq_state;
+	unsigned long dr7;
 
 	WARN_ON_ONCE(user_mode(regs));
 

@@ -2118,18 +2119,24 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 	if (mca_cfg.initialized && mce_check_crashing_cpu())
 		return;
 
+	dr7 = local_db_save();
 	irq_state = irqentry_nmi_enter(regs);
 
 	do_machine_check(regs);
 
 	irqentry_nmi_exit(regs, irq_state);
+	local_db_restore(dr7);
 }
 
 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
 {
+	unsigned long dr7;
+
 	irqentry_enter_from_user_mode(regs);
 
+	dr7 = local_db_save();
 	do_machine_check(regs);
+	local_db_restore(dr7);
 
 	irqentry_exit_to_user_mode(regs);
 }

@@ -2138,21 +2145,13 @@ static __always_inline void exc_machine_check_user(struct pt_regs *regs)
 /* MCE hit kernel mode */
 DEFINE_IDTENTRY_MCE(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 
 /* The user mode variant. */
 DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	exc_machine_check_user(regs);
-	local_db_restore(dr7);
 }
 
 #ifdef CONFIG_X86_FRED

@@ -2169,28 +2168,20 @@ DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
  */
 DEFINE_FREDENTRY_MCE(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	if (user_mode(regs))
 		exc_machine_check_user(regs);
 	else
 		exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 #endif
 #else
 /* 32bit unified entry point */
 DEFINE_IDTENTRY_RAW(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	if (user_mode(regs))
 		exc_machine_check_user(regs);
 	else
 		exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 #endif
 

index 00172f301f25..7d93be2872ff 100644
@@ -70,7 +70,8 @@ struct btf_anon_stack {
 const struct btf_member *btf_find_struct_member(struct btf *btf,
 						const struct btf_type *type,
 						const char *member_name,
-						u32 *anon_offset)
+						u32 *anon_offset,
+						const struct btf_type **member_type)
 {
 	struct btf_anon_stack *anon_stack;
 	const struct btf_member *member;

@@ -91,17 +92,23 @@ const struct btf_member *btf_find_struct_member(struct btf *btf,
 	for_each_member(i, type, member) {
 		if (!member->name_off) {
 			/* Anonymous union/struct: push it for later use */
+			u32 m_off = btf_type_kflag(type) ?
+				BTF_MEMBER_BIT_OFFSET(member->offset) :
+				member->offset;
+
 			if (btf_type_skip_modifiers(btf, member->type, &tid) &&
 			    top < BTF_ANON_STACK_MAX) {
 				anon_stack[top].tid = tid;
 				anon_stack[top++].offset =
-					cur_offset + member->offset;
+					cur_offset + m_off;
 			}
 		} else {
 			name = btf_name_by_offset(btf, member->name_off);
 			if (name && !strcmp(member_name, name)) {
 				if (anon_offset)
 					*anon_offset = cur_offset;
+				if (member_type)
+					*member_type = type;
 				goto out;
 			}
 		}

index 4bc44bc261e6..4bd26bceae23 100644
@@ -8,4 +8,5 @@ const struct btf_param *btf_get_func_param(const struct btf_type *func_proto,
 const struct btf_member *btf_find_struct_member(struct btf *btf,
 						const struct btf_type *type,
 						const char *member_name,
-						u32 *anon_offset);
+						u32 *anon_offset,
+						const struct btf_type **member_type);

index c4163904ba74..908b4b6bc2df 100644
@@ -625,6 +625,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty
 {
 	const struct btf_type *type = *ptype;
 	const struct btf_member *field;
+	const struct btf_type *mtype;
 	struct btf *btf = ctx_btf(ctx);
 	char *fieldname = *pfieldname;
 	int bitoffs = 0;

@@ -640,7 +641,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty
 
 		anon_offs = 0;
 		field = btf_find_struct_member(btf, type, fieldname,
-						&anon_offs);
+						&anon_offs, &mtype);
 		if (IS_ERR(field)) {
 			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 			return PTR_ERR(field);

@@ -653,7 +654,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty
 		bitoffs += anon_offs;
 
 		/* Accumulate the bit-offsets of the dot-connected fields */
-		if (btf_type_kflag(type)) {
+		if (btf_type_kflag(mtype)) {
 			bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
 			ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
 		} else {

@@ -661,11 +662,11 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty
 			ctx->last_bitsize = 0;
 		}
 
-			type = btf_type_skip_modifiers(btf, field->type, NULL);
-			if (!type) {
-				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
-				return -EINVAL;
-			}
+		type = btf_type_skip_modifiers(btf, field->type, NULL);
+		if (!type) {
+			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+			return -EINVAL;
+		}
 
 		if (next)
 			ctx->offset += next - fieldname;

index 8c4f3bb24429..e6de7ae55bda 100644
@@ -181,6 +181,7 @@ struct kprobe_blacklist_entry {
 	struct list_head list;
 	unsigned long start_addr;
 	unsigned long end_addr;
+	struct rcu_head rcu;
 };
 
 #ifdef CONFIG_KPROBES

index bfc89083daa9..6337da5cab9e 100644
@@ -1447,8 +1447,14 @@ static bool __within_kprobe_blacklist(unsigned long addr)
 	/*
 	 * If 'kprobe_blacklist' is defined, check the address and
 	 * reject any probe registration in the prohibited area.
+	 * Note: this can return true during transition period where
+	 * (start_addr, end_addr) in the black list is shrinking
+	 * but old entry has not been removed yet. This is acceptable
+	 * because the worst case is that we reject more probes than
+	 * we should.
 	 */
-	list_for_each_entry(ent, &kprobe_blacklist, list) {
+	guard(rcu)();
+	list_for_each_entry_rcu(ent, &kprobe_blacklist, list) {
 		if (addr >= ent->start_addr && addr < ent->end_addr)
 			return true;
 	}

@@ -2509,7 +2515,7 @@ int kprobe_add_ksym_blacklist(unsigned long entry)
 	ent->start_addr = entry;
 	ent->end_addr = entry + size;
 	INIT_LIST_HEAD(&ent->list);
-	list_add_tail(&ent->list, &kprobe_blacklist);
+	list_add_tail_rcu(&ent->list, &kprobe_blacklist);
 
 	return (int)size;
 }

@@ -2603,8 +2609,8 @@ static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
 	list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
 		if (ent->start_addr < start || ent->start_addr >= end)
 			continue;
-		list_del(&ent->list);
-		kfree(ent);
+		list_del_rcu(&ent->list);
+		kfree_rcu(ent, rcu);
 	}
 }
 

index a2c1f2d24b64..65e80cd62193 100644
@@ -18,6 +18,7 @@
 #define DR7_FIXED_1	0x00000400
 
 DECLARE_PER_CPU(unsigned long, cpu_dr7);
+DECLARE_PER_CPU(unsigned int, cpu_dr7_seq);
 
 #ifndef CONFIG_PARAVIRT_XXL
 /*

@@ -125,18 +126,20 @@ static __always_inline bool hw_breakpoint_active(void)
 
 extern void hw_breakpoint_restore(void);
 
-static __always_inline unsigned long local_db_save(void)
+static __always_inline void local_db_save(unsigned long *dr7,
+					  unsigned int *dr7_seq)
 {
-	unsigned long dr7;
+	*dr7 = 0;
+	*dr7_seq = this_cpu_read(cpu_dr7_seq);
 
 	if (static_cpu_has(X86_FEATURE_HYPERVISOR) && !hw_breakpoint_active())
-		return 0;
+		return;
 
-	get_debugreg(dr7, 7);
+	get_debugreg(*dr7, 7);
 
 	/* Architecturally set bit */
-	dr7 &= ~DR7_FIXED_1;
-	if (dr7)
+	*dr7 &= ~DR7_FIXED_1;
+	if (*dr7)
 		set_debugreg(DR7_FIXED_1, 7);
 
 	/*

@@ -145,20 +148,34 @@ static __always_inline unsigned long local_db_save(void)
 	 * be good.
 	 */
 	barrier();
-
-	return dr7;
 }
 
-static __always_inline void local_db_restore(unsigned long dr7)
+static __always_inline void local_db_restore(unsigned long dr7,
+					     unsigned int dr7_seq)
 {
+	unsigned int seq;
+
 	/*
 	 * Ensure the compiler doesn't raise this statement into
 	 * the critical section; enabling breakpoints early would
 	 * not be good.
 	 */
 	barrier();
-	if (dr7)
+
+	if (!dr7)
+		return;
+
+	do {
+		seq = this_cpu_read(cpu_dr7_seq);
+		if (seq != dr7_seq) {
+			dr7 = this_cpu_read(cpu_dr7);
+			if (!dr7)
+				dr7 = DR7_FIXED_1;
+		}
+
 		set_debugreg(dr7, 7);
+		barrier();
+	} while (unlikely(seq != this_cpu_read(cpu_dr7_seq)));
 }
 
 #ifdef CONFIG_CPU_SUP_AMD

index 61ade8b5c9d7..d45e8b780deb 100644
@@ -2109,6 +2109,7 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 {
 	irqentry_state_t irq_state;
 	unsigned long dr7;
+	unsigned int dr7_seq;
 
 	WARN_ON_ONCE(user_mode(regs));
 

@@ -2119,24 +2120,25 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 	if (mca_cfg.initialized && mce_check_crashing_cpu())
 		return;
 
-	dr7 = local_db_save();
+	local_db_save(&dr7, &dr7_seq);
 	irq_state = irqentry_nmi_enter(regs);
 
 	do_machine_check(regs);
 
 	irqentry_nmi_exit(regs, irq_state);
-	local_db_restore(dr7);
+	local_db_restore(dr7, dr7_seq);
 }
 
 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
 {
 	unsigned long dr7;
+	unsigned int dr7_seq;
 
 	irqentry_enter_from_user_mode(regs);
 
-	dr7 = local_db_save();
+	local_db_save(&dr7, &dr7_seq);
 	do_machine_check(regs);
-	local_db_restore(dr7);
+	local_db_restore(dr7, dr7_seq);
 
 	irqentry_exit_to_user_mode(regs);
 }

156 lines not on this page · raw message

index 3c9f60d6ca5a..c3806f7bd3e8 100644
@@ -531,11 +531,12 @@ enum nmi_states {
 };
 static DEFINE_PER_CPU(enum nmi_states, nmi_state);
 static DEFINE_PER_CPU(unsigned long, nmi_cr2);
-static DEFINE_PER_CPU(unsigned long, nmi_dr7);
 
 DEFINE_IDTENTRY_RAW(exc_nmi)
 {
 	irqentry_state_t irq_state;
+	unsigned long dr7;
+	unsigned int dr7_seq;
 	struct nmi_stats *nsp = this_cpu_ptr(&nmi_stats);
 
 	/*

@@ -572,7 +573,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi)
 	 */
 	sev_es_ist_enter(regs);
 
-	this_cpu_write(nmi_dr7, local_db_save());
+	local_db_save(&dr7, &dr7_seq);
 
 	irq_state = irqentry_nmi_enter(regs);
 

@@ -594,7 +595,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi)
 
 	irqentry_nmi_exit(regs, irq_state);
 
-	local_db_restore(this_cpu_read(nmi_dr7));
+	local_db_restore(dr7, dr7_seq);
 
 	sev_es_ist_exit();
 

index 30aa8369957e..018abe736285 100644
@@ -1231,8 +1231,12 @@ static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6)
 	 * it results in an endless recursion and stack overflow. Thus we stay
 	 * with the IDT approach, i.e., save DR7 and disable #DB.
 	 */
-	unsigned long dr7 = local_db_save();
-	irqentry_state_t irq_state = irqentry_nmi_enter(regs);
+	unsigned long dr7;
+	unsigned int dr7_seq;
+	irqentry_state_t irq_state;
+
+	local_db_save(&dr7, &dr7_seq);
+	irq_state = irqentry_nmi_enter(regs);
 	instrumentation_begin();
 
 	/*

@@ -1289,7 +1293,7 @@ static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6)
 	instrumentation_end();
 	irqentry_nmi_exit(regs, irq_state);
 
-	local_db_restore(dr7);
+	local_db_restore(dr7, dr7_seq);
 }
 
 static noinstr void exc_debug_user(struct pt_regs *regs, unsigned long dr6)

index 0bc931cd0698..f35ec491f6dd 100644
@@ -59,6 +59,8 @@ extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
 
 
 int arch_install_hw_breakpoint(struct perf_event *bp);
+int arch_modify_local_hw_breakpoint_addr(struct perf_event *bp,
+					 unsigned long addr);
 void arch_uninstall_hw_breakpoint(struct perf_event *bp);
 void hw_breakpoint_pmu_read(struct perf_event *bp);
 void hw_breakpoint_pmu_unthrottle(struct perf_event *bp);

50 lines not on this page · raw message

index db199d653dd1..bf65c7fffd99 100644
@@ -81,6 +81,9 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 			    perf_overflow_handler_t triggered,
 			    void *context);
 
+int modify_local_hw_breakpoint_addr(struct perf_event *bp,
+				    unsigned long addr);
+
 extern int register_perf_hw_breakpoint(struct perf_event *bp);
 extern void unregister_hw_breakpoint(struct perf_event *bp);
 extern void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events);

@@ -124,6 +127,9 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 			    perf_overflow_handler_t triggered,
 			    void *context)		{ return NULL; }
 static inline int
+modify_local_hw_breakpoint_addr(struct perf_event *bp,
+				unsigned long addr) { return -EOPNOTSUPP; }
+static inline int
 register_perf_hw_breakpoint(struct perf_event *bp)	{ return -ENOSYS; }
 static inline void unregister_hw_breakpoint(struct perf_event *bp)	{ }
 static inline void

54 lines not on this page · raw message

index 5cbd09c8be8d..43ffd9a76d88 100644
@@ -337,6 +337,7 @@ enum {
 	TRACE_EVENT_FL_UPROBE_BIT,
 	TRACE_EVENT_FL_EPROBE_BIT,
 	TRACE_EVENT_FL_FPROBE_BIT,
+	TRACE_EVENT_FL_WPROBE_BIT,
 	TRACE_EVENT_FL_CUSTOM_BIT,
 	TRACE_EVENT_FL_TEST_STR_BIT,
 };

@@ -367,6 +368,7 @@ enum {
 	TRACE_EVENT_FL_UPROBE		= (1 << TRACE_EVENT_FL_UPROBE_BIT),
 	TRACE_EVENT_FL_EPROBE		= (1 << TRACE_EVENT_FL_EPROBE_BIT),
 	TRACE_EVENT_FL_FPROBE		= (1 << TRACE_EVENT_FL_FPROBE_BIT),
+	TRACE_EVENT_FL_WPROBE		= (1 << TRACE_EVENT_FL_WPROBE_BIT),
 	TRACE_EVENT_FL_CUSTOM		= (1 << TRACE_EVENT_FL_CUSTOM_BIT),
 	TRACE_EVENT_FL_TEST_STR		= (1 << TRACE_EVENT_FL_TEST_STR_BIT),
 };

index 3e0907aef172..0e7063b2380e 100644
@@ -4294,8 +4294,12 @@ static const char readme_msg[] =
 	"  uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
 	"\t\t\t  Write into this file to define/undefine new trace events.\n"
 #endif
+#ifdef CONFIG_WPROBE_EVENTS
+	"  wprobe_events\t\t- Create/append/remove/show the hardware breakpoint dynamic events\n"
+	"\t\t\t  Write into this file to define/undefine new trace events.\n"
+#endif
 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \
-    defined(CONFIG_FPROBE_EVENTS)
+    defined(CONFIG_FPROBE_EVENTS) || defined(CONFIG_WPROBE_EVENTS)
 	"\t  accepts: event-definitions (one definition per line)\n"
 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
 	"\t   Format: p[:[<group>/][<event>]] <place> [<args>]\n"

@@ -4305,6 +4309,9 @@ static const char readme_msg[] =
 	"\t           f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
 	"\t           t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
 #endif
+#ifdef CONFIG_WPROBE_EVENTS
+	"\t           w[:[<group>/][<event>]] [r|w|rw]@<addr>[:<len>] [<args>]\n"
+#endif
 #ifdef CONFIG_HIST_TRIGGERS
 	"\t           s:[synthetic/]<event> <field> [<field>]\n"
 #endif

index 74a7a50d1e78..18588c550a98 100644
@@ -179,6 +179,11 @@ struct fexit_trace_entry_head {
 	unsigned long		ret_ip;
 };
 
+struct wprobe_trace_entry_head {
+	struct trace_entry	ent;
+	unsigned long		ip;
+};
+
 #define TRACE_BUF_SIZE		1024
 
 struct trace_array;

index 908b4b6bc2df..9c7ebe326b76 100644
@@ -1403,6 +1403,23 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
 			return 0;
 	}
 
+	/* wprobe only support "$addr" and "$value" variable */
+	if (ctx->flags & TPARG_FL_WPROBE) {
+		if (!strcmp(arg, "addr")) {
+			code->op = FETCH_OP_BADDR;
+			return 0;
+		}
+		if (!strcmp(arg, "value")) {
+			code->op = FETCH_OP_BADDR;
+			code++;
+			code->op = FETCH_OP_DEREF;
+			code->offset = 0;
+			*pcode = code;
+			return 0;
+		}
+		goto inval;
+	}
+
 	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 		code->op = FETCH_OP_COMM;
 		return 0;

@@ -1462,8 +1479,8 @@ static int parse_probe_arg_register(char *arg, struct fetch_insn *code,
 {
 	int ret;
 
-	if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
-		/* eprobe and fprobe do not handle registers */
+	if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE | TPARG_FL_WPROBE)) {
+		/* eprobe, fprobe and wprobe do not handle registers */
 		trace_probe_log_err(ctx->offset, BAD_VAR);
 		return -EINVAL;
 	}

index fba1af092a9b..6543d4c2cda5 100644
@@ -90,6 +90,7 @@ typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
 	FETCH_OP(STACK, param),		/* Stack: .param = index */	\
 	FETCH_OP(STACKP, none),		/* Stack pointer */		\
 	FETCH_OP(RETVAL, none),		/* Return value */		\
+	FETCH_OP(BADDR, none),		/* Break address */		\
 	FETCH_OP(IMM, imm),		/* Immediate: .immediate */	\
 	FETCH_OP(COMM, none),		/* Current comm */		\
 	FETCH_OP(CURRENT, none),	/* Current task_struct address */\

@@ -418,6 +419,7 @@ static inline int traceprobe_get_entry_data_size(struct trace_probe *tp)
 #define TPARG_FL_USER   BIT(4)
 #define TPARG_FL_FPROBE BIT(5)
 #define TPARG_FL_TPOINT BIT(6)
+#define TPARG_FL_WPROBE BIT(7)
 #define TPARG_FL_LOC_MASK	GENMASK(4, 0)
 
 static inline bool tparg_is_function_entry(unsigned int flags)

@@ -544,6 +546,10 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(ARG_TOO_LONG,		"Argument expression is too long"),		\
 	C(ARRAY_NO_CLOSE,	"Array is not closed"),		\
 	C(ARRAY_TOO_BIG,	"Array number is too big"),		\
+	C(BAD_ACCESS_ADDR,	"Invalid access memory address"),		\
+	C(BAD_ACCESS_FMT,	"Access memory address requires @"),		\
+	C(BAD_ACCESS_LEN,	"This memory access length is not supported"),	\
+	C(BAD_ACCESS_TYPE,	"Bad memory access type"),			\
 	C(BAD_ADDR_SUFFIX,	"Invalid probed address suffix"),		\
 	C(BAD_ARG_NAME,		"Argument name must follow the same rules as C identifiers"),	\
 	C(BAD_ARG_NUM,		"Invalid argument number"),		\

@@ -626,7 +632,9 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(TYPECAST_NOT_EVENT,	"Typecasts are only for eprobe fields"),	\
 	C(TYPECAST_REQ_FIELD,	"Typecast requires a field access"),	\
 	C(TYPECAST_SYM_OFFSET,	"@SYM+/-OFFSET with typecast needs parentheses"),	\
-	C(USED_ARG_NAME,	"This argument name is already used"),
+	C(USED_ARG_NAME,	"This argument name is already used"),		\
+	C(WPROBE_NO_MAXACT,	"Watchpoint probe does not support maxactive"),	\
+	C(WPROBE_NO_SIBLING,	"Watchpoint probe does not support sibling probes"),
 
 #undef C
 #define C(a, b)		TP_ERR_##a

767 lines not on this page · raw message

index 43ffd9a76d88..f81a5308c116 100644
@@ -738,6 +738,7 @@ enum event_trigger_type {
 	ETT_EVENT_HIST		= (1 << 4),
 	ETT_HIST_ENABLE		= (1 << 5),
 	ETT_EVENT_EPROBE	= (1 << 6),
+	ETT_EVENT_WPROBE	= (1 << 7),
 };
 
 extern int filter_match_preds(struct event_filter *filter, void *rec);

index 18588c550a98..a61f840b18bd 100644
@@ -1983,6 +1983,7 @@ trigger_data_alloc(struct event_command *cmd_ops, char *cmd, char *param,
 		   void *private_data);
 extern void trigger_data_free(struct event_trigger_data *data);
 extern int event_trigger_init(struct event_trigger_data *data);
+extern void event_trigger_free(struct event_trigger_data *data);
 extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
 					      int trigger_enable);
 extern void update_cond_flag(struct trace_event_file *file);

index 149300cc5e8a..120e6ec634b7 100644
@@ -589,7 +589,7 @@ int event_trigger_init(struct event_trigger_data *data)
  * Usually used directly as the @free method in event trigger
  * implementations.
  */
-static void
+void
 event_trigger_free(struct event_trigger_data *data)
 {
 	if (WARN_ON_ONCE(data->ref <= 0))

index 9c7ebe326b76..5033fffd71c5 100644
@@ -20,7 +20,7 @@
 #undef C
 #define C(a, b)		b
 
-static const char *trace_probe_err_text[] = { ERRORS };
+const char *trace_probe_err_text[] = { ERRORS };
 
 static const char *reserved_field_names[] = {
 	"common_type",

index 6543d4c2cda5..e08f17c99138 100644
@@ -634,7 +634,12 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(TYPECAST_SYM_OFFSET,	"@SYM+/-OFFSET with typecast needs parentheses"),	\
 	C(USED_ARG_NAME,	"This argument name is already used"),		\
 	C(WPROBE_NO_MAXACT,	"Watchpoint probe does not support maxactive"),	\
-	C(WPROBE_NO_SIBLING,	"Watchpoint probe does not support sibling probes"),
+	C(WPROBE_NO_SIBLING,	"Watchpoint probe does not support sibling probes"), \
+	C(WPROBE_ON_KPROBE,	"Wprobe trigger is not supported on kprobe event"), \
+	C(WPROBE_NOT_FOUND,	"Target wprobe event is not found"),		\
+	C(WPROBE_BUSY,		"Target wprobe event is already enabled"),	\
+	C(WPROBE_NEED_FIELD,	"Wprobe trigger requires a target field"),	\
+	C(WPROBE_BAD_FIELD,	"Target field must be pointer size"),
 
 #undef C
 #define C(a, b)		TP_ERR_##a

@@ -658,6 +663,8 @@ void __trace_probe_log_err(int offset, int err);
 
 DEFINE_FREE(trace_probe_log_clear, const char *, if (_T) trace_probe_log_clear())
 
+extern const char *trace_probe_err_text[];
+
 #define trace_probe_log_err(offs, err)	\
 	__trace_probe_log_err(offs, TP_ERR_##err)
 

660 lines not on this page · raw message

index 5033fffd71c5..97d8ca747fd1 100644
@@ -890,6 +890,16 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context *
 		ctx->struct_btf = NULL;
 	}
 
+	if (ctx->btf) {
+		id = btf_find_by_name_kind(ctx->btf, sname, BTF_KIND_STRUCT);
+		if (id > 0) {
+			btf_get(ctx->btf);
+			ctx->struct_btf = ctx->btf;
+			ctx->last_struct = btf_type_by_id(ctx->struct_btf, id);
+			return 0;
+		}
+	}
+
 	id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf);
 	if (id < 0)
 		return id;

@@ -965,7 +975,8 @@ static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
 
 	if (!(tparg_is_event_probe(ctx->flags) ||
 	      tparg_is_function_entry(ctx->flags) ||
-	      tparg_is_function_return(ctx->flags))) {
+	      tparg_is_function_return(ctx->flags) ||
+	      tparg_is_wprobe(ctx->flags))) {
 		trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 		return -EOPNOTSUPP;
 	}

index e08f17c99138..8c495097acf2 100644
@@ -437,6 +437,11 @@ static inline bool tparg_is_event_probe(unsigned int flags)
 	return !!(flags & TPARG_FL_TEVENT);
 }
 
+static inline bool tparg_is_wprobe(unsigned int flags)
+{
+	return !!(flags & TPARG_FL_WPROBE);
+}
+
 /* Each typecast consumes nested level. So the max number of typecast is 8. */
 #define TRACEPROBE_MAX_NESTED_LEVEL 8
 

index 29eae708ae7b..a6ea7344dc41 100644
@@ -27,6 +27,7 @@
 #include <asm/ptrace.h>
 
 #include "trace.h"
+#include "trace_btf.h"
 #include "trace_dynevent.h"
 #include "trace_probe.h"
 #include "trace_probe_kernel.h"

@@ -971,9 +972,10 @@ static int wprobe_trigger_print(struct seq_file *m,
 		seq_printf(m, ":count=%ld", data->count);
 
 	if (data->filter_str)
-		seq_printf(m, " if %s\n", data->filter_str);
-	else
-		seq_putc(m, '\n');
+		seq_printf(m, " if %s", data->filter_str);
+
+	seq_printf(m, " # offset:%d adjust:%ld\n",
+		   wprobe_data->offset, wprobe_data->adjust);
 
 	return 0;
 }

@@ -1011,6 +1013,182 @@ static void wprobe_trigger_free(struct event_trigger_data *data)
 	}
 }
 
+#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
+
+static int get_offset_of_field(struct btf *btf, const struct btf_type *type, char *field_name)
+{
+	const struct btf_member *field;
+	const struct btf_type *mtype;
+	int bitoffs = 0;
+	u32 anon_offs;
+	char *next;
+
+	do {
+		next = strchr(field_name, '.');
+		if (next)
+			*next++ = '\0';
+
+		field = btf_find_struct_member(btf, type, field_name, &anon_offs, &mtype);
+		if (IS_ERR_OR_NULL(field))
+			return -ENOENT;
+
+		if (btf_type_kflag(mtype)) {
+			/* Reject bitfield member access */
+			if (BTF_MEMBER_BITFIELD_SIZE(field->offset))
+				return -EINVAL;
+			bitoffs += anon_offs + BTF_MEMBER_BIT_OFFSET(field->offset);
+		} else {
+			bitoffs += anon_offs + field->offset;
+		}
+
+		field_name = next;
+		if (next) {
+			type = btf_type_skip_modifiers(btf, field->type, NULL);
+			if (!type)
+				return -ENOENT;
+		}
+	} while (next);
+	return bitoffs / BITS_PER_BYTE;
+}
+
+/* btf_put(NULL) is acceptable. */
+DEFINE_FREE(btf_put, struct btf *, btf_put(_T))
+
+/* parse typecast: (TYPE[,ASGN])EVENT_FIELD->FIELD[.SUBFIELD...] and set adjust. */
+static int wprobe_trigger_typecast_parse(char **field_str_ptr,
+					 struct trace_event_file *file,
+					 struct wprobe_trigger_data *wprobe_data,
+					 const char *glob)
+{
+	struct btf *btf __free(btf_put) = NULL;
+	const struct btf_type *type;
+	char *assign_field;
+	char *event_field;
+	char *type_field;
+	char *type_name;
+	char *offs;
+	long val = 0;
+	int id;
+	int adjust;
+
+	type_name = *field_str_ptr + 1;
+	event_field = strchr(type_name, ')');
+	if (!event_field) {
+		wprobe_trigger_log_err(file, glob, type_name - glob, DEREF_OPEN_BRACE);
+		return -EINVAL;
+	}
+	*event_field++ = '\0';
+
+	/* Check the optional assign field. */
+	assign_field = strchr(type_name, ',');
+	if (assign_field)
+		*assign_field++ = '\0';
+
+	/* Get the type field name. */
+	type_field = strstr(event_field, "->");
+	if (!type_field) {
+		wprobe_trigger_log_err(file, glob, event_field - glob, TYPECAST_REQ_FIELD);
+		return -EINVAL;
+	}
+	*type_field = '\0';
+	type_field += 2;
+
+	offs = strpbrk(type_field, "+-");
+	if (offs) {
+		if (kstrtol(offs, 0, &val) < 0) {
+			wprobe_trigger_log_err(file, glob, offs - glob, BAD_DEREF_OFFS);
+			return -EINVAL;
+		}
+		*offs = '\0';
+	}
+
+	/* find type from BTF */
+	id = bpf_find_btf_id(type_name, BTF_KIND_STRUCT, &btf);
+	if (id < 0) {
+		wprobe_trigger_log_err(file, glob, type_name - glob, BAD_BTF_TID);
+		return id;
+	}
+
+	type = btf_type_by_id(btf, id);
+	if (!type) {
+		wprobe_trigger_log_err(file, glob, type_name - glob, BAD_BTF_TID);
+		return -EINVAL;
+	}
+
+	adjust = get_offset_of_field(btf, type, type_field);
+	if (adjust < 0) {
+		wprobe_trigger_log_err(file, glob, type_field - glob, NO_BTF_FIELD);
+		return adjust;
+	}
+	wprobe_data->adjust = adjust + val;
+
+	if (assign_field) {
+		/* assign_field should be a struct field */
+		adjust = get_offset_of_field(btf, type, assign_field);
+		if (adjust < 0) {
+			wprobe_trigger_log_err(file, glob, assign_field - glob, NO_BTF_FIELD);
+			return adjust;
+		}
+		wprobe_data->adjust -= adjust;
+	}
+
+	*field_str_ptr = event_field;
+	return 0;
+}
+#else
+static int wprobe_trigger_typecast_parse(char **field_str_ptr,
+					 struct trace_event_file *file,
+					 struct wprobe_trigger_data *wprobe_data,
+					 const char *glob)
+{
+	wprobe_trigger_log_err(file, glob, *field_str_ptr - glob, NOSUP_BTFARG);
+	return -EOPNOTSUPP;
+}
+#endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */
+
+static int wprobe_trigger_field_parse(char *field_str, struct trace_event_file *file,
+					struct wprobe_trigger_data *wprobe_data,
+					const char *glob)
+{
+	struct ftrace_event_field *field;
+	char *offs;
+
+	if (field_str[0] == '(') {
+		int ret = wprobe_trigger_typecast_parse(&field_str, file, wprobe_data, glob);
+
+		if (ret < 0)
+			return ret;
+	} else {
+		offs = strpbrk(field_str, "+-");
+		if (offs) {
+			long val;
+
+			if (kstrtol(offs, 0, &val) < 0) {
+				wprobe_trigger_log_err(file, glob, offs - glob, BAD_DEREF_OFFS);
+				return -EINVAL;
+			}
+			wprobe_data->adjust = val;
+			*offs = '\0';
+		}
+	}
+
+	field = trace_find_event_field(file->event_call, field_str);
+	if (!field) {
+		wprobe_trigger_log_err(file, glob, field_str - glob, NO_EVENT_FIELD);
+		return -ENOENT;
+	}
+	if (field->size != sizeof(void *)) {
+		wprobe_trigger_log_err(file, glob, field_str - glob, WPROBE_BAD_FIELD);
+		return -ENOEXEC;
+	}
+	wprobe_data->offset = field->offset;
+	wprobe_data->field = kstrdup(field_str, GFP_KERNEL);
+	if (!wprobe_data->field)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int wprobe_trigger_cmd_parse(struct event_command *cmd_ops,
 				    struct trace_event_file *file,
 				    char *glob, char *cmd,

@@ -1023,10 +1201,8 @@ static int wprobe_trigger_cmd_parse(struct event_command *cmd_ops,
 	struct wprobe_trigger_data *wprobe_data = NULL;
 	struct event_trigger_data *trigger_data = NULL;
 	struct trace_event_file *wprobe_file;
+	char *event_str, *comment;
 	struct trace_array *tr = file->tr;
-	char *event_str, *field_str, *comment;
-	struct ftrace_event_field *field;
-	struct trace_event_call *event;
 	bool remove, clear = false;
 	unsigned long orig_addr = 0;
 	struct trace_wprobe *tw;

@@ -1087,86 +1263,24 @@ static int wprobe_trigger_cmd_parse(struct event_command *cmd_ops,
 
 	/* clear_wprobe does not need field, but can have optional field. */
 	if (!clear) {
-		char *offs;
+		char *field_str = strsep(&param, ":");
 
-		/* Find target field, which must be equivalent to "void *" */
-		field_str = strsep(&param, ":");
 		if (!field_str) {
 			wprobe_trigger_log_err(file, glob, strlen(glob), WPROBE_NEED_FIELD);
 			ret = -EINVAL;
 			goto out_free;
 		}
-
-		offs = strpbrk(field_str, "+-");
-		if (offs) {
-			long val;
-
-			if (kstrtol(offs, 0, &val) < 0) {
-				wprobe_trigger_log_err(file, glob, offs - glob, BAD_DEREF_OFFS);
-				ret = -EINVAL;
-				goto out_free;
-			}
-			wprobe_data->adjust = val;
-			*offs = '\0';
-		}
-
-		event = file->event_call;
-		field = trace_find_event_field(event, field_str);
-		if (!field) {
-			wprobe_trigger_log_err(file, glob, field_str - glob, NO_EVENT_FIELD);
-			ret = -ENOENT;
-			goto out_free;
-		}
-
-		if (field->size != sizeof(void *)) {
-			wprobe_trigger_log_err(file, glob, field_str - glob, WPROBE_BAD_FIELD);
-			ret = -ENOEXEC;
-			goto out_free;
-		}
-		wprobe_data->offset = field->offset;
-		wprobe_data->field = kstrdup(field_str, GFP_KERNEL);
-		if (!wprobe_data->field) {
-			ret = -ENOMEM;
+		ret = wprobe_trigger_field_parse(field_str, file, wprobe_data, glob);
+		if (ret < 0)
 			goto out_free;
-		}
-	} else if (param && (isalpha(param[0]) || param[0] == '_')) {
+	} else if (param && (isalpha(param[0]) || param[0] == '_' || param[0] == '(')) {
 		if (strncmp(param, "count=", 6) != 0 &&
 		    strncmp(param, "unlimited", 9) != 0) {
-			char *offs;
-
-			field_str = strsep(&param, ":");
-			offs = strpbrk(field_str, "+-");
-			if (offs) {
-				long val;
-
-				if (kstrtol(offs, 0, &val) < 0) {
-					wprobe_trigger_log_err(file, glob, offs - glob, BAD_DEREF_OFFS);
-					ret = -EINVAL;
-					goto out_free;
-				}
-				wprobe_data->adjust = val;
-				*offs = '\0';
-			}
+			char *field_str = strsep(&param, ":");
 
-			event = file->event_call;
-			field = trace_find_event_field(event, field_str);
-			if (!field) {
-				wprobe_trigger_log_err(file, glob, field_str - glob, NO_EVENT_FIELD);
-				ret = -ENOENT;
+			ret = wprobe_trigger_field_parse(field_str, file, wprobe_data, glob);
+			if (ret < 0)
 				goto out_free;
-			}
-
-			if (field->size != sizeof(void *)) {
-				wprobe_trigger_log_err(file, glob, field_str - glob, WPROBE_BAD_FIELD);
-				ret = -ENOEXEC;
-				goto out_free;
-			}
-			wprobe_data->offset = field->offset;
-			wprobe_data->field = kstrdup(field_str, GFP_KERNEL);
-			if (!wprobe_data->field) {
-				ret = -ENOMEM;
-				goto out_free;
-			}
 		}
 	}